Also dump system process threads halfway through the watchdog interval

This gives us a snapshot of what the system process was doing after 30 seconds
of apparent inactivity as well as after 1 minute, to help distinguishing actual
deadlocks from too-slow progress, livelock, etc.

Change-Id: I19758861d1b25f298e88788e8f1c7ec7bf828823

Author: Christopher Tate

services/java/com/android/server/Watchdog.java

@@ -54,7 +54,8 @@ public class Watchdog extends Thread {
     static final int MONITOR = 2718;
     static final int GLOBAL_PSS = 2719;
 
-    static final int TIME_TO_WAIT = DB ? 15*1000 : 60*1000;
+    static final int TIME_TO_RESTART = DB ? 15*1000 : 60*1000;
+    static final int TIME_TO_WAIT = TIME_TO_RESTART / 2;
 
     static final int MEMCHECK_DEFAULT_INTERVAL = DB ? 30 : 30*60; // 30 minutes
     static final int MEMCHECK_DEFAULT_LOG_REALTIME_INTERVAL = DB ? 60 : 2*60*60;      // 2 hours
@@ -792,6 +793,7 @@ static long computeCalendarTime(Calendar c, long curTime,
 
     @Override
     public void run() {
+        boolean waitedHalf = false;
         while (true) {
             mCompleted = false;
             mHandler.sendEmptyMessage(MONITOR);
@@ -801,7 +803,7 @@ public void run() {
 
                 // NOTE: We use uptimeMillis() here because we do not want to increment the time we
                 // wait while asleep. If the device is asleep then the thing that we are waiting
-                // to timeout on is asleep as well and won't have a chance to run. Causing a false
+                // to timeout on is asleep as well and won't have a chance to run, causing a false
                 // positive on when to kill things.
                 long start = SystemClock.uptimeMillis();
                 while (timeout > 0 && !mForceKillSystem) {
@@ -815,6 +817,17 @@ public void run() {
 
                 if (mCompleted && !mForceKillSystem) {
                     // The monitors have returned.
+                    waitedHalf = false;
+                    continue;
+                }
+
+                if (!waitedHalf) {
+                    // We've waited half the deadlock-detection interval.  Pull a stack
+                    // trace and wait another half.
+                    ArrayList pids = new ArrayList();
+                    pids.add(Process.myPid());
+                    File stack = ActivityManagerService.dumpStackTraces(true, pids);
+                    waitedHalf = true;
                     continue;
                 }
             }
@@ -829,7 +842,9 @@ public void run() {
             ArrayList pids = new ArrayList();
             pids.add(Process.myPid());
             if (mPhonePid > 0) pids.add(mPhonePid);
-            File stack = ActivityManagerService.dumpStackTraces(pids);
+            // Pass !waitedHalf so that just in case we somehow wind up here without having
+            // dumped the halfway stacks, we properly re-initialize the trace file.
+            File stack = ActivityManagerService.dumpStackTraces(!waitedHalf, pids);
 
             // Give some extra time to make sure the stack traces get written.
             // The system's been hanging for a minute, another second or two won't hurt much.
@@ -845,6 +860,8 @@ public void run() {
             } else {
                 Slog.w(TAG, "Debugger connected: Watchdog is *not* killing the system process");
             }
+
+            waitedHalf = false;
         }
     }
 }

services/java/com/android/server/am/ActivityManagerService.java

@@ -4779,10 +4779,13 @@ private final void appDiedLocked(ProcessRecord app, int pid,
 
     /**
      * If a stack trace dump file is configured, dump process stack traces.
+     * @param clearTraces causes the dump file to be erased prior to the new
+     *    traces being written, if true; when false, the new traces will be
+     *    appended to any existing file content.
      * @param pids of dalvik VM processes to dump stack traces for
      * @return file containing stack traces, or null if no dump file is configured
      */
-    public static File dumpStackTraces(ArrayList<Integer> pids) {
+    public static File dumpStackTraces(boolean clearTraces, ArrayList<Integer> pids) {
         String tracesPath = SystemProperties.get("dalvik.vm.stack-trace-file", null);
         if (tracesPath == null || tracesPath.length() == 0) {
             return null;
@@ -4794,7 +4797,7 @@ public static File dumpStackTraces(ArrayList<Integer> pids) {
             if (!tracesDir.exists()) tracesFile.mkdirs();
             FileUtils.setPermissions(tracesDir.getPath(), 0775, -1, -1);  // drwxrwxr-x
 
-            if (tracesFile.exists()) tracesFile.delete();
+            if (clearTraces && tracesFile.exists()) tracesFile.delete();
             tracesFile.createNewFile();
             FileUtils.setPermissions(tracesFile.getPath(), 0666, -1, -1); // -rw-rw-rw-
         } catch (IOException e) {
@@ -4869,7 +4872,7 @@ final void appNotResponding(ProcessRecord app, HistoryRecord activity,
             }
         }
 
-        File tracesFile = dumpStackTraces(pids);
+        File tracesFile = dumpStackTraces(true, pids);
 
         // Log the ANR to the main log.
         StringBuilder info = mStringBuilder;