🤖 perf: reduce SSH backoff max to 10s with jitter (#1249)

Reduces max SSH backoff from 60s to 10s for faster recovery after
transient failures.

**Changes:**
- Backoff schedule: `[1, 2, 4, 7, 10]` seconds (was `[1, 5, 10, 20, 40,
60]`)
- Add ±20% jitter to prevent thundering herd when different hosts
recover simultaneously
- Same-host serialization via singleflighting remains unchanged (herd
only released on success)

**Tests added:**
- Backoff caps at ~10s with jitter
- Callers waking from backoff share single probe

---
_Generated with `mux` • Model: `anthropic:claude-opus-4-5` • Thinking:
`high`_

Author: ammar-agent

src/node/runtime/sshConnectionPool.test.ts

@@ -185,6 +185,26 @@ describe("SSHConnectionPool", () => {
       expect(health?.backoffUntil).toBeDefined();
     });
 
+    test("backoff caps at ~10s with jitter", () => {
+      const pool = new SSHConnectionPool();
+      const config: SSHRuntimeConfig = {
+        host: "test.example.com",
+        srcBaseDir: "/work",
+      };
+
+      // Report many failures to hit the cap
+      for (let i = 0; i < 10; i++) {
+        pool.reportFailure(config, "Connection refused");
+      }
+
+      const health = pool.getConnectionHealth(config)!;
+      const backoffMs = health.backoffUntil!.getTime() - Date.now();
+
+      // Max base is 10s, jitter adds ±20%, so max is ~12s (10 * 1.2)
+      expect(backoffMs).toBeGreaterThan(7_500); // 10 * 0.8 - some tolerance
+      expect(backoffMs).toBeLessThanOrEqual(12_500); // 10 * 1.2 + some tolerance
+    });
+
     test("resetBackoff clears backoff state after failed probe", async () => {
       const pool = new SSHConnectionPool();
       const config: SSHRuntimeConfig = {
@@ -317,5 +337,52 @@ describe("SSHConnectionPool", () => {
       // Only 1 failure should be recorded (not 3) - proves singleflighting worked
       expect(pool.getConnectionHealth(config)?.consecutiveFailures).toBe(1);
     });
+
+    test("callers waking from backoff share single probe (herd only released on success)", async () => {
+      const pool = new SSHConnectionPool();
+      const config: SSHRuntimeConfig = {
+        host: "test.example.com",
+        srcBaseDir: "/work",
+      };
+
+      // Put connection in backoff
+      pool.reportFailure(config, "Initial failure");
+      expect(pool.getConnectionHealth(config)?.consecutiveFailures).toBe(1);
+
+      let probeCount = 0;
+      const sleepResolvers: Array<() => void> = [];
+
+      // Start 3 waiters - they'll all sleep through backoff
+      const waiters = [1, 2, 3].map(() =>
+        pool.acquireConnection(config, {
+          sleep: () =>
+            new Promise<void>((resolve) => {
+              sleepResolvers.push(() => {
+                // When sleep resolves, simulate recovery (mark healthy)
+                // This happens during the first probe - all waiters share it
+                if (probeCount === 0) {
+                  probeCount++;
+                  pool.markHealthy(config);
+                }
+                resolve();
+              });
+            }),
+        })
+      );
+
+      // Let all sleepers proceed
+      await Promise.resolve(); // Let all acquireConnection calls reach sleep
+      expect(sleepResolvers.length).toBe(3);
+
+      // Wake them all up "simultaneously"
+      sleepResolvers.forEach((resolve) => resolve());
+
+      // All should succeed
+      await Promise.all(waiters);
+
+      // Only one "probe" (markHealthy) should have happened
+      expect(probeCount).toBe(1);
+      expect(pool.getConnectionHealth(config)?.status).toBe("healthy");
+    });
   });
 });

src/node/runtime/sshConnectionPool.ts

@@ -54,9 +54,18 @@ export interface ConnectionHealth {
 }
 
 /**
- * Backoff schedule in seconds: 1s → 5s → 10s → 20s → 40s → 60s (cap)
+ * Backoff schedule in seconds: 1s → 2s → 4s → 7s → 10s (cap)
+ * Kept short to avoid blocking user actions; thundering herd is mitigated by jitter.
  */
-const BACKOFF_SCHEDULE = [1, 5, 10, 20, 40, 60];
+const BACKOFF_SCHEDULE = [1, 2, 4, 7, 10];
+
+/**
+ * Add ±20% jitter to prevent thundering herd when multiple clients recover simultaneously.
+ */
+function withJitter(seconds: number): number {
+  const jitterFactor = 0.8 + Math.random() * 0.4; // 0.8 to 1.2
+  return seconds * jitterFactor;
+}
 
 /**
  * Time after which a "healthy" connection should be re-probed.
@@ -315,7 +324,7 @@ export class SSHConnectionPool {
     const current = this.health.get(key);
     const failures = (current?.consecutiveFailures ?? 0) + 1;
     const backoffIndex = Math.min(failures - 1, BACKOFF_SCHEDULE.length - 1);
-    const backoffSecs = BACKOFF_SCHEDULE[backoffIndex];
+    const backoffSecs = withJitter(BACKOFF_SCHEDULE[backoffIndex]);
 
     this.health.set(key, {
       status: "unhealthy",
@@ -326,7 +335,7 @@ export class SSHConnectionPool {
     });
 
     log.warn(
-      `SSH connection failed (${failures} consecutive). Backoff for ${backoffSecs}s. Error: ${error}`
+      `SSH connection failed (${failures} consecutive). Backoff for ${backoffSecs.toFixed(1)}s. Error: ${error}`
     );
   }