add failed pod handler and tests

Author: nicktrn

apps/supervisor/src/env.ts

@@ -59,6 +59,10 @@ const Env = z.object({
   POD_CLEANER_ENABLED: BoolEnv.default(true),
   POD_CLEANER_INTERVAL_MS: z.coerce.number().int().default(10000),
   POD_CLEANER_BATCH_SIZE: z.coerce.number().int().default(500),
+
+  // Failed pod handler
+  FAILED_POD_HANDLER_ENABLED: BoolEnv.default(true),
+  FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS: z.coerce.number().int().default(1000),
 });
 
 export const env = Env.parse(stdEnv);

apps/supervisor/src/index.ts

@@ -22,6 +22,7 @@ import { createK8sApi } from "./clients/kubernetes.js";
 import { collectDefaultMetrics } from "prom-client";
 import { register } from "./metrics.js";
 import { PodCleaner } from "./services/podCleaner.js";
+import { FailedPodHandler } from "./services/failedPodHandler.js";
 
 if (env.METRICS_COLLECT_DEFAULTS) {
   collectDefaultMetrics({ register });
@@ -35,7 +36,9 @@ class ManagedSupervisor {
   private readonly logger = new SimpleStructuredLogger("managed-worker");
   private readonly resourceMonitor: ResourceMonitor;
   private readonly checkpointClient?: CheckpointClient;
+
   private readonly podCleaner?: PodCleaner;
+  private readonly failedPodHandler?: FailedPodHandler;
 
   private readonly isKubernetes = isKubernetesEnvironment(env.KUBERNETES_FORCE_ENABLED);
   private readonly warmStartUrl = env.TRIGGER_WARM_START_URL;
@@ -53,6 +56,13 @@ class ManagedSupervisor {
       });
     }
 
+    if (env.FAILED_POD_HANDLER_ENABLED) {
+      this.failedPodHandler = new FailedPodHandler({
+        namespace: env.KUBERNETES_NAMESPACE,
+        reconnectIntervalMs: env.FAILED_POD_HANDLER_RECONNECT_INTERVAL_MS,
+      });
+    }
+
     if (this.warmStartUrl) {
       this.logger.log("[ManagedWorker] 🔥 Warm starts enabled", {
         warmStartUrl: this.warmStartUrl,
@@ -293,6 +303,10 @@ class ManagedSupervisor {
       await this.podCleaner.start();
     }
 
+    if (this.failedPodHandler) {
+      await this.failedPodHandler.start();
+    }
+
     if (env.TRIGGER_WORKLOAD_API_ENABLED) {
       this.logger.log("[ManagedWorker] Workload API enabled", {
         protocol: env.TRIGGER_WORKLOAD_API_PROTOCOL,
@@ -316,6 +330,10 @@ class ManagedSupervisor {
     if (this.podCleaner) {
       await this.podCleaner.stop();
     }
+
+    if (this.failedPodHandler) {
+      await this.failedPodHandler.stop();
+    }
   }
 }