add missing prod timeline events

Author: nicktrn

apps/supervisor/src/workloadManager/docker.ts

@@ -28,6 +28,7 @@ export class DockerWorkloadManager implements WorkloadManager {
       "run",
       "--detach",
       `--network=${env.DOCKER_NETWORK}`,
+      `--env=TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`,
       `--env=TRIGGER_ENV_ID=${opts.envId}`,
       `--env=TRIGGER_RUN_ID=${opts.runFriendlyId}`,
       `--env=TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`,

apps/supervisor/src/workloadManager/kubernetes.ts

@@ -61,6 +61,10 @@ export class KubernetesWorkloadManager implements WorkloadManager {
                 ],
                 resources: this.#getResourcesForMachine(opts.machine),
                 env: [
+                  {
+                    name: "TRIGGER_POD_SCHEDULED_AT_MS",
+                    value: Date.now().toString(),
+                  },
                   {
                     name: "TRIGGER_RUN_ID",
                     value: opts.runFriendlyId,

packages/cli-v3/src/entryPoints/dev-run-controller.ts

@@ -516,8 +516,6 @@ export class DevRunController {
       snapshot: snapshot.friendlyId,
     });
 
-    // TODO: We may already be executing this run, this may be a new attempt
-    //  This is the only case where incrementing the attempt number is allowed
     this.enterRunPhase(run, snapshot);
 
     const metrics = [
@@ -543,9 +541,6 @@ export class DevRunController {
     try {
       return await this.executeRun({ run, snapshot, execution, envVars, metrics });
     } catch (error) {
-      // TODO: Handle the case where we're in the warm start phase or executing a new run
-      // This can happen if we kill the run while it's still executing, e.g. after receiving an attempt number mismatch
-
       logger.debug("Error while executing attempt", {
         error,
       });
@@ -574,8 +569,6 @@ export class DevRunController {
           error: completionResult.error,
         });
 
-        // TODO: Maybe we should keep retrying for a while longer
-
         this.runFinished();
         return;
       }

packages/cli-v3/src/entryPoints/managed-run-controller.ts

@@ -8,6 +8,7 @@ import {
   type CompleteRunAttemptResult,
   HeartbeatService,
   type RunExecutionData,
+  type TaskRunExecutionMetrics,
   type TaskRunExecutionResult,
   type TaskRunFailedExecutionResult,
   WorkerManifest,
@@ -50,6 +51,9 @@ const Env = z.object({
   TRIGGER_RUNNER_ID: z.string(),
   TRIGGER_METADATA_URL: z.string().optional(),
 
+  // Timeline metrics
+  TRIGGER_POD_SCHEDULED_AT_MS: z.coerce.date(),
+
   // May be overridden
   TRIGGER_SUPERVISOR_API_PROTOCOL: z.enum(["http", "https"]),
   TRIGGER_SUPERVISOR_API_DOMAIN: z.string(),
@@ -734,10 +738,14 @@ class ManagedRunController {
   private async startAndExecuteRunAttempt({
     runFriendlyId,
     snapshotFriendlyId,
+    dequeuedAt,
+    podScheduledAt,
     isWarmStart = false,
   }: {
     runFriendlyId: string;
     snapshotFriendlyId: string;
+    dequeuedAt?: Date;
+    podScheduledAt?: Date;
     isWarmStart?: boolean;
   }) {
     if (!this.socket) {
@@ -749,6 +757,8 @@ class ManagedRunController {
       snapshot: { friendlyId: snapshotFriendlyId },
     });
 
+    const attemptStartedAt = Date.now();
+
     const start = await this.httpClient.startRunAttempt(runFriendlyId, snapshotFriendlyId, {
       isWarmStart,
     });
@@ -760,28 +770,58 @@ class ManagedRunController {
       return;
     }
 
+    const attemptDuration = Date.now() - attemptStartedAt;
+
     const { run, snapshot, execution, envVars } = start.data;
 
     logger.debug("[ManagedRunController] Started run", {
       runId: run.friendlyId,
       snapshot: snapshot.friendlyId,
     });
 
-    // TODO: We may already be executing this run, this may be a new attempt
-    //  This is the only case where incrementing the attempt number is allowed
     this.enterRunPhase(run, snapshot);
 
+    const metrics = [
+      {
+        name: "start",
+        event: "create_attempt",
+        timestamp: attemptStartedAt,
+        duration: attemptDuration,
+      },
+    ]
+      .concat(
+        dequeuedAt
+          ? [
+              {
+                name: "start",
+                event: "dequeue",
+                timestamp: dequeuedAt.getTime(),
+                duration: 0,
+              },
+            ]
+          : []
+      )
+      .concat(
+        podScheduledAt
+          ? [
+              {
+                name: "start",
+                event: "pod_scheduled",
+                timestamp: podScheduledAt.getTime(),
+                duration: 0,
+              },
+            ]
+          : []
+      ) satisfies TaskRunExecutionMetrics;
+
     const taskRunEnv = {
       ...gatherProcessEnv(),
       ...envVars,
     };
 
     try {
-      return await this.executeRun({ run, snapshot, envVars: taskRunEnv, execution });
+      return await this.executeRun({ run, snapshot, envVars: taskRunEnv, execution, metrics });
     } catch (error) {
-      // TODO: Handle the case where we're in the warm start phase or executing a new run
-      // This can happen if we kill the run while it's still executing, e.g. after receiving an attempt number mismatch
-
       console.error("Error while executing attempt", {
         error,
       });
@@ -810,8 +850,6 @@ class ManagedRunController {
           error: completionResult.error,
         });
 
-        // TODO: Maybe we should keep retrying for a while longer
-
         this.waitForNextRun();
         return;
       }
@@ -923,6 +961,7 @@ class ManagedRunController {
       this.startAndExecuteRunAttempt({
         runFriendlyId: nextRun.run.friendlyId,
         snapshotFriendlyId: nextRun.snapshot.friendlyId,
+        dequeuedAt: nextRun.dequeuedAt,
         isWarmStart: true,
       }).finally(() => {});
       return;
@@ -1032,7 +1071,10 @@ class ManagedRunController {
     snapshot,
     envVars,
     execution,
-  }: WorkloadRunAttemptStartResponseBody) {
+    metrics,
+  }: WorkloadRunAttemptStartResponseBody & {
+    metrics?: TaskRunExecutionMetrics;
+  }) {
     this.snapshotPoller.start();
 
     if (!this.taskRunProcess || !this.taskRunProcess.isPreparedForNextRun) {
@@ -1058,6 +1100,7 @@ class ManagedRunController {
       payload: {
         execution,
         traceContext: execution.run.traceContext ?? {},
+        metrics,
       },
       messageId: run.friendlyId,
       env: envVars,
@@ -1212,6 +1255,8 @@ class ManagedRunController {
       this.startAndExecuteRunAttempt({
         runFriendlyId: env.TRIGGER_RUN_ID,
         snapshotFriendlyId: env.TRIGGER_SNAPSHOT_ID,
+        dequeuedAt: new Date(),
+        podScheduledAt: env.TRIGGER_POD_SCHEDULED_AT_MS,
       }).finally(() => {});
       return;
     }