From 90b64378c1ce364d7e38e465434122dd533514a4 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Wed, 19 Mar 2025 12:55:45 +0000
Subject: [PATCH 01/26] add shutdown manager

---
 packages/core/src/v3/serverOnly/index.ts      |   2 +
 .../src/v3/serverOnly/shutdownManager.test.ts | 175 ++++++++++++++++++
 .../core/src/v3/serverOnly/shutdownManager.ts |  83 +++++++++
 packages/core/src/v3/serverOnly/singleton.ts  |   8 +
 4 files changed, 268 insertions(+)
 create mode 100644 packages/core/src/v3/serverOnly/shutdownManager.test.ts
 create mode 100644 packages/core/src/v3/serverOnly/shutdownManager.ts
 create mode 100644 packages/core/src/v3/serverOnly/singleton.ts

diff --git a/packages/core/src/v3/serverOnly/index.ts b/packages/core/src/v3/serverOnly/index.ts
index 38941c03f1..3b311c7904 100644
--- a/packages/core/src/v3/serverOnly/index.ts
+++ b/packages/core/src/v3/serverOnly/index.ts
@@ -1,3 +1,5 @@
 export * from "./checkpointClient.js";
 export * from "./checkpointTest.js";
 export * from "./httpServer.js";
+export * from "./singleton.js";
+export * from "./shutdownManager.js";
diff --git a/packages/core/src/v3/serverOnly/shutdownManager.test.ts b/packages/core/src/v3/serverOnly/shutdownManager.test.ts
new file mode 100644
index 0000000000..53af9c11f4
--- /dev/null
+++ b/packages/core/src/v3/serverOnly/shutdownManager.test.ts
@@ -0,0 +1,175 @@
+import { describe, test, expect, vi, beforeEach } from "vitest";
+import { shutdownManager } from "./shutdownManager.js";
+
+// Type assertion to access private members for testing
+type PrivateShutdownManager = {
+  handlers: Map<string, { handler: NodeJS.SignalsListener; signals: Array<"SIGTERM" | "SIGINT"> }>;
+  shutdown: (signal: "SIGTERM" | "SIGINT") => Promise<void>;
+  _reset: () => void;
+};
+
+describe("ShutdownManager", { concurrent: false }, () => {
+  const manager = shutdownManager as unknown as PrivateShutdownManager;
+  // Mock process.exit to prevent actual exit
+  const mockExit = vi.spyOn(process, "exit").mockImplementation(() => undefined as never);
+
+  beforeEach(() => {
+    // Clear all mocks and reset the manager before each test
+    vi.clearAllMocks();
+    manager._reset();
+  });
+
+  test("should successfully register a new handler", () => {
+    const handler = vi.fn();
+    shutdownManager.register("test-handler", handler);
+
+    expect(manager.handlers.has("test-handler")).toBe(true);
+    const registeredHandler = manager.handlers.get("test-handler");
+    expect(registeredHandler?.handler).toBe(handler);
+    expect(registeredHandler?.signals).toEqual(["SIGTERM", "SIGINT"]);
+  });
+
+  test("should throw error when registering duplicate handler name", () => {
+    const handler = vi.fn();
+    shutdownManager.register("duplicate-handler", handler);
+
+    expect(() => {
+      shutdownManager.register("duplicate-handler", handler);
+    }).toThrow('Shutdown handler "duplicate-handler" already registered');
+  });
+
+  test("should register handler with custom signals", () => {
+    const handler = vi.fn();
+    shutdownManager.register("custom-signals", handler, ["SIGTERM"]);
+
+    const registeredHandler = manager.handlers.get("custom-signals");
+    expect(registeredHandler?.signals).toEqual(["SIGTERM"]);
+  });
+
+  test("should call registered handlers when shutdown is triggered", async () => {
+    const handler1 = vi.fn();
+    const handler2 = vi.fn();
+
+    shutdownManager.register("handler1", handler1);
+    shutdownManager.register("handler2", handler2);
+
+    await manager.shutdown("SIGTERM");
+
+    expect(handler1).toHaveBeenCalledWith("SIGTERM");
+    expect(handler2).toHaveBeenCalledWith("SIGTERM");
+    expect(mockExit).toHaveBeenCalledWith(128 + 15); // SIGTERM number
+  });
+
+  test("should only call handlers registered for specific signal", async () => {
+    const handler1 = vi.fn();
+    const handler2 = vi.fn();
+
+    shutdownManager.register("handler1", handler1, ["SIGTERM"]);
+    shutdownManager.register("handler2", handler2, ["SIGINT"]);
+
+    await manager.shutdown("SIGTERM");
+
+    expect(handler1).toHaveBeenCalledWith("SIGTERM");
+    expect(handler2).not.toHaveBeenCalled();
+    expect(mockExit).toHaveBeenCalledWith(128 + 15);
+  });
+
+  test("should handle errors in shutdown handlers gracefully", async () => {
+    const successHandler = vi.fn();
+    const errorHandler = vi.fn().mockRejectedValue(new Error("Handler failed"));
+
+    shutdownManager.register("success-handler", successHandler);
+    shutdownManager.register("error-handler", errorHandler);
+
+    await manager.shutdown("SIGTERM");
+
+    expect(successHandler).toHaveBeenCalledWith("SIGTERM");
+    expect(errorHandler).toHaveBeenCalledWith("SIGTERM");
+    expect(mockExit).toHaveBeenCalledWith(128 + 15);
+  });
+
+  test("should only run shutdown sequence once even if called multiple times", async () => {
+    const handler = vi.fn();
+    shutdownManager.register("test-handler", handler);
+
+    await Promise.all([manager.shutdown("SIGTERM"), manager.shutdown("SIGTERM")]);
+
+    expect(handler).toHaveBeenCalledTimes(1);
+    expect(mockExit).toHaveBeenCalledTimes(1);
+    expect(mockExit).toHaveBeenCalledWith(128 + 15);
+  });
+
+  test("should exit with correct signal number", async () => {
+    const handler = vi.fn();
+    shutdownManager.register("test-handler", handler);
+
+    await manager.shutdown("SIGINT");
+    expect(mockExit).toHaveBeenCalledWith(128 + 2); // SIGINT number
+
+    vi.clearAllMocks();
+    manager._reset();
+    shutdownManager.register("test-handler", handler);
+
+    await manager.shutdown("SIGTERM");
+    expect(mockExit).toHaveBeenCalledWith(128 + 15); // SIGTERM number
+  });
+
+  test("should only exit after all handlers have finished", async () => {
+    const sequence: string[] = [];
+
+    const handler1 = vi.fn().mockImplementation(async () => {
+      sequence.push("handler1 start");
+      await new Promise((resolve) => setTimeout(resolve, 10));
+      sequence.push("handler1 end");
+    });
+
+    const handler2 = vi.fn().mockImplementation(async () => {
+      sequence.push("handler2 start");
+      await new Promise((resolve) => setTimeout(resolve, 20));
+      sequence.push("handler2 end");
+    });
+
+    const handler3 = vi.fn().mockImplementation(async () => {
+      sequence.push("handler3 start");
+      await new Promise((resolve) => setTimeout(resolve, 5));
+      sequence.push("handler3 end");
+    });
+
+    // Store the current mock implementation
+    const currentExit = mockExit.getMockImplementation();
+
+    // Override with our sequence-tracking implementation
+    mockExit.mockImplementation((code?: number | string | null) => {
+      sequence.push("exit");
+      return undefined as never;
+    });
+
+    shutdownManager.register("handler1", handler1);
+    shutdownManager.register("handler2", handler2);
+    shutdownManager.register("handler3", handler3);
+
+    await manager.shutdown("SIGTERM");
+
+    // Verify the execution order
+    expect(sequence).toEqual([
+      "handler1 start",
+      "handler2 start",
+      "handler3 start",
+      "handler3 end",
+      "handler1 end",
+      "handler2 end",
+      "exit",
+    ]);
+
+    // Verify the handlers were called with correct arguments
+    expect(handler1).toHaveBeenCalledWith("SIGTERM");
+    expect(handler2).toHaveBeenCalledWith("SIGTERM");
+    expect(handler3).toHaveBeenCalledWith("SIGTERM");
+    expect(mockExit).toHaveBeenCalledWith(128 + 15);
+
+    // Restore original mock implementation
+    if (currentExit) {
+      mockExit.mockImplementation(currentExit);
+    }
+  });
+});
diff --git a/packages/core/src/v3/serverOnly/shutdownManager.ts b/packages/core/src/v3/serverOnly/shutdownManager.ts
new file mode 100644
index 0000000000..c3870d93f7
--- /dev/null
+++ b/packages/core/src/v3/serverOnly/shutdownManager.ts
@@ -0,0 +1,83 @@
+import { singleton } from "./singleton.js";
+
+type ShutdownHandler = NodeJS.SignalsListener;
+// We intentionally keep these limited to avoid unexpected issues with signal handling
+type ShutdownSignal = Extract<NodeJS.Signals, "SIGTERM" | "SIGINT">;
+
+class ShutdownManager {
+  private isShuttingDown = false;
+  private signalNumbers: Record<ShutdownSignal, number> = {
+    SIGINT: 2,
+    SIGTERM: 15,
+  };
+
+  private handlers: Map<string, { handler: ShutdownHandler; signals: ShutdownSignal[] }> =
+    new Map();
+
+  constructor() {
+    process.on("SIGTERM", () => this.shutdown("SIGTERM"));
+    process.on("SIGINT", () => this.shutdown("SIGINT"));
+  }
+
+  register(
+    name: string,
+    handler: ShutdownHandler,
+    signals: ShutdownSignal[] = ["SIGTERM", "SIGINT"]
+  ) {
+    if (this.handlers.has(name)) {
+      throw new Error(`Shutdown handler "${name}" already registered`);
+    }
+    this.handlers.set(name, { handler, signals });
+  }
+
+  async shutdown(signal: ShutdownSignal) {
+    if (this.isShuttingDown) return;
+    this.isShuttingDown = true;
+
+    console.log(`\nReceived ${signal}. Starting graceful shutdown...`);
+
+    // Get handlers that are registered for this signal
+    const handlersToRun = Array.from(this.handlers.entries()).filter(([_, { signals }]) =>
+      signals.includes(signal)
+    );
+
+    try {
+      const results = await Promise.allSettled(
+        handlersToRun.map(async ([name, { handler }]) => {
+          try {
+            console.log(`Running shutdown handler: ${name}`);
+            await handler(signal);
+            console.log(`Shutdown handler completed: ${name}`);
+          } catch (error) {
+            console.error(`Shutdown handler failed: ${name}`, error);
+            throw error;
+          }
+        })
+      );
+
+      // Log any failures
+      results.forEach((result, index) => {
+        if (result.status === "rejected") {
+          const handlerEntry = handlersToRun[index];
+          if (handlerEntry) {
+            const [name] = handlerEntry;
+            console.error(`Shutdown handler "${name}" failed:`, result.reason);
+          }
+        }
+      });
+    } catch (error) {
+      console.error("Error during shutdown:", error);
+    } finally {
+      // Exit with the correct signal number
+      process.exit(128 + this.signalNumbers[signal]);
+    }
+  }
+
+  // For testing purposes only - keep this
+  private _reset() {
+    this.isShuttingDown = false;
+    this.handlers.clear();
+  }
+}
+
+export const shutdownManager = singleton("shutdownManager", () => new ShutdownManager());
diff --git a/packages/core/src/v3/serverOnly/singleton.ts b/packages/core/src/v3/serverOnly/singleton.ts
new file mode 100644
index 0000000000..8f96dc6d8c
--- /dev/null
+++ b/packages/core/src/v3/serverOnly/singleton.ts
@@ -0,0 +1,8 @@
+export function singleton<T>(name: string, getValue: () => T): T {
+  const thusly = globalThis as unknown as {
+    __trigger_singletons: Record<string, T>;
+  };
+  thusly.__trigger_singletons ??= {};
+  thusly.__trigger_singletons[name] ??= getValue();
+  return thusly.__trigger_singletons[name];
+}

From 4395062819846d3fc1b0f1c33fdfbe11b9d5e348 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Wed, 19 Mar 2025 12:56:10 +0000
Subject: [PATCH 02/26] update ai test instructions

---
 ai/references/tests.md | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ai/references/tests.md b/ai/references/tests.md
index 7ca84df718..2bb236c75b 100644
--- a/ai/references/tests.md
+++ b/ai/references/tests.md
@@ -25,6 +25,11 @@ pnpm run test ./src/components/Button.test.ts
 
 We use vitest for testing. We almost NEVER mock anything. Start with a top-level "describe", and have multiple "it" statements inside of it.
 
+New test files should be placed right next to the file being tested. For example:
+
+- Source file: `./src/services/MyService.ts`
+- Test file: `./src/services/MyService.test.ts`
+
 When writing anything that needs redis or postgresql, we have some internal "testcontainers" that are used to spin up a local instance, redis, or both.
 
 redisTest:

From 966e42e9dce0898eec351483670bc74276f2fcf4 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Wed, 19 Mar 2025 12:57:07 +0000
Subject: [PATCH 03/26] add shutdown timeout to redis-worker

---
 apps/webapp/app/env.server.ts                 |  3 ++
 apps/webapp/app/v3/commonWorker.server.ts     |  1 +
 .../app/v3/legacyRunEngineWorker.server.ts    |  1 +
 apps/webapp/app/v3/runEngine.server.ts        |  1 +
 internal-packages/redis-worker/src/worker.ts  | 30 ++++++++++++++-----
 .../run-engine/src/engine/index.ts            |  1 +
 .../run-engine/src/engine/types.ts            |  1 +
 7 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts
index 84e3c52b91..aa104fad0f 100644
--- a/apps/webapp/app/env.server.ts
+++ b/apps/webapp/app/env.server.ts
@@ -424,6 +424,7 @@ const EnvironmentSchema = z.object({
   RUN_ENGINE_QUEUE_AGE_RANDOMIZATION_BIAS: z.coerce.number().default(0.25),
   RUN_ENGINE_REUSE_SNAPSHOT_COUNT: z.coerce.number().int().default(0),
   RUN_ENGINE_MAXIMUM_ENV_COUNT: z.coerce.number().int().optional(),
+  RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000),
 
   RUN_ENGINE_WORKER_REDIS_HOST: z
     .string()
@@ -585,6 +586,7 @@ const EnvironmentSchema = z.object({
   LEGACY_RUN_ENGINE_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
   LEGACY_RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50),
   LEGACY_RUN_ENGINE_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(100),
+  LEGACY_RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000),
 
   LEGACY_RUN_ENGINE_WORKER_REDIS_HOST: z
     .string()
@@ -627,6 +629,7 @@ const EnvironmentSchema = z.object({
   COMMON_WORKER_POLL_INTERVAL: z.coerce.number().int().default(1000),
   COMMON_WORKER_IMMEDIATE_POLL_INTERVAL: z.coerce.number().int().default(50),
   COMMON_WORKER_CONCURRENCY_LIMIT: z.coerce.number().int().default(100),
+  COMMON_WORKER_SHUTDOWN_TIMEOUT_MS: z.coerce.number().int().default(60_000),
 
   COMMON_WORKER_REDIS_HOST: z
     .string()
diff --git a/apps/webapp/app/v3/commonWorker.server.ts b/apps/webapp/app/v3/commonWorker.server.ts
index 18014bfa55..d1ac24763a 100644
--- a/apps/webapp/app/v3/commonWorker.server.ts
+++ b/apps/webapp/app/v3/commonWorker.server.ts
@@ -80,6 +80,7 @@ function initializeWorker() {
     },
     pollIntervalMs: env.COMMON_WORKER_POLL_INTERVAL,
     immediatePollIntervalMs: env.COMMON_WORKER_IMMEDIATE_POLL_INTERVAL,
+    shutdownTimeoutMs: env.COMMON_WORKER_SHUTDOWN_TIMEOUT_MS,
     logger: new Logger("CommonWorker", "debug"),
     jobs: {
       "v3.deliverAlert": async ({ payload }) => {
diff --git a/apps/webapp/app/v3/legacyRunEngineWorker.server.ts b/apps/webapp/app/v3/legacyRunEngineWorker.server.ts
index 7a82d5522c..a7e1e05d8f 100644
--- a/apps/webapp/app/v3/legacyRunEngineWorker.server.ts
+++ b/apps/webapp/app/v3/legacyRunEngineWorker.server.ts
@@ -67,6 +67,7 @@ function initializeWorker() {
     },
     pollIntervalMs: env.LEGACY_RUN_ENGINE_WORKER_POLL_INTERVAL,
     immediatePollIntervalMs: env.LEGACY_RUN_ENGINE_WORKER_IMMEDIATE_POLL_INTERVAL,
+    shutdownTimeoutMs: env.LEGACY_RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS,
     logger: new Logger("LegacyRunEngineWorker", "debug"),
     jobs: {
       runHeartbeat: async ({ payload }) => {
diff --git a/apps/webapp/app/v3/runEngine.server.ts b/apps/webapp/app/v3/runEngine.server.ts
index 382302ff16..b4f6ce246d 100644
--- a/apps/webapp/app/v3/runEngine.server.ts
+++ b/apps/webapp/app/v3/runEngine.server.ts
@@ -17,6 +17,7 @@ function createRunEngine() {
       workers: env.RUN_ENGINE_WORKER_COUNT,
       tasksPerWorker: env.RUN_ENGINE_TASKS_PER_WORKER,
       pollIntervalMs: env.RUN_ENGINE_WORKER_POLL_INTERVAL,
+      shutdownTimeoutMs: env.RUN_ENGINE_WORKER_SHUTDOWN_TIMEOUT_MS,
       redis: {
         keyPrefix: "engine:",
         port: env.RUN_ENGINE_WORKER_REDIS_PORT ?? undefined,
diff --git a/internal-packages/redis-worker/src/worker.ts b/internal-packages/redis-worker/src/worker.ts
index d4fca68c6d..18e337b460 100644
--- a/internal-packages/redis-worker/src/worker.ts
+++ b/internal-packages/redis-worker/src/worker.ts
@@ -8,6 +8,7 @@ import { AnyQueueItem, SimpleQueue } from "./queue.js";
 import { nanoid } from "nanoid";
 import pLimit from "p-limit";
 import { createRedisClient } from "@internal/redis";
+import { shutdownManager } from "@trigger.dev/core/v3/serverOnly";
 
 export type WorkerCatalog = {
   [key: string]: {
@@ -44,6 +45,7 @@ type WorkerOptions<TCatalog extends WorkerCatalog> = {
   concurrency?: WorkerConcurrencyOptions;
   pollIntervalMs?: number;
   immediatePollIntervalMs?: number;
+  shutdownTimeoutMs?: number;
   logger?: Logger;
   tracer?: Tracer;
 };
@@ -69,6 +71,7 @@ class Worker<TCatalog extends WorkerCatalog> {
   private workerLoops: Promise<void>[] = [];
   private isShuttingDown = false;
   private concurrency: Required<NonNullable<WorkerOptions<TCatalog>["concurrency"]>>;
+  private shutdownTimeoutMs: number;
 
   // The p-limit limiter to control overall concurrency.
   private limiter: ReturnType<typeof pLimit>;
@@ -77,6 +80,8 @@ class Worker<TCatalog extends WorkerCatalog> {
     this.logger = options.logger ?? new Logger("Worker", "debug");
     this.tracer = options.tracer ?? trace.getTracer(options.name);
 
+    this.shutdownTimeoutMs = options.shutdownTimeoutMs ?? 60_000;
+
     const schema: QueueCatalogFromWorkerCatalog<TCatalog> = Object.fromEntries(
       Object.entries(this.options.catalog).map(([key, value]) => [key, value.schema])
     ) as QueueCatalogFromWorkerCatalog<TCatalog>;
@@ -386,22 +391,33 @@ class Worker<TCatalog extends WorkerCatalog> {
   }
 
   private setupShutdownHandlers() {
-    process.on("SIGTERM", this.shutdown.bind(this));
-    process.on("SIGINT", this.shutdown.bind(this));
+    shutdownManager.register("redis-worker", this.shutdown.bind(this));
   }
 
-  private async shutdown() {
-    if (this.isShuttingDown) return;
+  private async shutdown(signal?: NodeJS.Signals) {
+    if (this.isShuttingDown) {
+      this.logger.log("Worker already shutting down", { signal });
+      return;
+    }
+
     this.isShuttingDown = true;
-    this.logger.log("Shutting down worker loops...");
+    this.logger.log("Shutting down worker loops...", { signal });
 
     // Wait for all worker loops to finish.
-    await Promise.all(this.workerLoops);
+    await Promise.race([
+      Promise.all(this.workerLoops),
+      Worker.delay(this.shutdownTimeoutMs).then(() => {
+        this.logger.error("Worker shutdown timed out", {
+          signal,
+          shutdownTimeoutMs: this.shutdownTimeoutMs,
+        });
+      }),
+    ]);
 
     await this.subscriber?.unsubscribe();
     await this.subscriber?.quit();
     await this.queue.close();
-    this.logger.log("All workers and subscribers shut down.");
+    this.logger.log("All workers and subscribers shut down.", { signal });
   }
 
   public async stop() {
diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts
index 834652fad6..1901e64b59 100644
--- a/internal-packages/run-engine/src/engine/index.ts
+++ b/internal-packages/run-engine/src/engine/index.ts
@@ -180,6 +180,7 @@ export class RunEngine {
       concurrency: options.worker,
       pollIntervalMs: options.worker.pollIntervalMs,
       immediatePollIntervalMs: options.worker.immediatePollIntervalMs,
+      shutdownTimeoutMs: options.worker.shutdownTimeoutMs,
       logger: new Logger("RunEngineWorker", "debug"),
       jobs: {
         finishWaitpoint: async ({ payload }) => {
diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts
index aea71d605b..00656d05f3 100644
--- a/internal-packages/run-engine/src/engine/types.ts
+++ b/internal-packages/run-engine/src/engine/types.ts
@@ -12,6 +12,7 @@ export type RunEngineOptions = {
     redis: RedisOptions;
     pollIntervalMs?: number;
     immediatePollIntervalMs?: number;
+    shutdownTimeoutMs?: number;
   };
   machines: {
     defaultMachine: MachinePresetName;

From 0144edc5878decb1143561ac3dfbd1f56380f215 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Wed, 19 Mar 2025 14:58:26 +0000
Subject: [PATCH 04/26] move redis worker to packages

---
 {internal-packages => packages}/redis-worker/README.md           | 0
 {internal-packages => packages}/redis-worker/package.json        | 0
 {internal-packages => packages}/redis-worker/src/index.ts        | 0
 {internal-packages => packages}/redis-worker/src/queue.test.ts   | 0
 {internal-packages => packages}/redis-worker/src/queue.ts        | 0
 {internal-packages => packages}/redis-worker/src/worker.test.ts  | 0
 {internal-packages => packages}/redis-worker/src/worker.ts       | 0
 {internal-packages => packages}/redis-worker/tsconfig.build.json | 0
 {internal-packages => packages}/redis-worker/tsconfig.json       | 0
 {internal-packages => packages}/redis-worker/tsconfig.src.json   | 0
 {internal-packages => packages}/redis-worker/tsconfig.test.json  | 0
 {internal-packages => packages}/redis-worker/vitest.config.ts    | 0
 12 files changed, 0 insertions(+), 0 deletions(-)
 rename {internal-packages => packages}/redis-worker/README.md (100%)
 rename {internal-packages => packages}/redis-worker/package.json (100%)
 rename {internal-packages => packages}/redis-worker/src/index.ts (100%)
 rename {internal-packages => packages}/redis-worker/src/queue.test.ts (100%)
 rename {internal-packages => packages}/redis-worker/src/queue.ts (100%)
 rename {internal-packages => packages}/redis-worker/src/worker.test.ts (100%)
 rename {internal-packages => packages}/redis-worker/src/worker.ts (100%)
 rename {internal-packages => packages}/redis-worker/tsconfig.build.json (100%)
 rename {internal-packages => packages}/redis-worker/tsconfig.json (100%)
 rename {internal-packages => packages}/redis-worker/tsconfig.src.json (100%)
 rename {internal-packages => packages}/redis-worker/tsconfig.test.json (100%)
 rename {internal-packages => packages}/redis-worker/vitest.config.ts (100%)

diff --git a/internal-packages/redis-worker/README.md b/packages/redis-worker/README.md
similarity index 100%
rename from internal-packages/redis-worker/README.md
rename to packages/redis-worker/README.md
diff --git a/internal-packages/redis-worker/package.json b/packages/redis-worker/package.json
similarity index 100%
rename from internal-packages/redis-worker/package.json
rename to packages/redis-worker/package.json
diff --git a/internal-packages/redis-worker/src/index.ts b/packages/redis-worker/src/index.ts
similarity index 100%
rename from internal-packages/redis-worker/src/index.ts
rename to packages/redis-worker/src/index.ts
diff --git a/internal-packages/redis-worker/src/queue.test.ts b/packages/redis-worker/src/queue.test.ts
similarity index 100%
rename from internal-packages/redis-worker/src/queue.test.ts
rename to packages/redis-worker/src/queue.test.ts
diff --git a/internal-packages/redis-worker/src/queue.ts b/packages/redis-worker/src/queue.ts
similarity index 100%
rename from internal-packages/redis-worker/src/queue.ts
rename to packages/redis-worker/src/queue.ts
diff --git a/internal-packages/redis-worker/src/worker.test.ts b/packages/redis-worker/src/worker.test.ts
similarity index 100%
rename from internal-packages/redis-worker/src/worker.test.ts
rename to packages/redis-worker/src/worker.test.ts
diff --git a/internal-packages/redis-worker/src/worker.ts b/packages/redis-worker/src/worker.ts
similarity index 100%
rename from internal-packages/redis-worker/src/worker.ts
rename to packages/redis-worker/src/worker.ts
diff --git a/internal-packages/redis-worker/tsconfig.build.json b/packages/redis-worker/tsconfig.build.json
similarity index 100%
rename from internal-packages/redis-worker/tsconfig.build.json
rename to packages/redis-worker/tsconfig.build.json
diff --git a/internal-packages/redis-worker/tsconfig.json b/packages/redis-worker/tsconfig.json
similarity index 100%
rename from internal-packages/redis-worker/tsconfig.json
rename to packages/redis-worker/tsconfig.json
diff --git a/internal-packages/redis-worker/tsconfig.src.json b/packages/redis-worker/tsconfig.src.json
similarity index 100%
rename from internal-packages/redis-worker/tsconfig.src.json
rename to packages/redis-worker/tsconfig.src.json
diff --git a/internal-packages/redis-worker/tsconfig.test.json b/packages/redis-worker/tsconfig.test.json
similarity index 100%
rename from internal-packages/redis-worker/tsconfig.test.json
rename to packages/redis-worker/tsconfig.test.json
diff --git a/internal-packages/redis-worker/vitest.config.ts b/packages/redis-worker/vitest.config.ts
similarity index 100%
rename from internal-packages/redis-worker/vitest.config.ts
rename to packages/redis-worker/vitest.config.ts

From 1180166d5848bdcc739e7dca85b0feb734b30333 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Wed, 19 Mar 2025 15:18:49 +0000
Subject: [PATCH 05/26] add unregister method

---
 packages/core/src/v3/serverOnly/shutdownManager.ts | 4 ++++
 packages/redis-worker/src/worker.ts                | 1 +
 2 files changed, 5 insertions(+)

diff --git a/packages/core/src/v3/serverOnly/shutdownManager.ts b/packages/core/src/v3/serverOnly/shutdownManager.ts
index c3870d93f7..80cf763b54 100644
--- a/packages/core/src/v3/serverOnly/shutdownManager.ts
+++ b/packages/core/src/v3/serverOnly/shutdownManager.ts
@@ -30,6 +30,10 @@ class ShutdownManager {
     this.handlers.set(name, { handler, signals });
   }
 
+  unregister(name: string) {
+    this.handlers.delete(name);
+  }
+
   async shutdown(signal: ShutdownSignal) {
     if (this.isShuttingDown) return;
     this.isShuttingDown = true;
diff --git a/packages/redis-worker/src/worker.ts b/packages/redis-worker/src/worker.ts
index 18e337b460..d6d4d0979b 100644
--- a/packages/redis-worker/src/worker.ts
+++ b/packages/redis-worker/src/worker.ts
@@ -421,6 +421,7 @@ class Worker<TCatalog extends WorkerCatalog> {
   }
 
   public async stop() {
+    shutdownManager.unregister("redis-worker");
     await this.shutdown();
   }
 }

From 33d1f4e653cbcd10acd17ccb4b937e13d6d80972 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Wed, 19 Mar 2025 17:05:49 +0000
Subject: [PATCH 06/26] prep for publishing package

---
 apps/webapp/app/v3/commonWorker.server.ts     |   2 +-
 .../app/v3/legacyRunEngineWorker.server.ts    |   2 +-
 apps/webapp/package.json                      |   2 +-
 internal-packages/run-engine/package.json     |   4 +-
 .../run-engine/src/engine/index.ts            |   2 +-
 .../run-engine/src/engine/types.ts            |   2 +-
 packages/redis-worker/package.json            |  75 ++++++++----
 packages/redis-worker/tsconfig.json           |  15 ++-
 packages/redis-worker/tsconfig.src.json       |  19 +--
 packages/redis-worker/tsconfig.test.json      |  19 +--
 pnpm-lock.yaml                                | 112 +++++++++---------
 11 files changed, 136 insertions(+), 118 deletions(-)

diff --git a/apps/webapp/app/v3/commonWorker.server.ts b/apps/webapp/app/v3/commonWorker.server.ts
index d1ac24763a..2bc6c43b4e 100644
--- a/apps/webapp/app/v3/commonWorker.server.ts
+++ b/apps/webapp/app/v3/commonWorker.server.ts
@@ -1,4 +1,4 @@
-import { Worker as RedisWorker } from "@internal/redis-worker";
+import { Worker as RedisWorker } from "@trigger.dev/redis-worker";
 import { Logger } from "@trigger.dev/core/logger";
 import { z } from "zod";
 import { env } from "~/env.server";
diff --git a/apps/webapp/app/v3/legacyRunEngineWorker.server.ts b/apps/webapp/app/v3/legacyRunEngineWorker.server.ts
index a7e1e05d8f..989a206da5 100644
--- a/apps/webapp/app/v3/legacyRunEngineWorker.server.ts
+++ b/apps/webapp/app/v3/legacyRunEngineWorker.server.ts
@@ -1,4 +1,4 @@
-import { Worker as RedisWorker } from "@internal/redis-worker";
+import { Worker as RedisWorker } from "@trigger.dev/redis-worker";
 import { Logger } from "@trigger.dev/core/logger";
 import { z } from "zod";
 import { env } from "~/env.server";
diff --git a/apps/webapp/package.json b/apps/webapp/package.json
index 25cd5f3844..1a0d040206 100644
--- a/apps/webapp/package.json
+++ b/apps/webapp/package.json
@@ -53,7 +53,7 @@
     "@internal/run-engine": "workspace:*",
     "@internal/zod-worker": "workspace:*",
     "@internal/redis": "workspace:*",
-    "@internal/redis-worker": "workspace:*",
+    "@trigger.dev/redis-worker": "workspace:*",
     "@internationalized/date": "^3.5.1",
     "@lezer/highlight": "^1.1.6",
     "@opentelemetry/api": "1.9.0",
diff --git a/internal-packages/run-engine/package.json b/internal-packages/run-engine/package.json
index 450ea1cb06..cbfc74b4c9 100644
--- a/internal-packages/run-engine/package.json
+++ b/internal-packages/run-engine/package.json
@@ -15,7 +15,7 @@
   },
   "dependencies": {
     "@internal/redis": "workspace:*",
-    "@internal/redis-worker": "workspace:*",
+    "@trigger.dev/redis-worker": "workspace:*",
     "@internal/tracing": "workspace:*",
     "@trigger.dev/core": "workspace:*",
     "@trigger.dev/database": "workspace:*",
@@ -39,4 +39,4 @@
     "build": "pnpm run clean && tsc -p tsconfig.build.json",
     "dev": "tsc --watch  -p tsconfig.build.json"
   }
-}
\ No newline at end of file
+}
diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts
index 1901e64b59..b8c672cdc3 100644
--- a/internal-packages/run-engine/src/engine/index.ts
+++ b/internal-packages/run-engine/src/engine/index.ts
@@ -1,5 +1,5 @@
 import { createRedisClient, Redis } from "@internal/redis";
-import { Worker } from "@internal/redis-worker";
+import { Worker } from "@trigger.dev/redis-worker";
 import { Attributes, Span, SpanKind, trace, Tracer } from "@internal/tracing";
 import { assertExhaustive } from "@trigger.dev/core";
 import { Logger } from "@trigger.dev/core/logger";
diff --git a/internal-packages/run-engine/src/engine/types.ts b/internal-packages/run-engine/src/engine/types.ts
index 00656d05f3..34928b8d1e 100644
--- a/internal-packages/run-engine/src/engine/types.ts
+++ b/internal-packages/run-engine/src/engine/types.ts
@@ -1,4 +1,4 @@
-import { type WorkerConcurrencyOptions } from "@internal/redis-worker";
+import { type WorkerConcurrencyOptions } from "@trigger.dev/redis-worker";
 import { Tracer } from "@internal/tracing";
 import { MachinePreset, MachinePresetName, QueueOptions, RetryOptions } from "@trigger.dev/core/v3";
 import { PrismaClient } from "@trigger.dev/database";
diff --git a/packages/redis-worker/package.json b/packages/redis-worker/package.json
index 19e6efc8da..7da5ad3c4e 100644
--- a/packages/redis-worker/package.json
+++ b/packages/redis-worker/package.json
@@ -1,17 +1,41 @@
 {
-  "name": "@internal/redis-worker",
-  "private": true,
-  "version": "0.0.1",
-  "main": "./dist/src/index.js",
-  "types": "./dist/src/index.d.ts",
+  "name": "@trigger.dev/redis-worker",
+  "version": "3.3.17",
+  "description": "Redis worker for trigger.dev",
+  "license": "MIT",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/triggerdotdev/trigger.dev",
+    "directory": "packages/redis-worker"
+  },
   "type": "module",
-  "exports": {
-    ".": {
-      "@triggerdotdev/source": "./src/index.ts",
-      "import": "./dist/src/index.js",
-      "types": "./dist/src/index.d.ts",
-      "default": "./dist/src/index.js"
-    }
+  "files": [
+    "dist"
+  ],
+  "tshy": {
+    "selfLink": false,
+    "module": true,
+    "project": "./tsconfig.src.json",
+    "dialects": [
+      "esm"
+    ],
+    "exports": {
+      "./package.json": "./package.json",
+      ".": "./src/index.ts"
+    },
+    "sourceDialects": [
+      "@triggerdotdev/source"
+    ]
+  },
+  "scripts": {
+    "clean": "rimraf dist .tshy .tshy-build .turbo",
+    "build": "tshy",
+    "dev": "tshy --watch",
+    "typecheck": "tsc --noEmit -p tsconfig.src.json",
+    "test": "vitest --sequence.concurrent=false --no-file-parallelism"
   },
   "dependencies": {
     "@internal/tracing": "workspace:*",
@@ -26,13 +50,22 @@
     "@internal/testcontainers": "workspace:*",
     "@types/lodash.omit": "^4.5.7",
     "vitest": "^1.4.0",
-    "rimraf": "6.0.1"
+    "rimraf": "6.0.1",
+    "tshy": "^3.0.2",
+    "tsx": "4.17.0"
   },
-  "scripts": {
-    "clean": "rimraf dist",
-    "typecheck": "tsc --noEmit -p tsconfig.build.json",
-    "test": "vitest --sequence.concurrent=false --no-file-parallelism",
-    "build": "pnpm run clean && tsc -p tsconfig.build.json",
-    "dev": "tsc --watch  -p tsconfig.build.json"
-  }
-}
\ No newline at end of file
+  "engines": {
+    "node": ">=18.20.0"
+  },
+  "exports": {
+    "./package.json": "./package.json",
+    ".": {
+      "import": {
+        "@triggerdotdev/source": "./src/index.ts",
+        "types": "./dist/esm/index.d.ts",
+        "default": "./dist/esm/index.js"
+      }
+    }
+  },
+  "module": "./dist/esm/index.js"
+}
diff --git a/packages/redis-worker/tsconfig.json b/packages/redis-worker/tsconfig.json
index af630abe1f..cc79a63894 100644
--- a/packages/redis-worker/tsconfig.json
+++ b/packages/redis-worker/tsconfig.json
@@ -1,8 +1,11 @@
 {
-  "references": [{ "path": "./tsconfig.src.json" }, { "path": "./tsconfig.test.json" }],
-  "compilerOptions": {
-    "moduleResolution": "Node16",
-    "module": "Node16",
-    "customConditions": ["@triggerdotdev/source"]
-  }
+  "extends": "../../.configs/tsconfig.base.json",
+  "references": [
+    {
+      "path": "./tsconfig.src.json"
+    },
+    {
+      "path": "./tsconfig.test.json"
+    }
+  ]
 }
diff --git a/packages/redis-worker/tsconfig.src.json b/packages/redis-worker/tsconfig.src.json
index 5617aa970c..db06c53317 100644
--- a/packages/redis-worker/tsconfig.src.json
+++ b/packages/redis-worker/tsconfig.src.json
@@ -1,19 +1,10 @@
 {
-  "include": ["src/**/*.ts"],
-  "exclude": ["node_modules", "src/**/*.test.ts"],
+  "extends": "./tsconfig.json",
+  "include": ["./src/**/*.ts"],
   "compilerOptions": {
+    "isolatedDeclarations": false,
     "composite": true,
-    "target": "ES2019",
-    "lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
-    "module": "Node16",
-    "moduleResolution": "Node16",
-    "moduleDetection": "force",
-    "verbatimModuleSyntax": false,
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "isolatedModules": true,
-    "preserveWatchOutput": true,
-    "skipLibCheck": true,
-    "strict": true
+    "sourceMap": true,
+    "customConditions": ["@triggerdotdev/source"]
   }
 }
diff --git a/packages/redis-worker/tsconfig.test.json b/packages/redis-worker/tsconfig.test.json
index b68d234bd7..fb90f380ec 100644
--- a/packages/redis-worker/tsconfig.test.json
+++ b/packages/redis-worker/tsconfig.test.json
@@ -1,20 +1,11 @@
 {
-  "include": ["src/**/*.test.ts"],
+  "extends": "./tsconfig.json",
+  "include": ["./test/**/*.ts"],
   "references": [{ "path": "./tsconfig.src.json" }],
   "compilerOptions": {
+    "isolatedDeclarations": false,
     "composite": true,
-    "target": "ES2019",
-    "lib": ["ES2019", "DOM", "DOM.Iterable", "DOM.AsyncIterable"],
-    "module": "Node16",
-    "moduleResolution": "Node16",
-    "moduleDetection": "force",
-    "verbatimModuleSyntax": false,
-    "types": ["vitest/globals"],
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "isolatedModules": true,
-    "preserveWatchOutput": true,
-    "skipLibCheck": true,
-    "strict": true
+    "sourceMap": true,
+    "types": ["vitest/globals"]
   }
 }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 371734e70e..6e7e5f28a0 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -243,9 +243,6 @@ importers:
       '@internal/redis':
         specifier: workspace:*
         version: link:../../internal-packages/redis
-      '@internal/redis-worker':
-        specifier: workspace:*
-        version: link:../../internal-packages/redis-worker
       '@internal/run-engine':
         specifier: workspace:*
         version: link:../../internal-packages/run-engine
@@ -402,6 +399,9 @@ importers:
       '@trigger.dev/platform':
         specifier: 1.0.14
         version: 1.0.14
+      '@trigger.dev/redis-worker':
+        specifier: workspace:*
+        version: link:../../packages/redis-worker
       '@trigger.dev/sdk':
         specifier: workspace:*
         version: link:../../packages/trigger-sdk
@@ -932,51 +932,11 @@ importers:
         specifier: ^1.4.0
         version: 1.6.0(@types/node@20.14.14)
 
-  internal-packages/redis-worker:
-    dependencies:
-      '@internal/redis':
-        specifier: workspace:*
-        version: link:../redis
-      '@internal/tracing':
-        specifier: workspace:*
-        version: link:../tracing
-      '@trigger.dev/core':
-        specifier: workspace:*
-        version: link:../../packages/core
-      lodash.omit:
-        specifier: ^4.5.0
-        version: 4.5.0
-      nanoid:
-        specifier: ^5.0.7
-        version: 5.0.7
-      p-limit:
-        specifier: ^6.2.0
-        version: 6.2.0
-      zod:
-        specifier: 3.23.8
-        version: 3.23.8
-    devDependencies:
-      '@internal/testcontainers':
-        specifier: workspace:*
-        version: link:../testcontainers
-      '@types/lodash.omit':
-        specifier: ^4.5.7
-        version: 4.5.7
-      rimraf:
-        specifier: 6.0.1
-        version: 6.0.1
-      vitest:
-        specifier: ^1.4.0
-        version: 1.6.0(@types/node@20.14.14)
-
   internal-packages/run-engine:
     dependencies:
       '@internal/redis':
         specifier: workspace:*
         version: link:../redis
-      '@internal/redis-worker':
-        specifier: workspace:*
-        version: link:../redis-worker
       '@internal/tracing':
         specifier: workspace:*
         version: link:../tracing
@@ -986,6 +946,9 @@ importers:
       '@trigger.dev/database':
         specifier: workspace:*
         version: link:../database
+      '@trigger.dev/redis-worker':
+        specifier: workspace:*
+        version: link:../../packages/redis-worker
       '@unkey/cache':
         specifier: ^1.5.0
         version: 1.5.0
@@ -1569,6 +1532,49 @@ importers:
         specifier: 4.17.0
         version: 4.17.0
 
+  packages/redis-worker:
+    dependencies:
+      '@internal/redis':
+        specifier: workspace:*
+        version: link:../../internal-packages/redis
+      '@internal/tracing':
+        specifier: workspace:*
+        version: link:../../internal-packages/tracing
+      '@trigger.dev/core':
+        specifier: workspace:*
+        version: link:../core
+      lodash.omit:
+        specifier: ^4.5.0
+        version: 4.5.0
+      nanoid:
+        specifier: ^5.0.7
+        version: 5.1.2
+      p-limit:
+        specifier: ^6.2.0
+        version: 6.2.0
+      zod:
+        specifier: 3.23.8
+        version: 3.23.8
+    devDependencies:
+      '@internal/testcontainers':
+        specifier: workspace:*
+        version: link:../../internal-packages/testcontainers
+      '@types/lodash.omit':
+        specifier: ^4.5.7
+        version: 4.5.7
+      rimraf:
+        specifier: 6.0.1
+        version: 6.0.1
+      tshy:
+        specifier: ^3.0.2
+        version: 3.0.2
+      tsx:
+        specifier: 4.17.0
+        version: 4.17.0
+      vitest:
+        specifier: ^1.4.0
+        version: 1.6.0(@types/node@20.14.14)
+
   packages/rsc:
     dependencies:
       '@trigger.dev/core':
@@ -18298,7 +18304,7 @@ packages:
     resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
     engines: {node: '>= 6.0.0'}
     dependencies:
-      debug: 4.3.7
+      debug: 4.4.0
     transitivePeerDependencies:
       - supports-color
     dev: false
@@ -18307,7 +18313,7 @@ packages:
     resolution: {integrity: sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg==}
     engines: {node: '>= 14'}
     dependencies:
-      debug: 4.3.7
+      debug: 4.4.0
     transitivePeerDependencies:
       - supports-color
 
@@ -19530,7 +19536,7 @@ packages:
   /capnp-ts@0.7.0:
     resolution: {integrity: sha512-XKxXAC3HVPv7r674zP0VC3RTXz+/JKhfyw94ljvF80yynK6VkTnqE3jMuN8b3dUVmmc43TjyxjW4KTsmB3c86g==}
     dependencies:
-      debug: 4.3.7
+      debug: 4.4.0
       tslib: 2.6.2
     transitivePeerDependencies:
       - supports-color
@@ -23762,7 +23768,7 @@ packages:
     engines: {node: '>= 14'}
     dependencies:
       agent-base: 7.1.0
-      debug: 4.3.7
+      debug: 4.4.0
     transitivePeerDependencies:
       - supports-color
 
@@ -23790,7 +23796,7 @@ packages:
     engines: {node: '>= 6'}
     dependencies:
       agent-base: 6.0.2
-      debug: 4.3.7
+      debug: 4.4.0
     transitivePeerDependencies:
       - supports-color
     dev: false
@@ -23800,7 +23806,7 @@ packages:
     engines: {node: '>= 14'}
     dependencies:
       agent-base: 7.1.1
-      debug: 4.3.7
+      debug: 4.4.0
     transitivePeerDependencies:
       - supports-color
 
@@ -26150,12 +26156,6 @@ packages:
     hasBin: true
     dev: false
 
-  /nanoid@5.0.7:
-    resolution: {integrity: sha512-oLxFY2gd2IqnjcYyOXD8XGCftpGtZP2AbHbOkthDkvRywH5ayNtPVy9YlOPcHckXzbLTCHpkb7FB+yuxKV13pQ==}
-    engines: {node: ^18 || >=20}
-    hasBin: true
-    dev: false
-
   /nanoid@5.1.2:
     resolution: {integrity: sha512-b+CiXQCNMUGe0Ri64S9SXFcP9hogjAJ2Rd6GdVxhPLRm7mhGaM7VgOvCAJ1ZshfHbqVDI3uqTI5C8/GaKuLI7g==}
     engines: {node: ^18 || >=20}
@@ -32599,7 +32599,7 @@ packages:
     hasBin: true
     dependencies:
       cac: 6.7.14
-      debug: 4.3.7
+      debug: 4.4.0
       pathe: 1.1.2
       picocolors: 1.1.1
       vite: 5.2.7(@types/node@20.14.14)

From b202f9c5e405f2760842d9c5270a3aa75082cb7a Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Wed, 19 Mar 2025 17:06:02 +0000
Subject: [PATCH 07/26] fix types

---
 internal-packages/redis/src/index.ts    |  2 +-
 internal-packages/tracing/src/index.ts  |  4 ++--
 packages/redis-worker/src/queue.test.ts | 23 +++++++++++++++++++++++
 packages/redis-worker/src/worker.ts     |  8 ++++++--
 4 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/internal-packages/redis/src/index.ts b/internal-packages/redis/src/index.ts
index 13264773b9..27e5731429 100644
--- a/internal-packages/redis/src/index.ts
+++ b/internal-packages/redis/src/index.ts
@@ -1,4 +1,4 @@
-import { Redis, RedisOptions } from "ioredis";
+import { Redis, type RedisOptions } from "ioredis";
 import { Logger } from "@trigger.dev/core/logger";
 
 export { Redis, type Callback, type RedisOptions, type Result, type RedisCommander } from "ioredis";
diff --git a/internal-packages/tracing/src/index.ts b/internal-packages/tracing/src/index.ts
index 9fe3532776..a2fbb0e2b6 100644
--- a/internal-packages/tracing/src/index.ts
+++ b/internal-packages/tracing/src/index.ts
@@ -1,5 +1,5 @@
-import { Span, SpanOptions, SpanStatusCode, Tracer } from "@opentelemetry/api";
-import { Logger, SeverityNumber } from "@opentelemetry/api-logs";
+import { type Span, type SpanOptions, SpanStatusCode, type Tracer } from "@opentelemetry/api";
+import { type Logger, SeverityNumber } from "@opentelemetry/api-logs";
 import { flattenAttributes } from "@trigger.dev/core/v3/utils/flattenAttributes";
 
 export * from "@opentelemetry/semantic-conventions";
diff --git a/packages/redis-worker/src/queue.test.ts b/packages/redis-worker/src/queue.test.ts
index 023a9564a8..902dcb9d58 100644
--- a/packages/redis-worker/src/queue.test.ts
+++ b/packages/redis-worker/src/queue.test.ts
@@ -30,6 +30,11 @@ describe("SimpleQueue", () => {
       expect(await queue.size()).toBe(2);
 
       const [first] = await queue.dequeue(1);
+
+      if (!first) {
+        throw new Error("No item dequeued");
+      }
+
       expect(first).toEqual(
         expect.objectContaining({
           id: "1",
@@ -47,6 +52,11 @@ describe("SimpleQueue", () => {
       expect(await queue.size({ includeFuture: true })).toBe(1);
 
       const [second] = await queue.dequeue(1);
+
+      if (!second) {
+        throw new Error("No item dequeued");
+      }
+
       expect(second).toEqual(
         expect.objectContaining({
           id: "2",
@@ -253,6 +263,10 @@ describe("SimpleQueue", () => {
       expect(await queue.size()).toBe(1);
       expect(await queue.size({ includeFuture: true })).toBe(3);
 
+      if (!dequeued[0] || !dequeued[1]) {
+        throw new Error("No items dequeued");
+      }
+
       await queue.ack(dequeued[0].id);
       await queue.ack(dequeued[1].id);
 
@@ -270,6 +284,10 @@ describe("SimpleQueue", () => {
         })
       );
 
+      if (!last) {
+        throw new Error("No item dequeued");
+      }
+
       await queue.ack(last.id);
       expect(await queue.size({ includeFuture: true })).toBe(0);
     } finally {
@@ -315,6 +333,11 @@ describe("SimpleQueue", () => {
 
       // Dequeue the redriven item
       const [redrivenItem] = await queue.dequeue(1);
+
+      if (!redrivenItem) {
+        throw new Error("No item dequeued");
+      }
+
       expect(redrivenItem).toEqual(
         expect.objectContaining({
           id: "1",
diff --git a/packages/redis-worker/src/worker.ts b/packages/redis-worker/src/worker.ts
index d6d4d0979b..502b7d0e5d 100644
--- a/packages/redis-worker/src/worker.ts
+++ b/packages/redis-worker/src/worker.ts
@@ -152,7 +152,11 @@ class Worker<TCatalog extends WorkerCatalog> {
       this.tracer,
       "enqueue",
       async (span) => {
-        const timeout = visibilityTimeoutMs ?? this.options.catalog[job].visibilityTimeoutMs;
+        const timeout = visibilityTimeoutMs ?? this.options.catalog[job]?.visibilityTimeoutMs;
+
+        if (!timeout) {
+          throw new Error(`No visibility timeout found for job ${String(job)} with id ${id}`);
+        }
 
         span.setAttribute("job_visibility_timeout_ms", timeout);
 
@@ -306,7 +310,7 @@ class Worker<TCatalog extends WorkerCatalog> {
         const newAttempt = attempt + 1;
         const retrySettings = {
           ...defaultRetrySettings,
-          ...catalogItem.retry,
+          ...catalogItem?.retry,
         };
         const retryDelay = calculateNextRetryDelay(retrySettings, newAttempt);
 

From 22e9dc1fdf7e0d19ff8c11d41e91e3bf08f0bd83 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Wed, 19 Mar 2025 17:06:09 +0000
Subject: [PATCH 08/26] update ai files

---
 .cursor/rules/webapp.mdc | 2 +-
 ai/references/repo.md    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.cursor/rules/webapp.mdc b/.cursor/rules/webapp.mdc
index 825a54b8b6..6cda973951 100644
--- a/.cursor/rules/webapp.mdc
+++ b/.cursor/rules/webapp.mdc
@@ -9,7 +9,7 @@ The main trigger.dev webapp, which powers it's API and dashboard and makes up th
 - `@trigger.dev/database` exports a Prisma 5.4.1 client that is used extensively in the webapp to access a PostgreSQL instance. The schema file is [schema.prisma](mdc:internal-packages/database/prisma/schema.prisma)
 - `@trigger.dev/core` is a published package and is used to share code between the `@trigger.dev/sdk` and the webapp. It includes functionality but also a load of Zod schemas for data validation. When importing from `@trigger.dev/core` in the webapp, we never import the root `@trigger.dev/core` path, instead we favor one of the subpath exports that you can find in [package.json](mdc:packages/core/package.json)
 - `@internal/run-engine` has all the code needed to trigger a run and take it through it's lifecycle to completion.
-- `@internal/redis-worker` is a custom redis based background job/worker system that's used in the webapp and also used inside the run engine.
+- `@trigger.dev/redis-worker` is a custom redis based background job/worker system that's used in the webapp and also used inside the run engine.
 
 ## Environment variables and testing
 
diff --git a/ai/references/repo.md b/ai/references/repo.md
index 22e5478f4b..0e9b49b460 100644
--- a/ai/references/repo.md
+++ b/ai/references/repo.md
@@ -14,13 +14,13 @@ This is a pnpm 8.15.5 monorepo that uses turborepo @turbo.json. The following wo
 - <root>/packages/core is the `@trigger.dev/core` package that is shared across the SDK and other packages
 - <root>/packages/build defines the types and prebuilt build extensions for trigger.dev. See our [build extensions docs](https://trigger.dev/docs/config/extensions/overview.md) for more information.
 - <root>/packages/react-hooks defines some useful react hooks like our realtime hooks. See our [Realtime hooks](https://trigger.dev/docs/frontend/react-hooks/realtime.md) and our [Trigger hooks](https://trigger.dev/docs/frontend/react-hooks/triggering.md) for more information.
+- <root>/packages/redis-worker is the `@trigger.dev/redis-worker` package that implements a custom background job/worker sytem powered by redis for offloading work to the background, used in the webapp and also in the Run Engine 2.0.
 
 ## Internal Packages
 
 - <root>/internal-packages/\* are packages that are used internally only, not published, and usually they have a tsc build step and are used in the webapp
 - <root>/internal-packages/database is the `@trigger.dev/database` package that exports a prisma client, has the schema file, and exports a few other helpers.
 - <root>/internal-packages/run-engine is the `@internal/run-engine` package that is "Run Engine 2.0" and handles moving a run all the way through it's lifecycle
-- <root>/internal-packages/redis-worker is the `@internal/redis-worker` package that implements a custom background job/worker sytem powered by redis for offloading work to the background, used in the webapp and also in the Run Engine 2.0.
 - <root>/internal-packages/redis is the `@internal/redis` package that exports Redis types and the `createRedisClient` function to unify how we create redis clients in the repo. It's not used everywhere yet, but it's the preferred way to create redis clients from now on.
 - <root>/internal-packages/testcontainers is the `@internal/testcontainers` package that exports a few useful functions for spinning up local testcontainers when writing vitest tests. See our [tests.md](./tests.md) file for more information.
 - <root>/internal-packages/zodworker is the `@internal/zodworker` package that implements a wrapper around graphile-worker that allows us to use zod to validate our background jobs. We are moving away from using graphile-worker as our background job system, replacing it with our own redis-worker package.

From ba68340ec2c791f5a05ddd3c9854d1eef4ed6691 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Thu, 20 Mar 2025 09:49:16 +0000
Subject: [PATCH 09/26] fix cursor terminal links

---
 package.json                            |  3 ++-
 patches/supports-hyperlinks@2.3.0.patch | 16 ++++++++++++++++
 pnpm-lock.yaml                          | 10 +++++++---
 3 files changed, 25 insertions(+), 4 deletions(-)
 create mode 100644 patches/supports-hyperlinks@2.3.0.patch

diff --git a/package.json b/package.json
index 9c9e85925f..13698ce2e1 100644
--- a/package.json
+++ b/package.json
@@ -74,7 +74,8 @@
       "@changesets/assemble-release-plan@5.2.4": "patches/@changesets__assemble-release-plan@5.2.4.patch",
       "engine.io-parser@5.2.2": "patches/engine.io-parser@5.2.2.patch",
       "graphile-worker@0.16.6": "patches/graphile-worker@0.16.6.patch",
-      "redlock@5.0.0-beta.2": "patches/redlock@5.0.0-beta.2.patch"
+      "redlock@5.0.0-beta.2": "patches/redlock@5.0.0-beta.2.patch",
+      "supports-hyperlinks@2.3.0": "patches/supports-hyperlinks@2.3.0.patch"
     }
   }
 }
\ No newline at end of file
diff --git a/patches/supports-hyperlinks@2.3.0.patch b/patches/supports-hyperlinks@2.3.0.patch
new file mode 100644
index 0000000000..9442afbbfd
--- /dev/null
+++ b/patches/supports-hyperlinks@2.3.0.patch
@@ -0,0 +1,16 @@
+diff --git a/index.js b/index.js
+index 89f1b2cb86fad204b0493da3b8a3d5ed28937260..945f4cff27ed501fca75e269dfd7172e74c9c955 100644
+--- a/index.js
++++ b/index.js
+@@ -62,6 +62,11 @@ function supportsHyperlink(stream) {
+ 		return false;
+ 	}
+ 
++	// Cursor supports hyperlinks
++  if ("CURSOR_TRACE_ID" in env) {
++    return true;
++  }
++
+ 	if ('TERM_PROGRAM' in env) {
+ 		const version = parseVersion(env.TERM_PROGRAM_VERSION);
+ 
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 6e7e5f28a0..be35085c66 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -17,6 +17,9 @@ patchedDependencies:
   redlock@5.0.0-beta.2:
     hash: rwyegdki7iserrd7fgjwxkhnlu
     path: patches/redlock@5.0.0-beta.2.patch
+  supports-hyperlinks@2.3.0:
+    hash: xmw2etywyp5w2jf77wkqg4ob3a
+    path: patches/supports-hyperlinks@2.3.0.patch
 
 importers:
 
@@ -30748,12 +30751,13 @@ packages:
     dependencies:
       has-flag: 4.0.0
 
-  /supports-hyperlinks@2.3.0:
+  /supports-hyperlinks@2.3.0(patch_hash=xmw2etywyp5w2jf77wkqg4ob3a):
     resolution: {integrity: sha512-RpsAZlpWcDwOPQA22aCH4J0t7L8JmAvsCxfOSEwm7cQs3LshN36QaTkwd70DnBOXDWGssw2eUoc8CaRWT0XunA==}
     engines: {node: '>=8'}
     dependencies:
       has-flag: 4.0.0
       supports-color: 7.2.0
+    patched: true
 
   /supports-hyperlinks@3.1.0:
     resolution: {integrity: sha512-2rn0BZ+/f7puLOHZm1HOJfwBggfaHXUpPUSSG/SWM4TWp5KCfmNYwnC3hruy2rZlMnmWZ+QAGpZfchu3f3695A==}
@@ -31088,7 +31092,7 @@ packages:
     engines: {node: '>=8'}
     dependencies:
       ansi-escapes: 4.3.2
-      supports-hyperlinks: 2.3.0
+      supports-hyperlinks: 2.3.0(patch_hash=xmw2etywyp5w2jf77wkqg4ob3a)
     dev: true
 
   /terminal-link@3.0.0:
@@ -31096,7 +31100,7 @@ packages:
     engines: {node: '>=12'}
     dependencies:
       ansi-escapes: 5.0.0
-      supports-hyperlinks: 2.3.0
+      supports-hyperlinks: 2.3.0(patch_hash=xmw2etywyp5w2jf77wkqg4ob3a)
     dev: false
 
   /terser-webpack-plugin@5.3.7(@swc/core@1.3.101)(esbuild@0.19.11)(webpack@5.88.2):

From 25c1042ae93be521a52d475814d0fa02ff84089f Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Thu, 20 Mar 2025 09:50:23 +0000
Subject: [PATCH 10/26] prevent overly friendly ids

---
 packages/core/src/v3/isomorphic/friendlyId.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/packages/core/src/v3/isomorphic/friendlyId.ts b/packages/core/src/v3/isomorphic/friendlyId.ts
index 42b8e0dde8..36315f6c44 100644
--- a/packages/core/src/v3/isomorphic/friendlyId.ts
+++ b/packages/core/src/v3/isomorphic/friendlyId.ts
@@ -21,6 +21,10 @@ export function toFriendlyId(entityName: string, internalId: string): string {
     throw new Error("Internal ID cannot be empty");
   }
 
+  if (internalId.startsWith(`${entityName}_`)) {
+    return internalId;
+  }
+
   return `${entityName}_${internalId}`;
 }
 

From abcab335c8436478313581b3e976f89ea77d3960 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Thu, 20 Mar 2025 09:51:32 +0000
Subject: [PATCH 11/26] use structured logger

---
 packages/core/src/v3/serverOnly/shutdownManager.ts | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/packages/core/src/v3/serverOnly/shutdownManager.ts b/packages/core/src/v3/serverOnly/shutdownManager.ts
index 80cf763b54..9adb662a3c 100644
--- a/packages/core/src/v3/serverOnly/shutdownManager.ts
+++ b/packages/core/src/v3/serverOnly/shutdownManager.ts
@@ -1,3 +1,4 @@
+import { SimpleStructuredLogger } from "../utils/structuredLogger.js";
 import { singleton } from "./singleton.js";
 
 type ShutdownHandler = NodeJS.SignalsListener;
@@ -11,6 +12,7 @@ class ShutdownManager {
     SIGTERM: 15,
   };
 
+  private logger = new SimpleStructuredLogger("shutdownManager");
   private handlers: Map<string, { handler: ShutdownHandler; signals: ShutdownSignal[] }> =
     new Map();
 
@@ -38,7 +40,7 @@ class ShutdownManager {
     if (this.isShuttingDown) return;
     this.isShuttingDown = true;
 
-    console.log(`\nReceived ${signal}. Starting graceful shutdown...`);
+    this.logger.info(`Received ${signal}. Starting graceful shutdown...`);
 
     // Get handlers that are registered for this signal
     const handlersToRun = Array.from(this.handlers.entries()).filter(([_, { signals }]) =>
@@ -49,11 +51,11 @@ class ShutdownManager {
       const results = await Promise.allSettled(
         handlersToRun.map(async ([name, { handler }]) => {
           try {
-            console.log(`Running shutdown handler: ${name}`);
+            this.logger.info(`Running shutdown handler: ${name}`);
             await handler(signal);
-            console.log(`Shutdown handler completed: ${name}`);
+            this.logger.info(`Shutdown handler completed: ${name}`);
           } catch (error) {
-            console.error(`Shutdown handler failed: ${name}`, error);
+            this.logger.error(`Shutdown handler failed: ${name}`, { error });
             throw error;
           }
         })
@@ -65,12 +67,12 @@ class ShutdownManager {
           const handlerEntry = handlersToRun[index];
           if (handlerEntry) {
             const [name] = handlerEntry;
-            console.error(`Shutdown handler "${name}" failed:`, result.reason);
+            this.logger.error(`Shutdown handler "${name}" failed:`, { reason: result.reason });
           }
         }
       });
     } catch (error) {
-      console.error("Error during shutdown:", error);
+      this.logger.error("Error during shutdown:", { error });
     } finally {
       // Exit with the correct signal number
       process.exit(128 + this.signalNumbers[signal]);

From ecb9cad19ba3b252e6ec9acf9198ff167b1927a2 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Thu, 20 Mar 2025 09:52:08 +0000
Subject: [PATCH 12/26] use unique shutdown handler names

---
 packages/redis-worker/src/worker.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/redis-worker/src/worker.ts b/packages/redis-worker/src/worker.ts
index 502b7d0e5d..8dca728f29 100644
--- a/packages/redis-worker/src/worker.ts
+++ b/packages/redis-worker/src/worker.ts
@@ -395,7 +395,7 @@ class Worker<TCatalog extends WorkerCatalog> {
   }
 
   private setupShutdownHandlers() {
-    shutdownManager.register("redis-worker", this.shutdown.bind(this));
+    shutdownManager.register(`redis-worker:${this.options.name}`, this.shutdown.bind(this));
   }
 
   private async shutdown(signal?: NodeJS.Signals) {

From 66c0c5b3d0f4dc3fdee5ee1e5b03128f84590ca2 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Thu, 20 Mar 2025 09:52:39 +0000
Subject: [PATCH 13/26] rework suspend completion

---
 apps/supervisor/package.json                  |  2 +-
 apps/supervisor/src/workloadServer/index.ts   |  2 --
 .../src/v3/runEngineWorker/supervisor/http.ts | 34 +++++++++++++-----
 .../v3/runEngineWorker/supervisor/schemas.ts  |  8 +++++
 .../src/v3/serverOnly/checkpointClient.ts     | 35 +++++++------------
 5 files changed, 47 insertions(+), 34 deletions(-)

diff --git a/apps/supervisor/package.json b/apps/supervisor/package.json
index 3718e6514d..845165f173 100644
--- a/apps/supervisor/package.json
+++ b/apps/supervisor/package.json
@@ -6,7 +6,7 @@
   "type": "module",
   "scripts": {
     "build": "tsc",
-    "dev": "tsx --experimental-sqlite --require dotenv/config --watch src/index.ts",
+    "dev": "tsx --experimental-sqlite --require dotenv/config --watch src/index.ts || (echo '!! Remember to run: nvm use'; exit 1)",
     "start": "node --experimental-sqlite dist/index.js",
     "typecheck": "tsc --noEmit"
   },
diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts
index 42ccde3531..9ffa24e389 100644
--- a/apps/supervisor/src/workloadServer/index.ts
+++ b/apps/supervisor/src/workloadServer/index.ts
@@ -249,8 +249,6 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
               console.error("Failed to suspend run", { params });
               return;
             }
-
-            console.log("Suspended run", { params });
           },
         }
       )
diff --git a/packages/core/src/v3/runEngineWorker/supervisor/http.ts b/packages/core/src/v3/runEngineWorker/supervisor/http.ts
index 6a4bd49cdd..8814c84c35 100644
--- a/packages/core/src/v3/runEngineWorker/supervisor/http.ts
+++ b/packages/core/src/v3/runEngineWorker/supervisor/http.ts
@@ -15,6 +15,8 @@ import {
   WorkerApiRunHeartbeatResponseBody,
   WorkerApiRunLatestSnapshotResponseBody,
   WorkerApiDebugLogBody,
+  WorkerApiSuspendRunRequestBody,
+  WorkerApiSuspendRunResponseBody,
 } from "./schemas.js";
 import { SupervisorClientCommonOptions } from "./types.js";
 import { getDefaultWorkerHeaders } from "./util.js";
@@ -220,14 +222,30 @@ export class SupervisorHttpClient {
     );
   }
 
-  getSuspendCompletionUrl(runId: string, snapshotId: string, runnerId?: string) {
-    return {
-      url: `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/suspend`,
-      headers: {
-        ...this.defaultHeaders,
-        ...this.runnerIdHeader(runnerId),
-      },
-    };
+  async submitSuspendCompletion({
+    runId,
+    snapshotId,
+    runnerId,
+    body,
+  }: {
+    runId: string;
+    snapshotId: string;
+    runnerId?: string;
+    body: WorkerApiSuspendRunRequestBody;
+  }) {
+    return wrapZodFetch(
+      WorkerApiSuspendRunResponseBody,
+      `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/suspend`,
+      {
+        method: "POST",
+        headers: {
+          ...this.defaultHeaders,
+          ...this.runnerIdHeader(runnerId),
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(body),
+      }
+    );
   }
 
   private runnerIdHeader(runnerId?: string): Record<string, string> {
diff --git a/packages/core/src/v3/runEngineWorker/supervisor/schemas.ts b/packages/core/src/v3/runEngineWorker/supervisor/schemas.ts
index 270dcc9e8a..33106323fe 100644
--- a/packages/core/src/v3/runEngineWorker/supervisor/schemas.ts
+++ b/packages/core/src/v3/runEngineWorker/supervisor/schemas.ts
@@ -142,3 +142,11 @@ export const WorkerApiDebugLogBody = z.object({
   properties: Attributes.optional(),
 });
 export type WorkerApiDebugLogBody = z.infer<typeof WorkerApiDebugLogBody>;
+
+export const WorkerApiSuspendCompletionResponseBody = z.object({
+  success: z.boolean(),
+  error: z.string().optional(),
+});
+export type WorkerApiSuspendCompletionResponseBody = z.infer<
+  typeof WorkerApiSuspendCompletionResponseBody
+>;
diff --git a/packages/core/src/v3/serverOnly/checkpointClient.ts b/packages/core/src/v3/serverOnly/checkpointClient.ts
index 76bb69e2fc..4f441ad8cf 100644
--- a/packages/core/src/v3/serverOnly/checkpointClient.ts
+++ b/packages/core/src/v3/serverOnly/checkpointClient.ts
@@ -21,10 +21,6 @@ export class CheckpointClient {
     return new URL("/api/v1/restore", this.apiUrl);
   }
 
-  private get suspendUrl() {
-    return new URL("/api/v1/suspend", this.apiUrl);
-  }
-
   constructor(opts: CheckpointClientOptions) {
     this.apiUrl = opts.apiUrl;
     this.workerClient = opts.workerClient;
@@ -41,26 +37,19 @@ export class CheckpointClient {
     containerId: string;
     runnerId: string;
   }): Promise<boolean> {
-    const completionUrl = this.workerClient.getSuspendCompletionUrl(
-      runFriendlyId,
-      snapshotFriendlyId,
-      runnerId
-    );
-
-    const res = await fetch(this.suspendUrl, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({
-        type: "DOCKER",
-        containerId,
-        callbacks: {
-          completion: completionUrl.url,
-          headers: completionUrl.headers,
+    const res = await fetch(
+      new URL(`/api/v1/runs/${runFriendlyId}/snapshots/${snapshotFriendlyId}/suspend`, this.apiUrl),
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
         },
-      } satisfies CheckpointServiceSuspendRequestBodyInput),
-    });
+        body: JSON.stringify({
+          type: "DOCKER",
+          containerId,
+        } satisfies CheckpointServiceSuspendRequestBodyInput),
+      }
+    );
 
     if (!res.ok) {
       this.logger.error("[CheckpointClient] Suspend request failed", {

From e979dd933707b873f7efe11be3a0a05f580e1156 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Thu, 20 Mar 2025 11:25:40 +0000
Subject: [PATCH 14/26] add trycatch util

---
 packages/core/src/utils.ts | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts
index f934448bbb..4bc8d40304 100644
--- a/packages/core/src/utils.ts
+++ b/packages/core/src/utils.ts
@@ -1,3 +1,12 @@
 export function assertExhaustive(x: never): never {
   throw new Error("Unexpected object: " + x);
 }
+
+export async function tryCatch<T, E = Error>(promise: Promise<T>): Promise<[T, null] | [null, E]> {
+  try {
+    const data = await promise;
+    return [data, null];
+  } catch (error) {
+    return [null, error as E];
+  }
+}

From 26472d3256e0d173fd1f4abeda93b3da7cf2d5fe Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Thu, 20 Mar 2025 14:05:27 +0000
Subject: [PATCH 15/26] rework suspend restore

---
 apps/supervisor/src/env.ts                    |  6 ++
 apps/supervisor/src/index.ts                  | 18 +++-
 .../src/workloadManager/kubernetes.ts         |  9 +-
 apps/supervisor/src/workloadServer/index.ts   | 33 ++++++-
 .../src/entryPoints/managed-run-controller.ts |  4 +-
 packages/core/package.json                    |  1 +
 packages/core/src/utils.ts                    |  6 +-
 .../core/src/v3/runEngineWorker/consts.ts     |  2 +
 .../src/v3/runEngineWorker/workload/types.ts  |  4 +-
 .../src/v3/runEngineWorker/workload/util.ts   |  2 +
 packages/core/src/v3/schemas/checkpoints.ts   | 31 ++----
 packages/core/src/v3/schemas/runEngine.ts     | 94 ++++++++++---------
 .../src/v3/serverOnly/checkpointClient.ts     | 64 ++++++-------
 packages/core/src/v3/serverOnly/index.ts      |  1 +
 packages/core/src/v3/serverOnly/k8s.ts        | 16 ++++
 pnpm-lock.yaml                                |  3 +
 16 files changed, 170 insertions(+), 124 deletions(-)
 create mode 100644 packages/core/src/v3/serverOnly/k8s.ts

diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts
index 2b4ca5eb5e..69a8024f34 100644
--- a/apps/supervisor/src/env.ts
+++ b/apps/supervisor/src/env.ts
@@ -45,6 +45,12 @@ const Env = z.object({
   // Used by the resource monitor
   OVERRIDE_CPU_TOTAL: z.coerce.number().optional(),
   OVERRIDE_MEMORY_TOTAL_GB: z.coerce.number().optional(),
+
+  // Kubernetes specific settings
+  KUBERNETES_FORCE_ENABLED: BoolEnv.default(false),
+  KUBERNETES_NAMESPACE: z.string().default("default"),
+  EPHEMERAL_STORAGE_SIZE_LIMIT: z.string().default("10Gi"),
+  EPHEMERAL_STORAGE_SIZE_REQUEST: z.string().default("2Gi"),
 });
 
 export const env = Env.parse(stdEnv);
diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts
index 94088e885a..28c96f2ab7 100644
--- a/apps/supervisor/src/index.ts
+++ b/apps/supervisor/src/index.ts
@@ -13,7 +13,11 @@ import {
 } from "./resourceMonitor.js";
 import { KubernetesWorkloadManager } from "./workloadManager/kubernetes.js";
 import { DockerWorkloadManager } from "./workloadManager/docker.js";
-import { HttpServer, CheckpointClient } from "@trigger.dev/core/v3/serverOnly";
+import {
+  HttpServer,
+  CheckpointClient,
+  isKubernetesEnvironment,
+} from "@trigger.dev/core/v3/serverOnly";
 import { createK8sApi, RUNTIME_ENV } from "./clients/kubernetes.js";
 
 class ManagedSupervisor {
@@ -25,7 +29,7 @@ class ManagedSupervisor {
   private readonly resourceMonitor: ResourceMonitor;
   private readonly checkpointClient?: CheckpointClient;
 
-  private readonly isKubernetes = RUNTIME_ENV === "kubernetes";
+  private readonly isKubernetes = isKubernetesEnvironment(env.KUBERNETES_FORCE_ENABLED);
   private readonly warmStartUrl = env.TRIGGER_WARM_START_URL;
 
   constructor() {
@@ -94,6 +98,7 @@ class ManagedSupervisor {
       this.checkpointClient = new CheckpointClient({
         apiUrl: new URL(env.TRIGGER_CHECKPOINT_URL),
         workerClient: this.workerSession.httpClient,
+        orchestrator: this.isKubernetes ? "KUBERNETES" : "DOCKER",
       });
     }
 
@@ -127,7 +132,9 @@ class ManagedSupervisor {
         return;
       }
 
-      if (message.checkpoint) {
+      const { checkpoint, ...rest } = message;
+
+      if (checkpoint) {
         this.logger.log("[ManagedWorker] Restoring run", { runId: message.run.id });
 
         if (!this.checkpointClient) {
@@ -139,7 +146,10 @@ class ManagedSupervisor {
           const didRestore = await this.checkpointClient.restoreRun({
             runFriendlyId: message.run.friendlyId,
             snapshotFriendlyId: message.snapshot.friendlyId,
-            checkpoint: message.checkpoint,
+            body: {
+              ...rest,
+              checkpoint,
+            },
           });
 
           if (didRestore) {
diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts
index 39b8227f16..90977ed21b 100644
--- a/apps/supervisor/src/workloadManager/kubernetes.ts
+++ b/apps/supervisor/src/workloadManager/kubernetes.ts
@@ -9,9 +9,6 @@ import type { EnvironmentType, MachinePreset } from "@trigger.dev/core/v3";
 import { env } from "../env.js";
 import { type K8sApi, createK8sApi, type k8s } from "../clients/kubernetes.js";
 
-const POD_EPHEMERAL_STORAGE_SIZE_LIMIT = process.env.POD_EPHEMERAL_STORAGE_SIZE_LIMIT || "10Gi";
-const POD_EPHEMERAL_STORAGE_SIZE_REQUEST = process.env.POD_EPHEMERAL_STORAGE_SIZE_REQUEST || "2Gi";
-
 type ResourceQuantities = {
   [K in "cpu" | "memory" | "ephemeral-storage"]?: string;
 };
@@ -19,7 +16,7 @@ type ResourceQuantities = {
 export class KubernetesWorkloadManager implements WorkloadManager {
   private readonly logger = new SimpleStructuredLogger("kubernetes-workload-provider");
   private k8s: K8sApi;
-  private namespace = "default";
+  private namespace = env.KUBERNETES_NAMESPACE;
 
   constructor(private opts: WorkloadManagerOptions) {
     this.k8s = createK8sApi();
@@ -205,13 +202,13 @@ export class KubernetesWorkloadManager implements WorkloadManager {
 
   get #defaultResourceRequests(): ResourceQuantities {
     return {
-      "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_REQUEST,
+      "ephemeral-storage": env.EPHEMERAL_STORAGE_SIZE_REQUEST,
     };
   }
 
   get #defaultResourceLimits(): ResourceQuantities {
     return {
-      "ephemeral-storage": POD_EPHEMERAL_STORAGE_SIZE_LIMIT,
+      "ephemeral-storage": env.EPHEMERAL_STORAGE_SIZE_LIMIT,
     };
   }
 
diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts
index 9ffa24e389..1421d8f98d 100644
--- a/apps/supervisor/src/workloadServer/index.ts
+++ b/apps/supervisor/src/workloadServer/index.ts
@@ -94,8 +94,8 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
     this.websocketServer = this.createWebsocketServer();
   }
 
-  private runnerIdFromRequest(req: IncomingMessage): string | undefined {
-    const value = req.headers[WORKLOAD_HEADERS.RUNNER_ID];
+  private headerValueFromRequest(req: IncomingMessage, headerName: string): string | undefined {
+    const value = req.headers[headerName];
 
     if (Array.isArray(value)) {
       return value[0];
@@ -104,6 +104,22 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
     return value;
   }
 
+  private runnerIdFromRequest(req: IncomingMessage): string | undefined {
+    return this.headerValueFromRequest(req, WORKLOAD_HEADERS.RUNNER_ID);
+  }
+
+  private deploymentIdFromRequest(req: IncomingMessage): string | undefined {
+    return this.headerValueFromRequest(req, WORKLOAD_HEADERS.DEPLOYMENT_ID);
+  }
+
+  private deploymentVersionFromRequest(req: IncomingMessage): string | undefined {
+    return this.headerValueFromRequest(req, WORKLOAD_HEADERS.DEPLOYMENT_VERSION);
+  }
+
+  private projectRefFromRequest(req: IncomingMessage): string | undefined {
+    return this.headerValueFromRequest(req, WORKLOAD_HEADERS.PROJECT_REF);
+  }
+
   private createHttpServer({ host, port }: { host: string; port: number }) {
     return new HttpServer({ port, host })
       .route(
@@ -213,8 +229,10 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
             }
 
             const runnerId = this.runnerIdFromRequest(req);
+            const deploymentVersion = this.deploymentVersionFromRequest(req);
+            const projectRef = this.projectRefFromRequest(req);
 
-            if (!runnerId) {
+            if (!runnerId || !deploymentVersion || !projectRef) {
               console.error("Invalid headers for suspend request", {
                 ...params,
                 headers: req.headers,
@@ -241,8 +259,13 @@ export class WorkloadServer extends EventEmitter<WorkloadServerEvents> {
             const suspendResult = await this.checkpointClient.suspendRun({
               runFriendlyId: params.runFriendlyId,
               snapshotFriendlyId: params.snapshotFriendlyId,
-              containerId: runnerId,
-              runnerId,
+              body: {
+                runnerId,
+                runId: params.runFriendlyId,
+                snapshotId: params.snapshotFriendlyId,
+                projectRef,
+                deploymentVersion,
+              },
             });
 
             if (!suspendResult) {
diff --git a/packages/cli-v3/src/entryPoints/managed-run-controller.ts b/packages/cli-v3/src/entryPoints/managed-run-controller.ts
index 5eab5839f7..1496148216 100644
--- a/packages/cli-v3/src/entryPoints/managed-run-controller.ts
+++ b/packages/cli-v3/src/entryPoints/managed-run-controller.ts
@@ -149,8 +149,10 @@ class ManagedRunController {
 
     this.httpClient = new WorkloadHttpClient({
       workerApiUrl: this.workerApiUrl,
-      deploymentId: env.TRIGGER_DEPLOYMENT_ID,
       runnerId: env.TRIGGER_RUNNER_ID,
+      deploymentId: env.TRIGGER_DEPLOYMENT_ID,
+      deploymentVersion: env.TRIGGER_DEPLOYMENT_VERSION,
+      projectRef: env.TRIGGER_PROJECT_REF,
     });
 
     if (env.TRIGGER_WARM_START_URL) {
diff --git a/packages/core/package.json b/packages/core/package.json
index 7b82035357..d58795e09c 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -191,6 +191,7 @@
     "nanoid": "^3.3.4",
     "socket.io": "4.7.4",
     "socket.io-client": "4.7.5",
+    "std-env": "^3.8.1",
     "superjson": "^2.2.1",
     "tinyexec": "^0.3.2",
     "zod": "3.23.8",
diff --git a/packages/core/src/utils.ts b/packages/core/src/utils.ts
index 4bc8d40304..1fb82cc714 100644
--- a/packages/core/src/utils.ts
+++ b/packages/core/src/utils.ts
@@ -2,11 +2,11 @@ export function assertExhaustive(x: never): never {
   throw new Error("Unexpected object: " + x);
 }
 
-export async function tryCatch<T, E = Error>(promise: Promise<T>): Promise<[T, null] | [null, E]> {
+export async function tryCatch<T, E = Error>(promise: Promise<T>): Promise<[null, T] | [E, null]> {
   try {
     const data = await promise;
-    return [data, null];
+    return [null, data];
   } catch (error) {
-    return [null, error as E];
+    return [error as E, null];
   }
 }
diff --git a/packages/core/src/v3/runEngineWorker/consts.ts b/packages/core/src/v3/runEngineWorker/consts.ts
index 205a0154cc..29dc76b535 100644
--- a/packages/core/src/v3/runEngineWorker/consts.ts
+++ b/packages/core/src/v3/runEngineWorker/consts.ts
@@ -8,4 +8,6 @@ export const WORKER_HEADERS = {
 export const WORKLOAD_HEADERS = {
   DEPLOYMENT_ID: "x-trigger-workload-deployment-id",
   RUNNER_ID: "x-trigger-workload-runner-id",
+  DEPLOYMENT_VERSION: "x-trigger-workload-deployment-version",
+  PROJECT_REF: "x-trigger-workload-project-ref",
 };
diff --git a/packages/core/src/v3/runEngineWorker/workload/types.ts b/packages/core/src/v3/runEngineWorker/workload/types.ts
index d29d28bdba..13926748cd 100644
--- a/packages/core/src/v3/runEngineWorker/workload/types.ts
+++ b/packages/core/src/v3/runEngineWorker/workload/types.ts
@@ -1,5 +1,7 @@
 export type WorkloadClientCommonOptions = {
   workerApiUrl: string;
-  deploymentId: string;
   runnerId: string;
+  deploymentId: string;
+  deploymentVersion: string;
+  projectRef: string;
 };
diff --git a/packages/core/src/v3/runEngineWorker/workload/util.ts b/packages/core/src/v3/runEngineWorker/workload/util.ts
index a54dc5c1dc..74f58d9bb8 100644
--- a/packages/core/src/v3/runEngineWorker/workload/util.ts
+++ b/packages/core/src/v3/runEngineWorker/workload/util.ts
@@ -8,5 +8,7 @@ export function getDefaultWorkloadHeaders(
   return createHeaders({
     [WORKLOAD_HEADERS.DEPLOYMENT_ID]: options.deploymentId,
     [WORKLOAD_HEADERS.RUNNER_ID]: options.runnerId,
+    [WORKLOAD_HEADERS.DEPLOYMENT_VERSION]: options.deploymentVersion,
+    [WORKLOAD_HEADERS.PROJECT_REF]: options.projectRef,
   });
 }
diff --git a/packages/core/src/v3/schemas/checkpoints.ts b/packages/core/src/v3/schemas/checkpoints.ts
index 8cb5ecc32d..d255c035cf 100644
--- a/packages/core/src/v3/schemas/checkpoints.ts
+++ b/packages/core/src/v3/schemas/checkpoints.ts
@@ -1,4 +1,4 @@
-import { CheckpointType } from "./runEngine.js";
+import { CheckpointType, DequeuedMessage } from "./runEngine.js";
 import z from "zod";
 
 const CallbackUrl = z
@@ -8,20 +8,12 @@ const CallbackUrl = z
 
 export const CheckpointServiceSuspendRequestBody = z.object({
   type: CheckpointType,
-  containerId: z.string(),
-  simulate: z.boolean().optional(),
-  leaveRunning: z.boolean().optional(),
+  runId: z.string(),
+  snapshotId: z.string(),
+  runnerId: z.string(),
+  projectRef: z.string(),
+  deploymentVersion: z.string(),
   reason: z.string().optional(),
-  callbacks: z
-    .object({
-      /** These headers will sent to all callbacks */
-      headers: z.record(z.string()).optional(),
-      /** This will be hit before suspending the container. Suspension will proceed unless we receive an error response. */
-      preSuspend: CallbackUrl.optional(),
-      /** This will be hit after suspending or failure to suspend the container */
-      completion: CallbackUrl.optional(),
-    })
-    .optional(),
 });
 
 export type CheckpointServiceSuspendRequestBody = z.infer<
@@ -39,16 +31,7 @@ export type CheckpointServiceSuspendResponseBody = z.infer<
   typeof CheckpointServiceSuspendResponseBody
 >;
 
-export const CheckpointServiceRestoreRequestBody = z.discriminatedUnion("type", [
-  z.object({
-    type: z.literal(CheckpointType.Enum.DOCKER),
-    containerId: z.string(),
-  }),
-  z.object({
-    type: z.literal(CheckpointType.Enum.KUBERNETES),
-    containerId: z.string(),
-  }),
-]);
+export const CheckpointServiceRestoreRequestBody = DequeuedMessage.required({ checkpoint: true });
 
 export type CheckpointServiceRestoreRequestBody = z.infer<
   typeof CheckpointServiceRestoreRequestBody
diff --git a/packages/core/src/v3/schemas/runEngine.ts b/packages/core/src/v3/schemas/runEngine.ts
index db0c9221cd..2e810041a8 100644
--- a/packages/core/src/v3/schemas/runEngine.ts
+++ b/packages/core/src/v3/schemas/runEngine.ts
@@ -123,52 +123,6 @@ export const ExecutionResult = z.object({
 
 export type ExecutionResult = z.infer<typeof ExecutionResult>;
 
-/** This is sent to a Worker when a run is dequeued (a new run or continuing run) */
-export const DequeuedMessage = z.object({
-  version: z.literal("1"),
-  snapshot: ExecutionSnapshot,
-  dequeuedAt: z.coerce.date(),
-  image: z.string().optional(),
-  checkpoint: z
-    .object({
-      id: z.string(),
-      type: z.string(),
-      location: z.string(),
-      reason: z.string().nullish(),
-    })
-    .optional(),
-  completedWaitpoints: z.array(CompletedWaitpoint),
-  backgroundWorker: z.object({
-    id: z.string(),
-    friendlyId: z.string(),
-    version: z.string(),
-  }),
-  deployment: z.object({
-    id: z.string().optional(),
-    friendlyId: z.string().optional(),
-  }),
-  run: z.object({
-    id: z.string(),
-    friendlyId: z.string(),
-    isTest: z.boolean(),
-    machine: MachinePreset,
-    attemptNumber: z.number(),
-    masterQueue: z.string(),
-    traceContext: z.record(z.unknown()),
-  }),
-  environment: z.object({
-    id: z.string(),
-    type: EnvironmentType,
-  }),
-  organization: z.object({
-    id: z.string(),
-  }),
-  project: z.object({
-    id: z.string(),
-  }),
-});
-export type DequeuedMessage = z.infer<typeof DequeuedMessage>;
-
 /** The response to the Worker when starting an attempt */
 export const StartRunAttemptResult = ExecutionResult.and(
   z.object({
@@ -256,3 +210,51 @@ export const MachineResources = z.object({
   memory: z.number(),
 });
 export type MachineResources = z.infer<typeof MachineResources>;
+
+export const DequeueMessageCheckpoint = z.object({
+  id: z.string(),
+  type: CheckpointType,
+  location: z.string(),
+  imageRef: z.string(),
+  reason: z.string().nullish(),
+});
+export type DequeueMessageCheckpoint = z.infer<typeof DequeueMessageCheckpoint>;
+
+/** This is sent to a Worker when a run is dequeued (a new run or continuing run) */
+export const DequeuedMessage = z.object({
+  version: z.literal("1"),
+  snapshot: ExecutionSnapshot,
+  dequeuedAt: z.coerce.date(),
+  image: z.string().optional(),
+  checkpoint: DequeueMessageCheckpoint.optional(),
+  completedWaitpoints: z.array(CompletedWaitpoint),
+  backgroundWorker: z.object({
+    id: z.string(),
+    friendlyId: z.string(),
+    version: z.string(),
+  }),
+  deployment: z.object({
+    id: z.string().optional(),
+    friendlyId: z.string().optional(),
+  }),
+  run: z.object({
+    id: z.string(),
+    friendlyId: z.string(),
+    isTest: z.boolean(),
+    machine: MachinePreset,
+    attemptNumber: z.number(),
+    masterQueue: z.string(),
+    traceContext: z.record(z.unknown()),
+  }),
+  environment: z.object({
+    id: z.string(),
+    type: EnvironmentType,
+  }),
+  organization: z.object({
+    id: z.string(),
+  }),
+  project: z.object({
+    id: z.string(),
+  }),
+});
+export type DequeuedMessage = z.infer<typeof DequeuedMessage>;
diff --git a/packages/core/src/v3/serverOnly/checkpointClient.ts b/packages/core/src/v3/serverOnly/checkpointClient.ts
index 4f441ad8cf..57bdb09ab3 100644
--- a/packages/core/src/v3/serverOnly/checkpointClient.ts
+++ b/packages/core/src/v3/serverOnly/checkpointClient.ts
@@ -4,49 +4,42 @@ import {
   CheckpointServiceSuspendResponseBody,
   CheckpointServiceRestoreRequestBodyInput,
 } from "../schemas/checkpoints.js";
-import { DequeuedMessage } from "../schemas/runEngine.js";
+import { CheckpointType, DequeuedMessage } from "../schemas/runEngine.js";
 import { SimpleStructuredLogger } from "../utils/structuredLogger.js";
 
 export type CheckpointClientOptions = {
   apiUrl: URL;
   workerClient: SupervisorHttpClient;
+  orchestrator: CheckpointType;
 };
 
 export class CheckpointClient {
   private readonly logger = new SimpleStructuredLogger("checkpoint-client");
-  private readonly apiUrl: URL;
-  private readonly workerClient: SupervisorHttpClient;
 
-  private get restoreUrl() {
-    return new URL("/api/v1/restore", this.apiUrl);
-  }
-
-  constructor(opts: CheckpointClientOptions) {
-    this.apiUrl = opts.apiUrl;
-    this.workerClient = opts.workerClient;
-  }
+  constructor(private readonly opts: CheckpointClientOptions) {}
 
   async suspendRun({
     runFriendlyId,
     snapshotFriendlyId,
-    containerId,
-    runnerId,
+    body,
   }: {
     runFriendlyId: string;
     snapshotFriendlyId: string;
-    containerId: string;
-    runnerId: string;
+    body: Omit<CheckpointServiceSuspendRequestBodyInput, "type">;
   }): Promise<boolean> {
     const res = await fetch(
-      new URL(`/api/v1/runs/${runFriendlyId}/snapshots/${snapshotFriendlyId}/suspend`, this.apiUrl),
+      new URL(
+        `/api/v1/runs/${runFriendlyId}/snapshots/${snapshotFriendlyId}/suspend`,
+        this.opts.apiUrl
+      ),
       {
         method: "POST",
         headers: {
           "Content-Type": "application/json",
         },
         body: JSON.stringify({
-          type: "DOCKER",
-          containerId,
+          type: this.opts.orchestrator,
+          ...body,
         } satisfies CheckpointServiceSuspendRequestBodyInput),
       }
     );
@@ -55,7 +48,7 @@ export class CheckpointClient {
       this.logger.error("[CheckpointClient] Suspend request failed", {
         runFriendlyId,
         snapshotFriendlyId,
-        containerId,
+        body,
       });
       return false;
     }
@@ -63,7 +56,7 @@ export class CheckpointClient {
     this.logger.debug("[CheckpointClient] Suspend request success", {
       runFriendlyId,
       snapshotFriendlyId,
-      containerId,
+      body,
       status: res.status,
       contentType: res.headers.get("content-type"),
     });
@@ -76,7 +69,7 @@ export class CheckpointClient {
         this.logger.error("[CheckpointClient] Suspend response invalid", {
           runFriendlyId,
           snapshotFriendlyId,
-          containerId,
+          body,
           data,
         });
         return false;
@@ -95,28 +88,31 @@ export class CheckpointClient {
   async restoreRun({
     runFriendlyId,
     snapshotFriendlyId,
-    checkpoint,
+    body,
   }: {
     runFriendlyId: string;
     snapshotFriendlyId: string;
-    checkpoint: NonNullable<DequeuedMessage["checkpoint"]>;
+    body: CheckpointServiceRestoreRequestBodyInput;
   }): Promise<boolean> {
-    const res = await fetch(this.restoreUrl, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-      },
-      body: JSON.stringify({
-        type: "DOCKER",
-        containerId: checkpoint.location,
-      } satisfies CheckpointServiceRestoreRequestBodyInput),
-    });
+    const res = await fetch(
+      new URL(
+        `/api/v1/runs/${runFriendlyId}/snapshots/${snapshotFriendlyId}/restore`,
+        this.opts.apiUrl
+      ),
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+        },
+        body: JSON.stringify(body),
+      }
+    );
 
     if (!res.ok) {
       this.logger.error("[CheckpointClient] Restore request failed", {
         runFriendlyId,
         snapshotFriendlyId,
-        checkpoint,
+        body,
       });
       return false;
     }
diff --git a/packages/core/src/v3/serverOnly/index.ts b/packages/core/src/v3/serverOnly/index.ts
index 3b311c7904..05e7b08a24 100644
--- a/packages/core/src/v3/serverOnly/index.ts
+++ b/packages/core/src/v3/serverOnly/index.ts
@@ -3,3 +3,4 @@ export * from "./checkpointTest.js";
 export * from "./httpServer.js";
 export * from "./singleton.js";
 export * from "./shutdownManager.js";
+export * from "./k8s.js";
diff --git a/packages/core/src/v3/serverOnly/k8s.ts b/packages/core/src/v3/serverOnly/k8s.ts
new file mode 100644
index 0000000000..333166a038
--- /dev/null
+++ b/packages/core/src/v3/serverOnly/k8s.ts
@@ -0,0 +1,16 @@
+import { env } from "std-env";
+
+export function isKubernetesEnvironment(override?: boolean): boolean {
+  if (override !== undefined) {
+    return override;
+  }
+
+  // Then check for common Kubernetes environment variables
+  const k8sIndicators = [
+    env.KUBERNETES_PORT,
+    env.KUBERNETES_SERVICE_HOST,
+    env.KUBERNETES_SERVICE_PORT,
+  ];
+
+  return k8sIndicators.some(Boolean);
+}
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index be35085c66..41c06273c6 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1405,6 +1405,9 @@ importers:
       socket.io-client:
         specifier: 4.7.5
         version: 4.7.5
+      std-env:
+        specifier: ^3.8.1
+        version: 3.8.1
       superjson:
         specifier: ^2.2.1
         version: 2.2.1

From b3eeb487a499344b1922d8e0dcf54197426e172f Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Thu, 20 Mar 2025 20:52:42 +0000
Subject: [PATCH 16/26] add http server metrics

---
 packages/core/src/v3/serverOnly/httpServer.ts | 98 ++++++++++++++++---
 pnpm-lock.yaml                                |  3 +
 2 files changed, 86 insertions(+), 15 deletions(-)

diff --git a/packages/core/src/v3/serverOnly/httpServer.ts b/packages/core/src/v3/serverOnly/httpServer.ts
index 30c06db5fd..1e68ec4ca7 100644
--- a/packages/core/src/v3/serverOnly/httpServer.ts
+++ b/packages/core/src/v3/serverOnly/httpServer.ts
@@ -2,6 +2,8 @@ import { createServer, type IncomingMessage, type ServerResponse } from "node:ht
 import { z } from "zod";
 import { SimpleStructuredLogger } from "../utils/structuredLogger.js";
 import { HttpReply, getJsonBody } from "../apps/http.js";
+import { Registry, Histogram, Counter } from "prom-client";
+import { tryCatch } from "../../utils.js";
 
 const logger = new SimpleStructuredLogger("worker-http");
 
@@ -51,21 +53,72 @@ type RouteMap = Partial<{
 type HttpServerOptions = {
   port: number;
   host: string;
+  metrics?: {
+    register?: Registry;
+    expose?: boolean;
+    collect?: boolean;
+  };
 };
 
 export class HttpServer {
+  private static httpRequestDuration?: Histogram;
+  private static httpRequestTotal?: Counter;
+
+  private readonly metricsRegister?: Registry;
+
   private readonly port: number;
   private readonly host: string;
   private routes: RouteMap = {};
-
   public readonly server: ReturnType<typeof createServer>;
 
   constructor(options: HttpServerOptions) {
     this.port = options.port;
     this.host = options.host;
+    this.metricsRegister = options.metrics?.register;
+    const collectMetrics = options.metrics?.collect ?? true;
+    const exposeMetrics = options.metrics?.expose ?? false;
+
+    // Initialize metrics only if registry is provided and not already initialized
+    if (this.metricsRegister && collectMetrics) {
+      if (!HttpServer.httpRequestDuration) {
+        HttpServer.httpRequestDuration = new Histogram({
+          name: "http_request_duration_seconds",
+          help: "Duration of HTTP requests in seconds",
+          labelNames: ["method", "route", "status", "port", "host"],
+          registers: [this.metricsRegister],
+        });
+      }
+
+      if (!HttpServer.httpRequestTotal) {
+        HttpServer.httpRequestTotal = new Counter({
+          name: "http_requests_total",
+          help: "Total number of HTTP requests",
+          labelNames: ["method", "route", "status", "port", "host"],
+          registers: [this.metricsRegister],
+        });
+      }
+    }
+
+    if (exposeMetrics) {
+      // Register metrics route
+      this.route("/metrics", "GET", {
+        handler: async ({ reply }) => {
+          if (!this.metricsRegister) {
+            return reply.text("Metrics registry not found", 500);
+          }
+
+          return reply.text(
+            await this.metricsRegister.metrics(),
+            200,
+            this.metricsRegister.contentType
+          );
+        },
+      });
+    }
 
     this.server = createServer(async (req, res) => {
       const reply = new HttpReply(res);
+      const startTime = process.hrtime();
 
       try {
         const { url, method } = req;
@@ -98,13 +151,6 @@ export class HttpServer {
 
         const routeDefinition = this.routes[route]?.[httpMethod.data];
 
-        // logger.debug("Matched route", {
-        //   url,
-        //   method,
-        //   route,
-        //   routeDefinition,
-        // });
-
         if (!routeDefinition) {
           logger.error("Invalid method", { url, method, parsedMethod: httpMethod.data });
           return reply.empty(405);
@@ -141,25 +187,28 @@ export class HttpServer {
           return reply.json({ ok: false, error: "Invalid body" }, false, 400);
         }
 
-        try {
-          await handler({
+        const [error] = await tryCatch(
+          handler({
             reply,
             req,
             res,
             params: parsedParams.data,
             queryParams: parsedQueryParams.data,
             body: parsedBody.data,
-          });
-        } catch (handlerError) {
-          logger.error("Route handler error", { error: handlerError });
+          })
+        );
+
+        if (error) {
+          logger.error("Route handler error", { error });
           return reply.empty(500);
         }
       } catch (error) {
         logger.error("Failed to handle request", { error });
         return reply.empty(500);
+      } finally {
+        this.collectMetrics(req, res, startTime);
+        return;
       }
-
-      return;
     });
 
     this.server.on("clientError", (_, socket) => {
@@ -197,6 +246,25 @@ export class HttpServer {
     });
   }
 
+  private collectMetrics(req: IncomingMessage, res: ServerResponse, startTime: [number, number]) {
+    if (!this.metricsRegister || !HttpServer.httpRequestDuration || !HttpServer.httpRequestTotal) {
+      return;
+    }
+
+    const [seconds, nanoseconds] = process.hrtime(startTime);
+    const duration = seconds + nanoseconds / 1e9;
+
+    const route = this.findRoute(req.url ?? "") ?? "unknown";
+    const method = req.method ?? "unknown";
+    const status = res.statusCode.toString();
+
+    HttpServer.httpRequestDuration.observe(
+      { method, route, status, port: this.port, host: this.host },
+      duration
+    );
+    HttpServer.httpRequestTotal.inc({ method, route, status, port: this.port, host: this.host });
+  }
+
   private optionalSchema<
     TSchema extends z.ZodFirstPartySchemaTypes | undefined,
     TData extends TSchema extends z.ZodFirstPartySchemaTypes ? z.TypeOf<TSchema> : TData,
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 41c06273c6..48ecfdd8e7 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1399,6 +1399,9 @@ importers:
       nanoid:
         specifier: ^3.3.4
         version: 3.3.7
+      prom-client:
+        specifier: ^15.1.0
+        version: 15.1.0
       socket.io:
         specifier: 4.7.4
         version: 4.7.4

From 4d45f5235b59381754306c7628ea3038af2c4e42 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Fri, 21 Mar 2025 08:51:59 +0000
Subject: [PATCH 17/26] add missing prom-client to core

---
 packages/core/package.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/core/package.json b/packages/core/package.json
index d58795e09c..d98520608e 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -189,6 +189,7 @@
     "humanize-duration": "^3.27.3",
     "jose": "^5.4.0",
     "nanoid": "^3.3.4",
+    "prom-client": "^15.1.0",
     "socket.io": "4.7.4",
     "socket.io-client": "4.7.5",
     "std-env": "^3.8.1",

From 05214e8bb9e124d6f76f72ec6f6d0fb6142d8491 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Fri, 21 Mar 2025 08:52:50 +0000
Subject: [PATCH 18/26] add prom metrics to redis worker

---
 packages/redis-worker/package.json  |   7 +-
 packages/redis-worker/src/worker.ts | 142 +++++++++++++++++++++++++---
 pnpm-lock.yaml                      |   3 +
 3 files changed, 136 insertions(+), 16 deletions(-)

diff --git a/packages/redis-worker/package.json b/packages/redis-worker/package.json
index 7da5ad3c4e..bcfdd468a9 100644
--- a/packages/redis-worker/package.json
+++ b/packages/redis-worker/package.json
@@ -38,21 +38,22 @@
     "test": "vitest --sequence.concurrent=false --no-file-parallelism"
   },
   "dependencies": {
-    "@internal/tracing": "workspace:*",
     "@internal/redis": "workspace:*",
+    "@internal/tracing": "workspace:*",
     "@trigger.dev/core": "workspace:*",
     "lodash.omit": "^4.5.0",
     "nanoid": "^5.0.7",
     "p-limit": "^6.2.0",
+    "prom-client": "^15.1.0",
     "zod": "3.23.8"
   },
   "devDependencies": {
     "@internal/testcontainers": "workspace:*",
     "@types/lodash.omit": "^4.5.7",
-    "vitest": "^1.4.0",
     "rimraf": "6.0.1",
     "tshy": "^3.0.2",
-    "tsx": "4.17.0"
+    "tsx": "4.17.0",
+    "vitest": "^1.4.0"
   },
   "engines": {
     "node": ">=18.20.0"
diff --git a/packages/redis-worker/src/worker.ts b/packages/redis-worker/src/worker.ts
index 8dca728f29..d01eefa97d 100644
--- a/packages/redis-worker/src/worker.ts
+++ b/packages/redis-worker/src/worker.ts
@@ -9,6 +9,7 @@ import { nanoid } from "nanoid";
 import pLimit from "p-limit";
 import { createRedisClient } from "@internal/redis";
 import { shutdownManager } from "@trigger.dev/core/v3/serverOnly";
+import { Registry, Histogram } from "prom-client";
 
 export type WorkerCatalog = {
   [key: string]: {
@@ -48,6 +49,9 @@ type WorkerOptions<TCatalog extends WorkerCatalog> = {
   shutdownTimeoutMs?: number;
   logger?: Logger;
   tracer?: Tracer;
+  metrics?: {
+    register: Registry;
+  };
 };
 
 // This results in attempt 12 being a delay of 1 hour
@@ -65,6 +69,16 @@ class Worker<TCatalog extends WorkerCatalog> {
   private subscriber: Redis | undefined;
   private tracer: Tracer;
 
+  private metrics: {
+    register?: Registry;
+    enqueueDuration?: Histogram;
+    dequeueDuration?: Histogram;
+    jobDuration?: Histogram;
+    ackDuration?: Histogram;
+    redriveDuration?: Histogram;
+    rescheduleDuration?: Histogram;
+  } = {};
+
   queue: SimpleQueue<QueueCatalogFromWorkerCatalog<TCatalog>>;
   private jobs: WorkerOptions<TCatalog>["jobs"];
   private logger: Logger;
@@ -100,6 +114,61 @@ class Worker<TCatalog extends WorkerCatalog> {
 
     // Create a p-limit instance using this limit.
     this.limiter = pLimit(this.concurrency.limit);
+
+    this.metrics.register = options.metrics?.register;
+
+    if (!this.metrics.register) {
+      return;
+    }
+
+    this.metrics.enqueueDuration = new Histogram({
+      name: "redis_worker_enqueue_duration_seconds",
+      help: "The duration of enqueue operations",
+      labelNames: ["worker_name", "job_type", "has_available_at"],
+      buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
+      registers: [this.metrics.register],
+    });
+
+    this.metrics.dequeueDuration = new Histogram({
+      name: "redis_worker_dequeue_duration_seconds",
+      help: "The duration of dequeue operations",
+      labelNames: ["worker_name", "worker_id", "task_count"],
+      buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
+      registers: [this.metrics.register],
+    });
+
+    this.metrics.jobDuration = new Histogram({
+      name: "redis_worker_job_duration_seconds",
+      help: "The duration of job operations",
+      labelNames: ["worker_name", "worker_id", "batch_size", "job_type", "attempt"],
+      // use different buckets here as jobs can take a while to run
+      buckets: [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 20, 30, 45, 60],
+      registers: [this.metrics.register],
+    });
+
+    this.metrics.ackDuration = new Histogram({
+      name: "redis_worker_ack_duration_seconds",
+      help: "The duration of ack operations",
+      labelNames: ["worker_name"],
+      buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
+      registers: [this.metrics.register],
+    });
+
+    this.metrics.redriveDuration = new Histogram({
+      name: "redis_worker_redrive_duration_seconds",
+      help: "The duration of redrive operations",
+      labelNames: ["worker_name"],
+      buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
+      registers: [this.metrics.register],
+    });
+
+    this.metrics.rescheduleDuration = new Histogram({
+      name: "redis_worker_reschedule_duration_seconds",
+      help: "The duration of reschedule operations",
+      labelNames: ["worker_name"],
+      buckets: [0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1],
+      registers: [this.metrics.register],
+    });
   }
 
   public start() {
@@ -160,18 +229,25 @@ class Worker<TCatalog extends WorkerCatalog> {
 
         span.setAttribute("job_visibility_timeout_ms", timeout);
 
-        return this.queue.enqueue({
-          id,
-          job,
-          item: payload,
-          visibilityTimeoutMs: timeout,
-          availableAt,
-        });
+        return this.withHistogram(
+          this.metrics.enqueueDuration,
+          this.queue.enqueue({
+            id,
+            job,
+            item: payload,
+            visibilityTimeoutMs: timeout,
+            availableAt,
+          }),
+          {
+            job_type: String(job),
+            has_available_at: availableAt ? "true" : "false",
+          }
+        );
       },
       {
         kind: SpanKind.PRODUCER,
         attributes: {
-          job_type: job as string,
+          job_type: String(job),
           job_id: id,
         },
       }
@@ -187,7 +263,10 @@ class Worker<TCatalog extends WorkerCatalog> {
       this.tracer,
       "reschedule",
       async (span) => {
-        return this.queue.reschedule(id, availableAt);
+        return this.withHistogram(
+          this.metrics.rescheduleDuration,
+          this.queue.reschedule(id, availableAt)
+        );
       },
       {
         kind: SpanKind.PRODUCER,
@@ -203,7 +282,7 @@ class Worker<TCatalog extends WorkerCatalog> {
       this.tracer,
       "ack",
       () => {
-        return this.queue.ack(id);
+        return this.withHistogram(this.metrics.ackDuration, this.queue.ack(id));
       },
       {
         attributes: {
@@ -229,7 +308,14 @@ class Worker<TCatalog extends WorkerCatalog> {
       }
 
       try {
-        const items = await this.queue.dequeue(taskCount);
+        const items = await this.withHistogram(
+          this.metrics.dequeueDuration,
+          this.queue.dequeue(taskCount),
+          {
+            worker_id: workerId,
+            task_count: taskCount,
+          }
+        );
 
         if (items.length === 0) {
           await Worker.delay(pollIntervalMs);
@@ -274,7 +360,17 @@ class Worker<TCatalog extends WorkerCatalog> {
       this.tracer,
       "processItem",
       async () => {
-        await handler({ id, payload: item, visibilityTimeoutMs, attempt });
+        await this.withHistogram(
+          this.metrics.jobDuration,
+          handler({ id, payload: item, visibilityTimeoutMs, attempt }),
+          {
+            worker_id: workerId,
+            batch_size: batchSize,
+            job_type: job,
+            attempt,
+          }
+        );
+
         // On success, acknowledge the item.
         await this.queue.ack(id);
       },
@@ -363,6 +459,23 @@ class Worker<TCatalog extends WorkerCatalog> {
     });
   }
 
+  private async withHistogram<T>(
+    histogram: Histogram<string> | undefined,
+    promise: Promise<T>,
+    labels?: Record<string, string | number>
+  ): Promise<T> {
+    if (!histogram || !this.metrics.register) {
+      return promise;
+    }
+
+    const end = histogram.startTimer({ worker_name: this.options.name, ...labels });
+    try {
+      return await promise;
+    } finally {
+      end();
+    }
+  }
+
   // A simple helper to delay for a given number of milliseconds.
   private static delay(ms: number): Promise<void> {
     return new Promise((resolve) => setTimeout(resolve, ms));
@@ -387,7 +500,10 @@ class Worker<TCatalog extends WorkerCatalog> {
       if (typeof id !== "string") {
         throw new Error("Invalid message format: id must be a string");
       }
-      await this.queue.redriveFromDeadLetterQueue(id);
+      await this.withHistogram(
+        this.metrics.redriveDuration,
+        this.queue.redriveFromDeadLetterQueue(id)
+      );
       this.logger.log(`Redrived item ${id} from Dead Letter Queue`);
     } catch (error) {
       this.logger.error("Error processing redrive message", { error, message });
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 48ecfdd8e7..017e4ddffc 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1561,6 +1561,9 @@ importers:
       p-limit:
         specifier: ^6.2.0
         version: 6.2.0
+      prom-client:
+        specifier: ^15.1.0
+        version: 15.1.0
       zod:
         specifier: 3.23.8
         version: 3.23.8

From ba57e069bc5b6c1fad8a462457fa021f07cc2c19 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Fri, 21 Mar 2025 11:01:03 +0000
Subject: [PATCH 19/26] bundle redis-worker

---
 packages/redis-worker/package.json   |  42 +-
 packages/redis-worker/tsup.config.ts |  27 ++
 pnpm-lock.yaml                       | 635 ++++++++++++++++++++++++++-
 3 files changed, 654 insertions(+), 50 deletions(-)
 create mode 100644 packages/redis-worker/tsup.config.ts

diff --git a/packages/redis-worker/package.json b/packages/redis-worker/package.json
index bcfdd468a9..bea9698613 100644
--- a/packages/redis-worker/package.json
+++ b/packages/redis-worker/package.json
@@ -15,31 +15,14 @@
   "files": [
     "dist"
   ],
-  "tshy": {
-    "selfLink": false,
-    "module": true,
-    "project": "./tsconfig.src.json",
-    "dialects": [
-      "esm"
-    ],
-    "exports": {
-      "./package.json": "./package.json",
-      ".": "./src/index.ts"
-    },
-    "sourceDialects": [
-      "@triggerdotdev/source"
-    ]
-  },
   "scripts": {
-    "clean": "rimraf dist .tshy .tshy-build .turbo",
-    "build": "tshy",
-    "dev": "tshy --watch",
+    "clean": "rimraf dist .turbo",
+    "build": "tsup",
+    "dev": "tsup --watch",
     "typecheck": "tsc --noEmit -p tsconfig.src.json",
     "test": "vitest --sequence.concurrent=false --no-file-parallelism"
   },
   "dependencies": {
-    "@internal/redis": "workspace:*",
-    "@internal/tracing": "workspace:*",
     "@trigger.dev/core": "workspace:*",
     "lodash.omit": "^4.5.0",
     "nanoid": "^5.0.7",
@@ -48,25 +31,26 @@
     "zod": "3.23.8"
   },
   "devDependencies": {
+    "@internal/redis": "workspace:*",
     "@internal/testcontainers": "workspace:*",
+    "@internal/tracing": "workspace:*",
     "@types/lodash.omit": "^4.5.7",
     "rimraf": "6.0.1",
-    "tshy": "^3.0.2",
+    "tsup": "^8.4.0",
     "tsx": "4.17.0",
     "vitest": "^1.4.0"
   },
   "engines": {
     "node": ">=18.20.0"
   },
+  "main": "./dist/index.cjs",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
   "exports": {
-    "./package.json": "./package.json",
     ".": {
-      "import": {
-        "@triggerdotdev/source": "./src/index.ts",
-        "types": "./dist/esm/index.d.ts",
-        "default": "./dist/esm/index.js"
-      }
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
     }
-  },
-  "module": "./dist/esm/index.js"
+  }
 }
diff --git a/packages/redis-worker/tsup.config.ts b/packages/redis-worker/tsup.config.ts
new file mode 100644
index 0000000000..fdcfbe41fd
--- /dev/null
+++ b/packages/redis-worker/tsup.config.ts
@@ -0,0 +1,27 @@
+import { defineConfig } from "tsup";
+
+export default defineConfig({
+  entry: ["src/index.ts"],
+  format: ["cjs", "esm"],
+  dts: true,
+  splitting: false,
+  sourcemap: true,
+  clean: true,
+  treeshake: true,
+  bundle: true,
+  minify: false,
+  noExternal: [
+    // Always bundle internal packages
+    /^@internal/,
+    // Always bundle ESM-only packages
+    "nanoid",
+    "p-limit",
+  ],
+  banner: ({ format }) => {
+    if (format !== "esm") return;
+
+    return {
+      js: `import { createRequire } from 'module'; const require = createRequire(import.meta.url);`,
+    };
+  },
+});
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 017e4ddffc..6949a028ef 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -1543,12 +1543,6 @@ importers:
 
   packages/redis-worker:
     dependencies:
-      '@internal/redis':
-        specifier: workspace:*
-        version: link:../../internal-packages/redis
-      '@internal/tracing':
-        specifier: workspace:*
-        version: link:../../internal-packages/tracing
       '@trigger.dev/core':
         specifier: workspace:*
         version: link:../core
@@ -1568,18 +1562,24 @@ importers:
         specifier: 3.23.8
         version: 3.23.8
     devDependencies:
+      '@internal/redis':
+        specifier: workspace:*
+        version: link:../../internal-packages/redis
       '@internal/testcontainers':
         specifier: workspace:*
         version: link:../../internal-packages/testcontainers
+      '@internal/tracing':
+        specifier: workspace:*
+        version: link:../../internal-packages/tracing
       '@types/lodash.omit':
         specifier: ^4.5.7
         version: 4.5.7
       rimraf:
         specifier: 6.0.1
         version: 6.0.1
-      tshy:
-        specifier: ^3.0.2
-        version: 3.0.2
+      tsup:
+        specifier: ^8.4.0
+        version: 8.4.0(postcss@8.4.44)(tsx@4.17.0)(typescript@5.5.4)
       tsx:
         specifier: 4.17.0
         version: 4.17.0
@@ -5891,13 +5891,13 @@ packages:
   /@electric-sql/client@0.4.0:
     resolution: {integrity: sha512-YVYSqHitqVIDC1RBTfmHMfAfqDNAKMK9/AFVTDFQQxN3Q85dIQS49zThAuJVecYiuYRJvTiqf40c4n39jZSNrQ==}
     optionalDependencies:
-      '@rollup/rollup-darwin-arm64': 4.21.3
+      '@rollup/rollup-darwin-arm64': 4.36.0
     dev: false
 
   /@electric-sql/client@1.0.0-beta.1:
     resolution: {integrity: sha512-Ei9jN3pDoGzc+a/bGqnB5ajb52IvSv7/n2btuyzUlcOHIR2kM9fqtYTJXPwZYKLkGZlHWlpHgWyRtrinkP2nHg==}
     optionalDependencies:
-      '@rollup/rollup-darwin-arm64': 4.21.3
+      '@rollup/rollup-darwin-arm64': 4.36.0
     dev: false
 
   /@electric-sql/react@0.3.5(react@18.2.0):
@@ -6010,6 +6010,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/aix-ppc64@0.25.1:
+    resolution: {integrity: sha512-kfYGy8IdzTGy+z0vFGvExZtxkFlA4zAxgKEahG9KE1ScBjpQnFsNOX8KTU5ojNru5ed5CVoJYXFtoxaq5nFbjQ==}
+    engines: {node: '>=18'}
+    cpu: [ppc64]
+    os: [aix]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/android-arm64@0.16.17:
     resolution: {integrity: sha512-MIGl6p5sc3RDTLLkYL1MyL8BMRN4tLMRCn+yRJJmEDvYZ2M7tmAf80hx1kbNEUX2KJ50RRtxZ4JHLvCfuB6kBg==}
     engines: {node: '>=12'}
@@ -6071,6 +6080,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/android-arm64@0.25.1:
+    resolution: {integrity: sha512-50tM0zCJW5kGqgG7fQ7IHvQOcAn9TKiVRuQ/lN0xR+T2lzEFvAi1ZcS8DiksFcEpf1t/GYOeOfCAgDHFpkiSmA==}
+    engines: {node: '>=18'}
+    cpu: [arm64]
+    os: [android]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/android-arm@0.15.18:
     resolution: {integrity: sha512-5GT+kcs2WVGjVs7+boataCkO5Fg0y4kCjzkB5bAip7H4jfnOS3dA6KPiww9W1OEKTKeAcUVhdZGvgI65OXmUnw==}
     engines: {node: '>=12'}
@@ -6141,6 +6159,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/android-arm@0.25.1:
+    resolution: {integrity: sha512-dp+MshLYux6j/JjdqVLnMglQlFu+MuVeNrmT5nk6q07wNhCdSnB7QZj+7G8VMUGh1q+vj2Bq8kRsuyA00I/k+Q==}
+    engines: {node: '>=18'}
+    cpu: [arm]
+    os: [android]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/android-x64@0.16.17:
     resolution: {integrity: sha512-a3kTv3m0Ghh4z1DaFEuEDfz3OLONKuFvI4Xqczqx4BqLyuFaFkuaG4j2MtA6fuWEFeC5x9IvqnX7drmRq/fyAQ==}
     engines: {node: '>=12'}
@@ -6202,6 +6229,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/android-x64@0.25.1:
+    resolution: {integrity: sha512-GCj6WfUtNldqUzYkN/ITtlhwQqGWu9S45vUXs7EIYf+7rCiiqH9bCloatO9VhxsL0Pji+PF4Lz2XXCES+Q8hDw==}
+    engines: {node: '>=18'}
+    cpu: [x64]
+    os: [android]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/darwin-arm64@0.16.17:
     resolution: {integrity: sha512-/2agbUEfmxWHi9ARTX6OQ/KgXnOWfsNlTeLcoV7HSuSTv63E4DqtAc+2XqGw1KHxKMHGZgbVCZge7HXWX9Vn+w==}
     engines: {node: '>=12'}
@@ -6263,6 +6299,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/darwin-arm64@0.25.1:
+    resolution: {integrity: sha512-5hEZKPf+nQjYoSr/elb62U19/l1mZDdqidGfmFutVUjjUZrOazAtwK+Kr+3y0C/oeJfLlxo9fXb1w7L+P7E4FQ==}
+    engines: {node: '>=18'}
+    cpu: [arm64]
+    os: [darwin]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/darwin-x64@0.16.17:
     resolution: {integrity: sha512-2By45OBHulkd9Svy5IOCZt376Aa2oOkiE9QWUK9fe6Tb+WDr8hXL3dpqi+DeLiMed8tVXspzsTAvd0jUl96wmg==}
     engines: {node: '>=12'}
@@ -6324,6 +6369,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/darwin-x64@0.25.1:
+    resolution: {integrity: sha512-hxVnwL2Dqs3fM1IWq8Iezh0cX7ZGdVhbTfnOy5uURtao5OIVCEyj9xIzemDi7sRvKsuSdtCAhMKarxqtlyVyfA==}
+    engines: {node: '>=18'}
+    cpu: [x64]
+    os: [darwin]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/freebsd-arm64@0.16.17:
     resolution: {integrity: sha512-mt+cxZe1tVx489VTb4mBAOo2aKSnJ33L9fr25JXpqQqzbUIw/yzIzi+NHwAXK2qYV1lEFp4OoVeThGjUbmWmdw==}
     engines: {node: '>=12'}
@@ -6385,6 +6439,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/freebsd-arm64@0.25.1:
+    resolution: {integrity: sha512-1MrCZs0fZa2g8E+FUo2ipw6jw5qqQiH+tERoS5fAfKnRx6NXH31tXBKI3VpmLijLH6yriMZsxJtaXUyFt/8Y4A==}
+    engines: {node: '>=18'}
+    cpu: [arm64]
+    os: [freebsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/freebsd-x64@0.16.17:
     resolution: {integrity: sha512-8ScTdNJl5idAKjH8zGAsN7RuWcyHG3BAvMNpKOBaqqR7EbUhhVHOqXRdL7oZvz8WNHL2pr5+eIT5c65kA6NHug==}
     engines: {node: '>=12'}
@@ -6446,6 +6509,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/freebsd-x64@0.25.1:
+    resolution: {integrity: sha512-0IZWLiTyz7nm0xuIs0q1Y3QWJC52R8aSXxe40VUxm6BB1RNmkODtW6LHvWRrGiICulcX7ZvyH6h5fqdLu4gkww==}
+    engines: {node: '>=18'}
+    cpu: [x64]
+    os: [freebsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/linux-arm64@0.16.17:
     resolution: {integrity: sha512-7S8gJnSlqKGVJunnMCrXHU9Q8Q/tQIxk/xL8BqAP64wchPCTzuM6W3Ra8cIa1HIflAvDnNOt2jaL17vaW+1V0g==}
     engines: {node: '>=12'}
@@ -6507,6 +6579,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/linux-arm64@0.25.1:
+    resolution: {integrity: sha512-jaN3dHi0/DDPelk0nLcXRm1q7DNJpjXy7yWaWvbfkPvI+7XNSc/lDOnCLN7gzsyzgu6qSAmgSvP9oXAhP973uQ==}
+    engines: {node: '>=18'}
+    cpu: [arm64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/linux-arm@0.16.17:
     resolution: {integrity: sha512-iihzrWbD4gIT7j3caMzKb/RsFFHCwqqbrbH9SqUSRrdXkXaygSZCZg1FybsZz57Ju7N/SHEgPyaR0LZ8Zbe9gQ==}
     engines: {node: '>=12'}
@@ -6568,6 +6649,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/linux-arm@0.25.1:
+    resolution: {integrity: sha512-NdKOhS4u7JhDKw9G3cY6sWqFcnLITn6SqivVArbzIaf3cemShqfLGHYMx8Xlm/lBit3/5d7kXvriTUGa5YViuQ==}
+    engines: {node: '>=18'}
+    cpu: [arm]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/linux-ia32@0.16.17:
     resolution: {integrity: sha512-kiX69+wcPAdgl3Lonh1VI7MBr16nktEvOfViszBSxygRQqSpzv7BffMKRPMFwzeJGPxcio0pdD3kYQGpqQ2SSg==}
     engines: {node: '>=12'}
@@ -6629,6 +6719,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/linux-ia32@0.25.1:
+    resolution: {integrity: sha512-OJykPaF4v8JidKNGz8c/q1lBO44sQNUQtq1KktJXdBLn1hPod5rE/Hko5ugKKZd+D2+o1a9MFGUEIUwO2YfgkQ==}
+    engines: {node: '>=18'}
+    cpu: [ia32]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/linux-loong64@0.15.18:
     resolution: {integrity: sha512-L4jVKS82XVhw2nvzLg/19ClLWg0y27ulRwuP7lcyL6AbUWB5aPglXY3M21mauDQMDfRLs8cQmeT03r/+X3cZYQ==}
     engines: {node: '>=12'}
@@ -6699,6 +6798,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/linux-loong64@0.25.1:
+    resolution: {integrity: sha512-nGfornQj4dzcq5Vp835oM/o21UMlXzn79KobKlcs3Wz9smwiifknLy4xDCLUU0BWp7b/houtdrgUz7nOGnfIYg==}
+    engines: {node: '>=18'}
+    cpu: [loong64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/linux-mips64el@0.16.17:
     resolution: {integrity: sha512-ezbDkp2nDl0PfIUn0CsQ30kxfcLTlcx4Foz2kYv8qdC6ia2oX5Q3E/8m6lq84Dj/6b0FrkgD582fJMIfHhJfSw==}
     engines: {node: '>=12'}
@@ -6760,6 +6868,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/linux-mips64el@0.25.1:
+    resolution: {integrity: sha512-1osBbPEFYwIE5IVB/0g2X6i1qInZa1aIoj1TdL4AaAb55xIIgbg8Doq6a5BzYWgr+tEcDzYH67XVnTmUzL+nXg==}
+    engines: {node: '>=18'}
+    cpu: [mips64el]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/linux-ppc64@0.16.17:
     resolution: {integrity: sha512-dzS678gYD1lJsW73zrFhDApLVdM3cUF2MvAa1D8K8KtcSKdLBPP4zZSLy6LFZ0jYqQdQ29bjAHJDgz0rVbLB3g==}
     engines: {node: '>=12'}
@@ -6821,6 +6938,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/linux-ppc64@0.25.1:
+    resolution: {integrity: sha512-/6VBJOwUf3TdTvJZ82qF3tbLuWsscd7/1w+D9LH0W/SqUgM5/JJD0lrJ1fVIfZsqB6RFmLCe0Xz3fmZc3WtyVg==}
+    engines: {node: '>=18'}
+    cpu: [ppc64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/linux-riscv64@0.16.17:
     resolution: {integrity: sha512-ylNlVsxuFjZK8DQtNUwiMskh6nT0vI7kYl/4fZgV1llP5d6+HIeL/vmmm3jpuoo8+NuXjQVZxmKuhDApK0/cKw==}
     engines: {node: '>=12'}
@@ -6882,6 +7008,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/linux-riscv64@0.25.1:
+    resolution: {integrity: sha512-nSut/Mx5gnilhcq2yIMLMe3Wl4FK5wx/o0QuuCLMtmJn+WeWYoEGDN1ipcN72g1WHsnIbxGXd4i/MF0gTcuAjQ==}
+    engines: {node: '>=18'}
+    cpu: [riscv64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/linux-s390x@0.16.17:
     resolution: {integrity: sha512-gzy7nUTO4UA4oZ2wAMXPNBGTzZFP7mss3aKR2hH+/4UUkCOyqmjXiKpzGrY2TlEUhbbejzXVKKGazYcQTZWA/w==}
     engines: {node: '>=12'}
@@ -6943,6 +7078,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/linux-s390x@0.25.1:
+    resolution: {integrity: sha512-cEECeLlJNfT8kZHqLarDBQso9a27o2Zd2AQ8USAEoGtejOrCYHNtKP8XQhMDJMtthdF4GBmjR2au3x1udADQQQ==}
+    engines: {node: '>=18'}
+    cpu: [s390x]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/linux-x64@0.16.17:
     resolution: {integrity: sha512-mdPjPxfnmoqhgpiEArqi4egmBAMYvaObgn4poorpUaqmvzzbvqbowRllQ+ZgzGVMGKaPkqUmPDOOFQRUFDmeUw==}
     engines: {node: '>=12'}
@@ -7004,6 +7148,24 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/linux-x64@0.25.1:
+    resolution: {integrity: sha512-xbfUhu/gnvSEg+EGovRc+kjBAkrvtk38RlerAzQxvMzlB4fXpCFCeUAYzJvrnhFtdeyVCDANSjJvOvGYoeKzFA==}
+    engines: {node: '>=18'}
+    cpu: [x64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@esbuild/netbsd-arm64@0.25.1:
+    resolution: {integrity: sha512-O96poM2XGhLtpTh+s4+nP7YCCAfb4tJNRVZHfIE7dgmax+yMP2WgMd2OecBuaATHKTHsLWHQeuaxMRnCsH8+5g==}
+    engines: {node: '>=18'}
+    cpu: [arm64]
+    os: [netbsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/netbsd-x64@0.16.17:
     resolution: {integrity: sha512-/PzmzD/zyAeTUsduZa32bn0ORug+Jd1EGGAUJvqfeixoEISYpGnAezN6lnJoskauoai0Jrs+XSyvDhppCPoKOA==}
     engines: {node: '>=12'}
@@ -7065,6 +7227,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/netbsd-x64@0.25.1:
+    resolution: {integrity: sha512-X53z6uXip6KFXBQ+Krbx25XHV/NCbzryM6ehOAeAil7X7oa4XIq+394PWGnwaSQ2WRA0KI6PUO6hTO5zeF5ijA==}
+    engines: {node: '>=18'}
+    cpu: [x64]
+    os: [netbsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/openbsd-arm64@0.23.0:
     resolution: {integrity: sha512-suXjq53gERueVWu0OKxzWqk7NxiUWSUlrxoZK7usiF50C6ipColGR5qie2496iKGYNLhDZkPxBI3erbnYkU0rQ==}
     engines: {node: '>=18'}
@@ -7073,6 +7244,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/openbsd-arm64@0.25.1:
+    resolution: {integrity: sha512-Na9T3szbXezdzM/Kfs3GcRQNjHzM6GzFBeU1/6IV/npKP5ORtp9zbQjvkDJ47s6BCgaAZnnnu/cY1x342+MvZg==}
+    engines: {node: '>=18'}
+    cpu: [arm64]
+    os: [openbsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/openbsd-x64@0.16.17:
     resolution: {integrity: sha512-2yaWJhvxGEz2RiftSk0UObqJa/b+rIAjnODJgv2GbGGpRwAfpgzyrg1WLK8rqA24mfZa9GvpjLcBBg8JHkoodg==}
     engines: {node: '>=12'}
@@ -7134,6 +7314,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/openbsd-x64@0.25.1:
+    resolution: {integrity: sha512-T3H78X2h1tszfRSf+txbt5aOp/e7TAz3ptVKu9Oyir3IAOFPGV6O9c2naym5TOriy1l0nNf6a4X5UXRZSGX/dw==}
+    engines: {node: '>=18'}
+    cpu: [x64]
+    os: [openbsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/sunos-x64@0.16.17:
     resolution: {integrity: sha512-xtVUiev38tN0R3g8VhRfN7Zl42YCJvyBhRKw1RJjwE1d2emWTVToPLNEQj/5Qxc6lVFATDiy6LjVHYhIPrLxzw==}
     engines: {node: '>=12'}
@@ -7195,6 +7384,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/sunos-x64@0.25.1:
+    resolution: {integrity: sha512-2H3RUvcmULO7dIE5EWJH8eubZAI4xw54H1ilJnRNZdeo8dTADEZ21w6J22XBkXqGJbe0+wnNJtw3UXRoLJnFEg==}
+    engines: {node: '>=18'}
+    cpu: [x64]
+    os: [sunos]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/win32-arm64@0.16.17:
     resolution: {integrity: sha512-ga8+JqBDHY4b6fQAmOgtJJue36scANy4l/rL97W+0wYmijhxKetzZdKOJI7olaBaMhWt8Pac2McJdZLxXWUEQw==}
     engines: {node: '>=12'}
@@ -7256,6 +7454,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/win32-arm64@0.25.1:
+    resolution: {integrity: sha512-GE7XvrdOzrb+yVKB9KsRMq+7a2U/K5Cf/8grVFRAGJmfADr/e/ODQ134RK2/eeHqYV5eQRFxb1hY7Nr15fv1NQ==}
+    engines: {node: '>=18'}
+    cpu: [arm64]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/win32-ia32@0.16.17:
     resolution: {integrity: sha512-WnsKaf46uSSF/sZhwnqE4L/F89AYNMiD4YtEcYekBt9Q7nj0DiId2XH2Ng2PHM54qi5oPrQ8luuzGszqi/veig==}
     engines: {node: '>=12'}
@@ -7317,6 +7524,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/win32-ia32@0.25.1:
+    resolution: {integrity: sha512-uOxSJCIcavSiT6UnBhBzE8wy3n0hOkJsBOzy7HDAuTDE++1DJMRRVCPGisULScHL+a/ZwdXPpXD3IyFKjA7K8A==}
+    engines: {node: '>=18'}
+    cpu: [ia32]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@esbuild/win32-x64@0.16.17:
     resolution: {integrity: sha512-y+EHuSchhL7FjHgvQL/0fnnFmO4T1bhvWANX6gcnqTjtnKWbTvUMCpGnv2+t+31d7RzyEAYAd4u2fnIhHL6N/Q==}
     engines: {node: '>=12'}
@@ -7378,6 +7594,15 @@ packages:
     requiresBuild: true
     optional: true
 
+  /@esbuild/win32-x64@0.25.1:
+    resolution: {integrity: sha512-Y1EQdcfwMSeQN/ujR5VayLOJ1BHaK+ssyk0AEzPjC+t1lITgsnccPqFjb6V+LsTp/9Iov4ysfjxLaGJ9RPtkVg==}
+    engines: {node: '>=18'}
+    cpu: [x64]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@eslint-community/eslint-utils@4.4.0(eslint@8.31.0):
     resolution: {integrity: sha512-1/sA4dwrzBAyeUoQ6oxahHKmrZvsnLCg4RfxW3ZFGGmQkSNQPFNLV9CUEFQP1x9EYXHTo5p6xdhZM1Ne9p/AfA==}
     engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0}
@@ -15280,6 +15505,14 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-android-arm-eabi@4.36.0:
+    resolution: {integrity: sha512-jgrXjjcEwN6XpZXL0HUeOVGfjXhPyxAbbhD0BlXUB+abTOpbPiN5Wb3kOT7yb+uEtATNYF5x5gIfwutmuBA26w==}
+    cpu: [arm]
+    os: [android]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-android-arm64@4.13.2:
     resolution: {integrity: sha512-GdxxXbAuM7Y/YQM9/TwwP+L0omeE/lJAR1J+olu36c3LqqZEBdsIWeQ91KBe6nxwOnb06Xh7JS2U5ooWU5/LgQ==}
     cpu: [arm64]
@@ -15288,6 +15521,14 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-android-arm64@4.36.0:
+    resolution: {integrity: sha512-NyfuLvdPdNUfUNeYKUwPwKsE5SXa2J6bCt2LdB/N+AxShnkpiczi3tcLJrm5mA+eqpy0HmaIY9F6XCa32N5yzg==}
+    cpu: [arm64]
+    os: [android]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-darwin-arm64@4.13.2:
     resolution: {integrity: sha512-mCMlpzlBgOTdaFs83I4XRr8wNPveJiJX1RLfv4hggyIVhfB5mJfN4P8Z6yKh+oE4Luz+qq1P3kVdWrCKcMYrrA==}
     cpu: [arm64]
@@ -15296,12 +15537,11 @@ packages:
     dev: true
     optional: true
 
-  /@rollup/rollup-darwin-arm64@4.21.3:
-    resolution: {integrity: sha512-P0UxIOrKNBFTQaXTxOH4RxuEBVCgEA5UTNV6Yz7z9QHnUJ7eLX9reOd/NYMO3+XZO2cco19mXTxDMXxit4R/eQ==}
+  /@rollup/rollup-darwin-arm64@4.36.0:
+    resolution: {integrity: sha512-JQ1Jk5G4bGrD4pWJQzWsD8I1n1mgPXq33+/vP4sk8j/z/C2siRuxZtaUA7yMTf71TCZTZl/4e1bfzwUmFb3+rw==}
     cpu: [arm64]
     os: [darwin]
     requiresBuild: true
-    dev: false
     optional: true
 
   /@rollup/rollup-darwin-x64@4.13.2:
@@ -15312,6 +15552,30 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-darwin-x64@4.36.0:
+    resolution: {integrity: sha512-6c6wMZa1lrtiRsbDziCmjE53YbTkxMYhhnWnSW8R/yqsM7a6mSJ3uAVT0t8Y/DGt7gxUWYuFM4bwWk9XCJrFKA==}
+    cpu: [x64]
+    os: [darwin]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@rollup/rollup-freebsd-arm64@4.36.0:
+    resolution: {integrity: sha512-KXVsijKeJXOl8QzXTsA+sHVDsFOmMCdBRgFmBb+mfEb/7geR7+C8ypAml4fquUt14ZyVXaw2o1FWhqAfOvA4sg==}
+    cpu: [arm64]
+    os: [freebsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@rollup/rollup-freebsd-x64@4.36.0:
+    resolution: {integrity: sha512-dVeWq1ebbvByI+ndz4IJcD4a09RJgRYmLccwlQ8bPd4olz3Y213uf1iwvc7ZaxNn2ab7bjc08PrtBgMu6nb4pQ==}
+    cpu: [x64]
+    os: [freebsd]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-linux-arm-gnueabihf@4.13.2:
     resolution: {integrity: sha512-GYbLs5ErswU/Xs7aGXqzc3RrdEjKdmoCrgzhJWyFL0r5fL3qd1NPcDKDowDnmcoSiGJeU68/Vy+OMUluRxPiLQ==}
     cpu: [arm]
@@ -15320,6 +15584,22 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-linux-arm-gnueabihf@4.36.0:
+    resolution: {integrity: sha512-bvXVU42mOVcF4le6XSjscdXjqx8okv4n5vmwgzcmtvFdifQ5U4dXFYaCB87namDRKlUL9ybVtLQ9ztnawaSzvg==}
+    cpu: [arm]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@rollup/rollup-linux-arm-musleabihf@4.36.0:
+    resolution: {integrity: sha512-JFIQrDJYrxOnyDQGYkqnNBtjDwTgbasdbUiQvcU8JmGDfValfH1lNpng+4FWlhaVIR4KPkeddYjsVVbmJYvDcg==}
+    cpu: [arm]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-linux-arm64-gnu@4.13.2:
     resolution: {integrity: sha512-L1+D8/wqGnKQIlh4Zre9i4R4b4noxzH5DDciyahX4oOz62CphY7WDWqJoQ66zNR4oScLNOqQJfNSIAe/6TPUmQ==}
     cpu: [arm64]
@@ -15328,6 +15608,14 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-linux-arm64-gnu@4.36.0:
+    resolution: {integrity: sha512-KqjYVh3oM1bj//5X7k79PSCZ6CvaVzb7Qs7VMWS+SlWB5M8p3FqufLP9VNp4CazJ0CsPDLwVD9r3vX7Ci4J56A==}
+    cpu: [arm64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-linux-arm64-musl@4.13.2:
     resolution: {integrity: sha512-tK5eoKFkXdz6vjfkSTCupUzCo40xueTOiOO6PeEIadlNBkadH1wNOH8ILCPIl8by/Gmb5AGAeQOFeLev7iZDOA==}
     cpu: [arm64]
@@ -15336,6 +15624,22 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-linux-arm64-musl@4.36.0:
+    resolution: {integrity: sha512-QiGnhScND+mAAtfHqeT+cB1S9yFnNQ/EwCg5yE3MzoaZZnIV0RV9O5alJAoJKX/sBONVKeZdMfO8QSaWEygMhw==}
+    cpu: [arm64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
+  /@rollup/rollup-linux-loongarch64-gnu@4.36.0:
+    resolution: {integrity: sha512-1ZPyEDWF8phd4FQtTzMh8FQwqzvIjLsl6/84gzUxnMNFBtExBtpL51H67mV9xipuxl1AEAerRBgBwFNpkw8+Lg==}
+    cpu: [loong64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-linux-powerpc64le-gnu@4.13.2:
     resolution: {integrity: sha512-zvXvAUGGEYi6tYhcDmb9wlOckVbuD+7z3mzInCSTACJ4DQrdSLPNUeDIcAQW39M3q6PDquqLWu7pnO39uSMRzQ==}
     cpu: [ppc64le]
@@ -15344,6 +15648,14 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-linux-powerpc64le-gnu@4.36.0:
+    resolution: {integrity: sha512-VMPMEIUpPFKpPI9GZMhJrtu8rxnp6mJR3ZzQPykq4xc2GmdHj3Q4cA+7avMyegXy4n1v+Qynr9fR88BmyO74tg==}
+    cpu: [ppc64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-linux-riscv64-gnu@4.13.2:
     resolution: {integrity: sha512-C3GSKvMtdudHCN5HdmAMSRYR2kkhgdOfye4w0xzyii7lebVr4riCgmM6lRiSCnJn2w1Xz7ZZzHKuLrjx5620kw==}
     cpu: [riscv64]
@@ -15352,6 +15664,14 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-linux-riscv64-gnu@4.36.0:
+    resolution: {integrity: sha512-ttE6ayb/kHwNRJGYLpuAvB7SMtOeQnVXEIpMtAvx3kepFQeowVED0n1K9nAdraHUPJ5hydEMxBpIR7o4nrm8uA==}
+    cpu: [riscv64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-linux-s390x-gnu@4.13.2:
     resolution: {integrity: sha512-l4U0KDFwzD36j7HdfJ5/TveEQ1fUTjFFQP5qIt9gBqBgu1G8/kCaq5Ok05kd5TG9F8Lltf3MoYsUMw3rNlJ0Yg==}
     cpu: [s390x]
@@ -15360,6 +15680,14 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-linux-s390x-gnu@4.36.0:
+    resolution: {integrity: sha512-4a5gf2jpS0AIe7uBjxDeUMNcFmaRTbNv7NxI5xOCs4lhzsVyGR/0qBXduPnoWf6dGC365saTiwag8hP1imTgag==}
+    cpu: [s390x]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-linux-x64-gnu@4.13.2:
     resolution: {integrity: sha512-xXMLUAMzrtsvh3cZ448vbXqlUa7ZL8z0MwHp63K2IIID2+DeP5iWIT6g1SN7hg1VxPzqx0xZdiDM9l4n9LRU1A==}
     cpu: [x64]
@@ -15368,6 +15696,14 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-linux-x64-gnu@4.36.0:
+    resolution: {integrity: sha512-5KtoW8UWmwFKQ96aQL3LlRXX16IMwyzMq/jSSVIIyAANiE1doaQsx/KRyhAvpHlPjPiSU/AYX/8m+lQ9VToxFQ==}
+    cpu: [x64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-linux-x64-musl@4.13.2:
     resolution: {integrity: sha512-M/JYAWickafUijWPai4ehrjzVPKRCyDb1SLuO+ZyPfoXgeCEAlgPkNXewFZx0zcnoIe3ay4UjXIMdXQXOZXWqA==}
     cpu: [x64]
@@ -15376,6 +15712,14 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-linux-x64-musl@4.36.0:
+    resolution: {integrity: sha512-sycrYZPrv2ag4OCvaN5js+f01eoZ2U+RmT5as8vhxiFz+kxwlHrsxOwKPSA8WyS+Wc6Epid9QeI/IkQ9NkgYyQ==}
+    cpu: [x64]
+    os: [linux]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-win32-arm64-msvc@4.13.2:
     resolution: {integrity: sha512-2YWwoVg9KRkIKaXSh0mz3NmfurpmYoBBTAXA9qt7VXk0Xy12PoOP40EFuau+ajgALbbhi4uTj3tSG3tVseCjuA==}
     cpu: [arm64]
@@ -15384,6 +15728,14 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-win32-arm64-msvc@4.36.0:
+    resolution: {integrity: sha512-qbqt4N7tokFwwSVlWDsjfoHgviS3n/vZ8LK0h1uLG9TYIRuUTJC88E1xb3LM2iqZ/WTqNQjYrtmtGmrmmawB6A==}
+    cpu: [arm64]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-win32-ia32-msvc@4.13.2:
     resolution: {integrity: sha512-2FSsE9aQ6OWD20E498NYKEQLneShWes0NGMPQwxWOdws35qQXH+FplabOSP5zEe1pVjurSDOGEVCE2agFwSEsw==}
     cpu: [ia32]
@@ -15392,6 +15744,14 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-win32-ia32-msvc@4.36.0:
+    resolution: {integrity: sha512-t+RY0JuRamIocMuQcfwYSOkmdX9dtkr1PbhKW42AMvaDQa+jOdpUYysroTF/nuPpAaQMWp7ye+ndlmmthieJrQ==}
+    cpu: [ia32]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rollup/rollup-win32-x64-msvc@4.13.2:
     resolution: {integrity: sha512-7h7J2nokcdPePdKykd8wtc8QqqkqxIrUz7MHj6aNr8waBRU//NLDVnNjQnqQO6fqtjrtCdftpbTuOKAyrAQETQ==}
     cpu: [x64]
@@ -15400,6 +15760,14 @@ packages:
     dev: true
     optional: true
 
+  /@rollup/rollup-win32-x64-msvc@4.36.0:
+    resolution: {integrity: sha512-aRXd7tRZkWLqGbChgcMMDEHjOKudo1kChb1Jt1IfR8cY/KIpgNviLeJy5FUb9IpSuQj8dU2fAYNMPW/hLKOSTw==}
+    cpu: [x64]
+    os: [win32]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /@rushstack/eslint-patch@1.2.0:
     resolution: {integrity: sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==}
     dev: true
@@ -17025,6 +17393,9 @@ packages:
   /@types/estree@1.0.5:
     resolution: {integrity: sha512-/kYRxGDLWzHOB7q+wtSUQlFrtcdUccpfy+X+9iMBpHK8QLLhx2wIPYuS5DYtR9Wa/YlZAbIovy7qVdB1Aq6Lyw==}
 
+  /@types/estree@1.0.6:
+    resolution: {integrity: sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==}
+
   /@types/eventsource@1.1.15:
     resolution: {integrity: sha512-XQmGcbnxUNa06HR3VBVkc9+A2Vpi9ZyLJcdS5dwaQQ/4ZMWFO+5c90FnMUpbtMZwB/FChoYHwuVg8TvkECacTA==}
     dev: true
@@ -19387,6 +19758,16 @@ packages:
       run-applescript: 7.0.0
     dev: false
 
+  /bundle-require@5.1.0(esbuild@0.25.1):
+    resolution: {integrity: sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    peerDependencies:
+      esbuild: '>=0.18'
+    dependencies:
+      esbuild: 0.25.1
+      load-tsconfig: 0.2.5
+    dev: true
+
   /busboy@1.6.0:
     resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==}
     engines: {node: '>=10.16.0'}
@@ -19725,6 +20106,13 @@ packages:
     optionalDependencies:
       fsevents: 2.3.3
 
+  /chokidar@4.0.3:
+    resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==}
+    engines: {node: '>= 14.16.0'}
+    dependencies:
+      readdirp: 4.1.2
+    dev: true
+
   /chownr@1.1.4:
     resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==}
 
@@ -19919,7 +20307,7 @@ packages:
     resolution: {integrity: sha512-7qJWqItLA8/VPVlKJlFXU+NBlo/qyfs39aJcuMT/2ere32ZqvF5OSxgdM5xOfJJ7O429gg2HM47y8v9P+9wrNw==}
     dependencies:
       '@jridgewell/sourcemap-codec': 1.5.0
-      '@types/estree': 1.0.5
+      '@types/estree': 1.0.6
       acorn: 8.12.1
       estree-walker: 3.0.3
       periscopic: 3.1.0
@@ -20081,6 +20469,10 @@ packages:
     engines: {node: ^14.18.0 || >=16.10.0}
     dev: false
 
+  /consola@3.4.2:
+    resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==}
+    engines: {node: ^14.18.0 || >=16.10.0}
+
   /content-disposition@0.5.4:
     resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==}
     engines: {node: '>= 0.6'}
@@ -21821,6 +22213,39 @@ packages:
       '@esbuild/win32-ia32': 0.23.0
       '@esbuild/win32-x64': 0.23.0
 
+  /esbuild@0.25.1:
+    resolution: {integrity: sha512-BGO5LtrGC7vxnqucAe/rmvKdJllfGaYWdyABvyMoXQlfYMb2bbRuReWR5tEGE//4LcNJj9XrkovTqNYRFZHAMQ==}
+    engines: {node: '>=18'}
+    hasBin: true
+    requiresBuild: true
+    optionalDependencies:
+      '@esbuild/aix-ppc64': 0.25.1
+      '@esbuild/android-arm': 0.25.1
+      '@esbuild/android-arm64': 0.25.1
+      '@esbuild/android-x64': 0.25.1
+      '@esbuild/darwin-arm64': 0.25.1
+      '@esbuild/darwin-x64': 0.25.1
+      '@esbuild/freebsd-arm64': 0.25.1
+      '@esbuild/freebsd-x64': 0.25.1
+      '@esbuild/linux-arm': 0.25.1
+      '@esbuild/linux-arm64': 0.25.1
+      '@esbuild/linux-ia32': 0.25.1
+      '@esbuild/linux-loong64': 0.25.1
+      '@esbuild/linux-mips64el': 0.25.1
+      '@esbuild/linux-ppc64': 0.25.1
+      '@esbuild/linux-riscv64': 0.25.1
+      '@esbuild/linux-s390x': 0.25.1
+      '@esbuild/linux-x64': 0.25.1
+      '@esbuild/netbsd-arm64': 0.25.1
+      '@esbuild/netbsd-x64': 0.25.1
+      '@esbuild/openbsd-arm64': 0.25.1
+      '@esbuild/openbsd-x64': 0.25.1
+      '@esbuild/sunos-x64': 0.25.1
+      '@esbuild/win32-arm64': 0.25.1
+      '@esbuild/win32-ia32': 0.25.1
+      '@esbuild/win32-x64': 0.25.1
+    dev: true
+
   /escalade@3.2.0:
     resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==}
     engines: {node: '>=6'}
@@ -22779,7 +23204,6 @@ packages:
         optional: true
     dependencies:
       picomatch: 4.0.2
-    dev: false
 
   /fflate@0.4.8:
     resolution: {integrity: sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==}
@@ -24528,6 +24952,11 @@ packages:
     resolution: {integrity: sha512-EyUPtOKyTYq+iMOszO42eobQllaIjJnwkZ2U93aJzNyPibCy7CEvT9UQnaCVB51IAd49gbNdCew1c0LcLTCB2g==}
     dev: false
 
+  /joycon@3.1.1:
+    resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==}
+    engines: {node: '>=10'}
+    dev: true
+
   /js-beautify@1.15.1:
     resolution: {integrity: sha512-ESjNzSlt/sWE8sciZH8kBF8BPlwXPwhR6pWKAw8bw4Bwj+iZcnKW6ONWUutJ7eObuBZQpiIb8S7OYspWrKt7rA==}
     engines: {node: '>=14'}
@@ -24902,6 +25331,11 @@ packages:
     resolution: {integrity: sha512-utWOt/GHzuUxnLKxB6dk81RoOeoNeHgbrXiuGk4yyF5qlRz+iIVWu56E2fqGHFrXz0QNUhLB/8nKqvRH66JKGQ==}
     engines: {node: '>=10'}
 
+  /lilconfig@3.1.3:
+    resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==}
+    engines: {node: '>=14'}
+    dev: true
+
   /lines-and-columns@1.2.4:
     resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==}
 
@@ -24915,6 +25349,11 @@ packages:
       strip-bom: 3.0.0
     dev: true
 
+  /load-tsconfig@0.2.5:
+    resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==}
+    engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0}
+    dev: true
+
   /load-yaml-file@0.2.0:
     resolution: {integrity: sha512-OfCBkGEw4nN6JLtgRidPX6QxjBQGQf72q3si2uvqyFEMbycSFFHwAZeXx6cJgFM9wmLrf9zBwCP3Ivqa+LLZPw==}
     engines: {node: '>=6'}
@@ -25034,6 +25473,10 @@ packages:
     resolution: {integrity: sha512-XeqSp49hNGmlkj2EJlfrQFIzQ6lXdNro9sddtQzcJY8QaoC2GO0DT7xaIokHeyM+mIT0mPMlPvkYzg2xCuHdZg==}
     dev: false
 
+  /lodash.sortby@4.7.0:
+    resolution: {integrity: sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==}
+    dev: true
+
   /lodash.startcase@4.4.0:
     resolution: {integrity: sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==}
     dev: false
@@ -27444,7 +27887,6 @@ packages:
   /picomatch@4.0.2:
     resolution: {integrity: sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==}
     engines: {node: '>=12'}
-    dev: false
 
   /pidtree@0.3.1:
     resolution: {integrity: sha512-qQbW94hLHEqCg7nhby4yRC7G2+jYHY4Rguc2bjw7Uug4GIJuu1tvf2uHaZv5Q8zdt+WKJ6qK1FOI6amaWUo5FA==}
@@ -27614,6 +28056,29 @@ packages:
       ts-node: 10.9.1(@swc/core@1.3.26)(@types/node@20.14.14)(typescript@5.5.4)
       yaml: 2.3.1
 
+  /postcss-load-config@6.0.1(postcss@8.4.44)(tsx@4.17.0):
+    resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==}
+    engines: {node: '>= 18'}
+    peerDependencies:
+      jiti: '>=1.21.0'
+      postcss: '>=8.0.9'
+      tsx: ^4.8.1
+      yaml: ^2.4.2
+    peerDependenciesMeta:
+      jiti:
+        optional: true
+      postcss:
+        optional: true
+      tsx:
+        optional: true
+      yaml:
+        optional: true
+    dependencies:
+      lilconfig: 3.1.3
+      postcss: 8.4.44
+      tsx: 4.17.0
+    dev: true
+
   /postcss-loader@8.1.1(postcss@8.4.44)(typescript@5.5.4)(webpack@5.88.2):
     resolution: {integrity: sha512-0IeqyAsG6tYiDRCYKQJLAmgQr47DX6N7sFSWvQxt6AcupX8DIdmykuk/o/tx0Lze3ErGHJEp5OSRxrelC6+NdQ==}
     engines: {node: '>= 18.12.0'}
@@ -29031,6 +29496,11 @@ packages:
     dependencies:
       picomatch: 2.3.1
 
+  /readdirp@4.1.2:
+    resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==}
+    engines: {node: '>= 14.18.0'}
+    dev: true
+
   /recharts-scale@0.4.5:
     resolution: {integrity: sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w==}
     dependencies:
@@ -29449,7 +29919,6 @@ packages:
   /resolve-from@5.0.0:
     resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==}
     engines: {node: '>=8'}
-    dev: false
 
   /resolve-import@2.0.0:
     resolution: {integrity: sha512-jpKjLibLuc8D1XEV2+7zb0aqN7I8d12u89g/v6IsgCzdVlccMQJq4TKkPw5fbhHdxhm7nbVtN+KvOTnjFf+nEA==}
@@ -29611,6 +30080,35 @@ packages:
       fsevents: 2.3.3
     dev: true
 
+  /rollup@4.36.0:
+    resolution: {integrity: sha512-zwATAXNQxUcd40zgtQG0ZafcRK4g004WtEl7kbuhTWPvf07PsfohXl39jVUvPF7jvNAIkKPQ2XrsDlWuxBd++Q==}
+    engines: {node: '>=18.0.0', npm: '>=8.0.0'}
+    hasBin: true
+    dependencies:
+      '@types/estree': 1.0.6
+    optionalDependencies:
+      '@rollup/rollup-android-arm-eabi': 4.36.0
+      '@rollup/rollup-android-arm64': 4.36.0
+      '@rollup/rollup-darwin-arm64': 4.36.0
+      '@rollup/rollup-darwin-x64': 4.36.0
+      '@rollup/rollup-freebsd-arm64': 4.36.0
+      '@rollup/rollup-freebsd-x64': 4.36.0
+      '@rollup/rollup-linux-arm-gnueabihf': 4.36.0
+      '@rollup/rollup-linux-arm-musleabihf': 4.36.0
+      '@rollup/rollup-linux-arm64-gnu': 4.36.0
+      '@rollup/rollup-linux-arm64-musl': 4.36.0
+      '@rollup/rollup-linux-loongarch64-gnu': 4.36.0
+      '@rollup/rollup-linux-powerpc64le-gnu': 4.36.0
+      '@rollup/rollup-linux-riscv64-gnu': 4.36.0
+      '@rollup/rollup-linux-s390x-gnu': 4.36.0
+      '@rollup/rollup-linux-x64-gnu': 4.36.0
+      '@rollup/rollup-linux-x64-musl': 4.36.0
+      '@rollup/rollup-win32-arm64-msvc': 4.36.0
+      '@rollup/rollup-win32-ia32-msvc': 4.36.0
+      '@rollup/rollup-win32-x64-msvc': 4.36.0
+      fsevents: 2.3.3
+    dev: true
+
   /router@2.1.0:
     resolution: {integrity: sha512-/m/NSLxeYEgWNtyC+WtNHCF7jbGxOibVWKnn+1Psff4dJGOfoXP+MuC/f2CwSmyiHdOIzYnYFp4W6GxWfekaLA==}
     engines: {node: '>= 18'}
@@ -30249,6 +30747,13 @@ packages:
     resolution: {integrity: sha512-l3BikUxvPOcn5E74dZiq5BGsTb5yEwhaTSzccU6t4sDOH8NWJCstKO5QT2CvtFoK6F0saL7p9xHAqHOlCPJygA==}
     engines: {node: '>= 8'}
 
+  /source-map@0.8.0-beta.0:
+    resolution: {integrity: sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==}
+    engines: {node: '>= 8'}
+    dependencies:
+      whatwg-url: 7.1.0
+    dev: true
+
   /sourcemap-codec@1.4.8:
     resolution: {integrity: sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==}
     deprecated: Please use @jridgewell/sourcemap-codec instead
@@ -30696,6 +31201,20 @@ packages:
       pirates: 4.0.5
       ts-interface-checker: 0.1.13
 
+  /sucrase@3.35.0:
+    resolution: {integrity: sha512-8EbVDiu9iN/nESwxeSxDKe0dunta1GOlHufmSSXxMD2z2/tMZpDMpvXQGsc+ajGo8y2uYUmixaSRUc/QPoQ0GA==}
+    engines: {node: '>=16 || 14 >=14.17'}
+    hasBin: true
+    dependencies:
+      '@jridgewell/gen-mapping': 0.3.5
+      commander: 4.1.1
+      glob: 10.3.10
+      lines-and-columns: 1.2.4
+      mz: 2.7.0
+      pirates: 4.0.5
+      ts-interface-checker: 0.1.13
+    dev: true
+
   /superagent@9.0.2:
     resolution: {integrity: sha512-xuW7dzkUpcJq7QnhOsnNUgtYp3xRwpt2F7abdRYIpCsAt0hhUqia0EdxyXZQQpNmGtsCzYHryaKSV3q3GJnq7w==}
     engines: {node: '>=14.18.0'}
@@ -30787,7 +31306,7 @@ packages:
       '@ampproject/remapping': 2.3.0
       '@jridgewell/sourcemap-codec': 1.5.0
       '@jridgewell/trace-mapping': 0.3.25
-      '@types/estree': 1.0.5
+      '@types/estree': 1.0.6
       acorn: 8.12.1
       aria-query: 5.3.0
       axobject-query: 4.1.0
@@ -31275,7 +31794,6 @@ packages:
 
   /tinyexec@0.3.2:
     resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==}
-    dev: false
 
   /tinyglobby@0.2.10:
     resolution: {integrity: sha512-Zc+8eJlFMvgatPZTl6A9L/yht8QqdmUNtURHaKZLmKBE12hNPSrqNkUp2cs3M/UKmNVVAMFQYSjYIVHDjW5zew==}
@@ -31285,6 +31803,14 @@ packages:
       picomatch: 4.0.2
     dev: false
 
+  /tinyglobby@0.2.12:
+    resolution: {integrity: sha512-qkf4trmKSIiMTs/E63cxH+ojC2unam7rJ0WrauAzpT3ECNTxGRMlaXxVbfxMUC/w0LaYk6jQ4y/nGR9uBO3tww==}
+    engines: {node: '>=12.0.0'}
+    dependencies:
+      fdir: 6.4.3(picomatch@4.0.2)
+      picomatch: 4.0.2
+    dev: true
+
   /tinyglobby@0.2.2:
     resolution: {integrity: sha512-mZ2sDMaySvi1PkTp4lTo1In2zjU+cY8OvZsfwrDrx3YGRbXPX1/cbPwCR9zkm3O/Fz9Jo0F1HNgIQ1b8BepqyQ==}
     engines: {node: '>=12.0.0'}
@@ -31416,6 +31942,17 @@ packages:
   /tr46@0.0.3:
     resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}
 
+  /tr46@1.0.1:
+    resolution: {integrity: sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==}
+    dependencies:
+      punycode: 2.2.0
+    dev: true
+
+  /tree-kill@1.2.2:
+    resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==}
+    hasBin: true
+    dev: true
+
   /trim-lines@3.0.1:
     resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==}
     dev: true
@@ -31634,6 +32171,50 @@ packages:
   /tslib@2.6.2:
     resolution: {integrity: sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q==}
 
+  /tsup@8.4.0(postcss@8.4.44)(tsx@4.17.0)(typescript@5.5.4):
+    resolution: {integrity: sha512-b+eZbPCjz10fRryaAA7C8xlIHnf8VnsaRqydheLIqwG/Mcpfk8Z5zp3HayX7GaTygkigHl5cBUs+IhcySiIexQ==}
+    engines: {node: '>=18'}
+    hasBin: true
+    peerDependencies:
+      '@microsoft/api-extractor': ^7.36.0
+      '@swc/core': ^1
+      postcss: ^8.4.12
+      typescript: '>=4.5.0'
+    peerDependenciesMeta:
+      '@microsoft/api-extractor':
+        optional: true
+      '@swc/core':
+        optional: true
+      postcss:
+        optional: true
+      typescript:
+        optional: true
+    dependencies:
+      bundle-require: 5.1.0(esbuild@0.25.1)
+      cac: 6.7.14
+      chokidar: 4.0.3
+      consola: 3.4.2
+      debug: 4.4.0
+      esbuild: 0.25.1
+      joycon: 3.1.1
+      picocolors: 1.1.1
+      postcss: 8.4.44
+      postcss-load-config: 6.0.1(postcss@8.4.44)(tsx@4.17.0)
+      resolve-from: 5.0.0
+      rollup: 4.36.0
+      source-map: 0.8.0-beta.0
+      sucrase: 3.35.0
+      tinyexec: 0.3.2
+      tinyglobby: 0.2.12
+      tree-kill: 1.2.2
+      typescript: 5.5.4
+    transitivePeerDependencies:
+      - jiti
+      - supports-color
+      - tsx
+      - yaml
+    dev: true
+
   /tsutils@3.21.0(typescript@5.5.4):
     resolution: {integrity: sha512-mHKK3iUXL+3UF6xL5k0PEhKRUBKPBCv/+RkEOpjRWxxx27KKRBmmA60A9pgOUvMi8GKhRMPEmjBRPzs2W7O1OA==}
     engines: {node: '>= 6'}
@@ -32051,7 +32632,7 @@ packages:
   /unenv-nightly@1.10.0-1717606461.a117952:
     resolution: {integrity: sha512-u3TfBX02WzbHTpaEfWEKwDijDSFAHcgXkayUZ+MVDrjhLFvgAJzFGTSTmwlEhwWi2exyRQey23ah9wELMM6etg==}
     dependencies:
-      consola: 3.2.3
+      consola: 3.4.2
       defu: 6.1.4
       mime: 3.0.0
       node-fetch-native: 1.6.4
@@ -33119,6 +33700,10 @@ packages:
   /webidl-conversions@3.0.1:
     resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==}
 
+  /webidl-conversions@4.0.2:
+    resolution: {integrity: sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==}
+    dev: true
+
   /webpack-bundle-analyzer@4.10.1:
     resolution: {integrity: sha512-s3P7pgexgT/HTUSYgxJyn28A+99mmLq4HsJepMPzu0R8ImJc52QNqaFYW1Z2z2uIb1/J3eYgaAWVpaC+v/1aAQ==}
     engines: {node: '>= 10.13.0'}
@@ -33236,6 +33821,14 @@ packages:
       tr46: 0.0.3
       webidl-conversions: 3.0.1
 
+  /whatwg-url@7.1.0:
+    resolution: {integrity: sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==}
+    dependencies:
+      lodash.sortby: 4.7.0
+      tr46: 1.0.1
+      webidl-conversions: 4.0.2
+    dev: true
+
   /which-boxed-primitive@1.0.2:
     resolution: {integrity: sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==}
     dependencies:

From c05688cdf3334c13cc5a859676594fc69bcbd7b9 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Fri, 21 Mar 2025 11:23:32 +0000
Subject: [PATCH 20/26] fix esm/cjs interop

---
 packages/redis-worker/tsup.config.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/redis-worker/tsup.config.ts b/packages/redis-worker/tsup.config.ts
index fdcfbe41fd..36d3ddf807 100644
--- a/packages/redis-worker/tsup.config.ts
+++ b/packages/redis-worker/tsup.config.ts
@@ -21,7 +21,7 @@ export default defineConfig({
     if (format !== "esm") return;
 
     return {
-      js: `import { createRequire } from 'module'; const require = createRequire(import.meta.url);`,
+      js: `import { createRequire } from 'module'; const require = createRequire(import.meta.url || process.cwd() + '/index.js');`,
     };
   },
 });

From 0198077f71ad118d659ed7bcc9bc289942cfe15d Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Fri, 21 Mar 2025 11:58:49 +0000
Subject: [PATCH 21/26] remove proxy from changeset ignore and add supervisor

---
 .changeset/config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.changeset/config.json b/.changeset/config.json
index fc8fb16a60..5ec3facf38 100644
--- a/.changeset/config.json
+++ b/.changeset/config.json
@@ -12,7 +12,7 @@
   "access": "public",
   "baseBranch": "main",
   "updateInternalDependencies": "patch",
-  "ignore": ["webapp", "proxy", "coordinator", "docker-provider", "kubernetes-provider"],
+  "ignore": ["webapp", "supervisor", "coordinator", "docker-provider", "kubernetes-provider"],
   "___experimentalUnsafeOptions_WILL_CHANGE_IN_PATCH": {
     "onlyUpdatePeerDependentsWhenOutOfRange": true
   }

From aa3e57de400914f191ca8fd4f2b622a31d3444f1 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Fri, 21 Mar 2025 12:31:23 +0000
Subject: [PATCH 22/26] add pause to prerelease script for any manual edits

---
 scripts/publish-prerelease.sh | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/scripts/publish-prerelease.sh b/scripts/publish-prerelease.sh
index 26c7d191a1..763a80fbe0 100755
--- a/scripts/publish-prerelease.sh
+++ b/scripts/publish-prerelease.sh
@@ -35,6 +35,17 @@ else
     echo "Git status is clean. Proceeding with the script.";
 fi
 
+# From here on, if the user aborts the script, we will clean up the git stage
+git_reset() {
+    git reset --hard HEAD
+}
+abort() {
+    echo "Aborted. Cleaning up..."
+    git_reset
+    exit 1
+}
+trap abort INT
+
 # Run your commands
 # Run changeset version command and capture its output
 echo "Running: pnpm exec changeset version --snapshot $version"
@@ -48,6 +59,8 @@ else
     exit 1
 fi
 
+read -e -p "Pausing for manual changes, press Enter when ready to continue..."
+
 echo "Running: pnpm run clean --filter \"@trigger.dev/*\" --filter \"trigger.dev\""
 pnpm run clean --filter "@trigger.dev/*" --filter "trigger.dev"
 
@@ -59,11 +72,9 @@ read -p "Do you wish to continue? (y/N): " prompt
 if [[ $prompt =~ [yY](es)* ]]; then
     pnpm exec changeset publish --no-git-tag --snapshot --tag $version
 else
-    echo "Publish command aborted by the user."
-    git reset --hard HEAD
-    exit 1;
+    abort
 fi
 
 # If there were no errors, clear the git stage
 echo "Commands ran successfully. Clearing the git stage."
-git reset --hard HEAD
+git_reset

From ff78b31c723bd3d9d729bb25fe164e47e697cc10 Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Fri, 21 Mar 2025 13:27:27 +0000
Subject: [PATCH 23/26] unregister the correct handler and add early detection

---
 packages/core/src/v3/serverOnly/shutdownManager.ts | 4 ++++
 packages/redis-worker/src/worker.ts                | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/core/src/v3/serverOnly/shutdownManager.ts b/packages/core/src/v3/serverOnly/shutdownManager.ts
index 9adb662a3c..b6e940ed85 100644
--- a/packages/core/src/v3/serverOnly/shutdownManager.ts
+++ b/packages/core/src/v3/serverOnly/shutdownManager.ts
@@ -33,6 +33,10 @@ class ShutdownManager {
   }
 
   unregister(name: string) {
+    if (!this.handlers.has(name)) {
+      throw new Error(`Shutdown handler "${name}" not registered`);
+    }
+
     this.handlers.delete(name);
   }
 
diff --git a/packages/redis-worker/src/worker.ts b/packages/redis-worker/src/worker.ts
index 7061891993..f78d493fae 100644
--- a/packages/redis-worker/src/worker.ts
+++ b/packages/redis-worker/src/worker.ts
@@ -541,7 +541,7 @@ class Worker<TCatalog extends WorkerCatalog> {
   }
 
   public async stop() {
-    shutdownManager.unregister("redis-worker");
+    shutdownManager.unregister(`redis-worker:${this.options.name}`);
     await this.shutdown();
   }
 }

From 821b918acb11d1a4824d25034435def757de8ede Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Fri, 21 Mar 2025 13:27:45 +0000
Subject: [PATCH 24/26] small change to http handler return

---
 packages/core/src/v3/serverOnly/httpServer.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/core/src/v3/serverOnly/httpServer.ts b/packages/core/src/v3/serverOnly/httpServer.ts
index 1e68ec4ca7..722fa777b3 100644
--- a/packages/core/src/v3/serverOnly/httpServer.ts
+++ b/packages/core/src/v3/serverOnly/httpServer.ts
@@ -207,8 +207,9 @@ export class HttpServer {
         return reply.empty(500);
       } finally {
         this.collectMetrics(req, res, startTime);
-        return;
       }
+
+      return reply.empty(501);
     });
 
     this.server.on("clientError", (_, socket) => {

From 59afe713c1354bfc8d33b1f5e0e6ad0271c6ce7e Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Fri, 21 Mar 2025 13:38:33 +0000
Subject: [PATCH 25/26] fix worker tests

---
 packages/redis-worker/src/worker.test.ts | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/packages/redis-worker/src/worker.test.ts b/packages/redis-worker/src/worker.test.ts
index 1768f39107..e621bd2d3a 100644
--- a/packages/redis-worker/src/worker.test.ts
+++ b/packages/redis-worker/src/worker.test.ts
@@ -1,12 +1,17 @@
 import { redisTest } from "@internal/testcontainers";
 import { Logger } from "@trigger.dev/core/logger";
 import { describe } from "node:test";
-import { expect } from "vitest";
+import { afterEach, expect } from "vitest";
 import { z } from "zod";
 import { Worker } from "./worker.js";
 import { createRedisClient } from "@internal/redis";
+import { shutdownManager } from "@trigger.dev/core/v3/serverOnly";
 
 describe("Worker", () => {
+  afterEach(() => {
+    (shutdownManager as unknown as { _reset: () => void })._reset();
+  });
+
   redisTest("Process items that don't throw", { timeout: 30_000 }, async ({ redisContainer }) => {
     const processedItems: number[] = [];
     const worker = new Worker({

From 921edcbf22a654677907d63bfe623a59a39d81bc Mon Sep 17 00:00:00 2001
From: nicktrn <55853254+nicktrn@users.noreply.github.com>
Date: Fri, 21 Mar 2025 14:38:01 +0000
Subject: [PATCH 26/26] fix shutdown manager tests

---
 .../src/v3/serverOnly/shutdownManager.test.ts | 76 ++++++++++---------
 .../core/src/v3/serverOnly/shutdownManager.ts | 31 ++++++--
 packages/redis-worker/src/worker.test.ts      |  7 +-
 3 files changed, 68 insertions(+), 46 deletions(-)

diff --git a/packages/core/src/v3/serverOnly/shutdownManager.test.ts b/packages/core/src/v3/serverOnly/shutdownManager.test.ts
index 53af9c11f4..8c66af777d 100644
--- a/packages/core/src/v3/serverOnly/shutdownManager.test.ts
+++ b/packages/core/src/v3/serverOnly/shutdownManager.test.ts
@@ -1,57 +1,55 @@
 import { describe, test, expect, vi, beforeEach } from "vitest";
-import { shutdownManager } from "./shutdownManager.js";
-
-// Type assertion to access private members for testing
-type PrivateShutdownManager = {
-  handlers: Map<string, { handler: NodeJS.SignalsListener; signals: Array<"SIGTERM" | "SIGINT"> }>;
-  shutdown: (signal: "SIGTERM" | "SIGINT") => Promise<void>;
-  _reset: () => void;
-};
+import { ShutdownManager } from "./shutdownManager.js";
 
 describe("ShutdownManager", { concurrent: false }, () => {
-  const manager = shutdownManager as unknown as PrivateShutdownManager;
   // Mock process.exit to prevent actual exit
   const mockExit = vi.spyOn(process, "exit").mockImplementation(() => undefined as never);
 
   beforeEach(() => {
-    // Clear all mocks and reset the manager before each test
     vi.clearAllMocks();
-    manager._reset();
   });
 
   test("should successfully register a new handler", () => {
+    const manager = new ShutdownManager(false);
+
     const handler = vi.fn();
-    shutdownManager.register("test-handler", handler);
+    manager.register("test-handler", handler);
 
-    expect(manager.handlers.has("test-handler")).toBe(true);
-    const registeredHandler = manager.handlers.get("test-handler");
+    expect(manager._getHandlersForTesting().has("test-handler")).toBe(true);
+    const registeredHandler = manager._getHandlersForTesting().get("test-handler");
     expect(registeredHandler?.handler).toBe(handler);
     expect(registeredHandler?.signals).toEqual(["SIGTERM", "SIGINT"]);
   });
 
   test("should throw error when registering duplicate handler name", () => {
+    const manager = new ShutdownManager(false);
+
     const handler = vi.fn();
-    shutdownManager.register("duplicate-handler", handler);
+    manager.register("duplicate-handler", handler);
 
     expect(() => {
-      shutdownManager.register("duplicate-handler", handler);
+      manager.register("duplicate-handler", handler);
     }).toThrow('Shutdown handler "duplicate-handler" already registered');
   });
 
   test("should register handler with custom signals", () => {
+    const manager = new ShutdownManager(false);
+
     const handler = vi.fn();
-    shutdownManager.register("custom-signals", handler, ["SIGTERM"]);
+    manager.register("custom-signals", handler, ["SIGTERM"]);
 
-    const registeredHandler = manager.handlers.get("custom-signals");
+    const registeredHandler = manager._getHandlersForTesting().get("custom-signals");
     expect(registeredHandler?.signals).toEqual(["SIGTERM"]);
   });
 
   test("should call registered handlers when shutdown is triggered", async () => {
+    const manager = new ShutdownManager(false);
+
     const handler1 = vi.fn();
     const handler2 = vi.fn();
 
-    shutdownManager.register("handler1", handler1);
-    shutdownManager.register("handler2", handler2);
+    manager.register("handler1", handler1);
+    manager.register("handler2", handler2);
 
     await manager.shutdown("SIGTERM");
 
@@ -61,11 +59,13 @@ describe("ShutdownManager", { concurrent: false }, () => {
   });
 
   test("should only call handlers registered for specific signal", async () => {
+    const manager = new ShutdownManager(false);
+
     const handler1 = vi.fn();
     const handler2 = vi.fn();
 
-    shutdownManager.register("handler1", handler1, ["SIGTERM"]);
-    shutdownManager.register("handler2", handler2, ["SIGINT"]);
+    manager.register("handler1", handler1, ["SIGTERM"]);
+    manager.register("handler2", handler2, ["SIGINT"]);
 
     await manager.shutdown("SIGTERM");
 
@@ -75,11 +75,13 @@ describe("ShutdownManager", { concurrent: false }, () => {
   });
 
   test("should handle errors in shutdown handlers gracefully", async () => {
+    const manager = new ShutdownManager(false);
+
     const successHandler = vi.fn();
     const errorHandler = vi.fn().mockRejectedValue(new Error("Handler failed"));
 
-    shutdownManager.register("success-handler", successHandler);
-    shutdownManager.register("error-handler", errorHandler);
+    manager.register("success-handler", successHandler);
+    manager.register("error-handler", errorHandler);
 
     await manager.shutdown("SIGTERM");
 
@@ -89,8 +91,10 @@ describe("ShutdownManager", { concurrent: false }, () => {
   });
 
   test("should only run shutdown sequence once even if called multiple times", async () => {
+    const manager = new ShutdownManager(false);
+
     const handler = vi.fn();
-    shutdownManager.register("test-handler", handler);
+    manager.register("test-handler", handler);
 
     await Promise.all([manager.shutdown("SIGTERM"), manager.shutdown("SIGTERM")]);
 
@@ -99,16 +103,19 @@ describe("ShutdownManager", { concurrent: false }, () => {
     expect(mockExit).toHaveBeenCalledWith(128 + 15);
   });
 
-  test("should exit with correct signal number", async () => {
-    const handler = vi.fn();
-    shutdownManager.register("test-handler", handler);
+  test("should exit with correct signal number on SIGINT", async () => {
+    const manager = new ShutdownManager(false);
+
+    manager.register("test-handler", vi.fn());
 
     await manager.shutdown("SIGINT");
     expect(mockExit).toHaveBeenCalledWith(128 + 2); // SIGINT number
+  });
 
-    vi.clearAllMocks();
-    manager._reset();
-    shutdownManager.register("test-handler", handler);
+  test("should exit with correct signal number on SIGTERM", async () => {
+    const manager = new ShutdownManager(false);
+
+    manager.register("test-handler", vi.fn());
 
     await manager.shutdown("SIGTERM");
     expect(mockExit).toHaveBeenCalledWith(128 + 15); // SIGTERM number
@@ -116,6 +123,7 @@ describe("ShutdownManager", { concurrent: false }, () => {
 
   test("should only exit after all handlers have finished", async () => {
     const sequence: string[] = [];
+    const manager = new ShutdownManager(false);
 
     const handler1 = vi.fn().mockImplementation(async () => {
       sequence.push("handler1 start");
@@ -144,9 +152,9 @@ describe("ShutdownManager", { concurrent: false }, () => {
       return undefined as never;
     });
 
-    shutdownManager.register("handler1", handler1);
-    shutdownManager.register("handler2", handler2);
-    shutdownManager.register("handler3", handler3);
+    manager.register("handler1", handler1);
+    manager.register("handler2", handler2);
+    manager.register("handler3", handler3);
 
     await manager.shutdown("SIGTERM");
 
diff --git a/packages/core/src/v3/serverOnly/shutdownManager.ts b/packages/core/src/v3/serverOnly/shutdownManager.ts
index b6e940ed85..c2aa807781 100644
--- a/packages/core/src/v3/serverOnly/shutdownManager.ts
+++ b/packages/core/src/v3/serverOnly/shutdownManager.ts
@@ -1,3 +1,4 @@
+import { isTest } from "std-env";
 import { SimpleStructuredLogger } from "../utils/structuredLogger.js";
 import { singleton } from "./singleton.js";
 
@@ -5,7 +6,7 @@ type ShutdownHandler = NodeJS.SignalsListener;
 // We intentionally keep these limited to avoid unexpected issues with signal handling
 type ShutdownSignal = Extract<NodeJS.Signals, "SIGTERM" | "SIGINT">;
 
-class ShutdownManager {
+export class ShutdownManager {
   private isShuttingDown = false;
   private signalNumbers: Record<ShutdownSignal, number> = {
     SIGINT: 2,
@@ -16,7 +17,9 @@ class ShutdownManager {
   private handlers: Map<string, { handler: ShutdownHandler; signals: ShutdownSignal[] }> =
     new Map();
 
-  constructor() {
+  constructor(private disableForTesting = true) {
+    if (disableForTesting) return;
+
     process.on("SIGTERM", () => this.shutdown("SIGTERM"));
     process.on("SIGINT", () => this.shutdown("SIGINT"));
   }
@@ -26,6 +29,8 @@ class ShutdownManager {
     handler: ShutdownHandler,
     signals: ShutdownSignal[] = ["SIGTERM", "SIGINT"]
   ) {
+    if (!this.isEnabled()) return;
+
     if (this.handlers.has(name)) {
       throw new Error(`Shutdown handler "${name}" already registered`);
     }
@@ -33,6 +38,8 @@ class ShutdownManager {
   }
 
   unregister(name: string) {
+    if (!this.isEnabled()) return;
+
     if (!this.handlers.has(name)) {
       throw new Error(`Shutdown handler "${name}" not registered`);
     }
@@ -41,6 +48,8 @@ class ShutdownManager {
   }
 
   async shutdown(signal: ShutdownSignal) {
+    if (!this.isEnabled()) return;
+
     if (this.isShuttingDown) return;
     this.isShuttingDown = true;
 
@@ -83,10 +92,20 @@ class ShutdownManager {
     }
   }
 
-  // For testing purposes only - keep this
-  private _reset() {
-    this.isShuttingDown = false;
-    this.handlers.clear();
+  private isEnabled() {
+    if (!this.disableForTesting) {
+      return true;
+    }
+
+    return !isTest;
+  }
+
+  // Only for testing
+  public _getHandlersForTesting(): ReadonlyMap<
+    string,
+    { handler: ShutdownHandler; signals: ShutdownSignal[] }
+  > {
+    return new Map(this.handlers);
   }
 }
 
diff --git a/packages/redis-worker/src/worker.test.ts b/packages/redis-worker/src/worker.test.ts
index e621bd2d3a..1768f39107 100644
--- a/packages/redis-worker/src/worker.test.ts
+++ b/packages/redis-worker/src/worker.test.ts
@@ -1,17 +1,12 @@
 import { redisTest } from "@internal/testcontainers";
 import { Logger } from "@trigger.dev/core/logger";
 import { describe } from "node:test";
-import { afterEach, expect } from "vitest";
+import { expect } from "vitest";
 import { z } from "zod";
 import { Worker } from "./worker.js";
 import { createRedisClient } from "@internal/redis";
-import { shutdownManager } from "@trigger.dev/core/v3/serverOnly";
 
 describe("Worker", () => {
-  afterEach(() => {
-    (shutdownManager as unknown as { _reset: () => void })._reset();
-  });
-
   redisTest("Process items that don't throw", { timeout: 30_000 }, async ({ redisContainer }) => {
     const processedItems: number[] = [];
     const worker = new Worker({