From 64cc9958ff25fc40045d9f6773f792517c45da84 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 2 Dec 2025 15:30:50 +0000 Subject: [PATCH 1/2] feat(cli): implements content-addressable store for the dev CLI build outputs, reducing disk usage --- .changeset/polite-eels-divide.md | 5 ++ packages/cli-v3/src/build/bundle.ts | 32 +++++-- packages/cli-v3/src/build/manifests.ts | 83 +++++++++++++++++-- packages/cli-v3/src/dev/devSession.ts | 19 ++++- packages/cli-v3/src/utilities/fileSystem.ts | 53 ++++++++++++ .../cli-v3/src/utilities/tempDirectories.ts | 12 +++ packages/core/src/v3/schemas/build.ts | 2 + references/d3-chat/src/trigger/chat.ts | 2 - 8 files changed, 192 insertions(+), 16 deletions(-) create mode 100644 .changeset/polite-eels-divide.md diff --git a/.changeset/polite-eels-divide.md b/.changeset/polite-eels-divide.md new file mode 100644 index 0000000000..54290d55e0 --- /dev/null +++ b/.changeset/polite-eels-divide.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +feat(cli): implements content-addressable store for the dev CLI build outputs, reducing disk usage diff --git a/packages/cli-v3/src/build/bundle.ts b/packages/cli-v3/src/build/bundle.ts index 597b46854a..c085937f68 100644 --- a/packages/cli-v3/src/build/bundle.ts +++ b/packages/cli-v3/src/build/bundle.ts @@ -3,8 +3,8 @@ import { DEFAULT_RUNTIME, ResolvedConfig } from "@trigger.dev/core/v3/build"; import { BuildManifest, BuildTarget, TaskFile } from "@trigger.dev/core/v3/schemas"; import * as esbuild from "esbuild"; import { createHash } from "node:crypto"; -import { join, relative, resolve } from "node:path"; -import { createFile } from "../utilities/fileSystem.js"; +import { basename, join, relative, resolve } from "node:path"; +import { createFile, createFileWithStore } from "../utilities/fileSystem.js"; import { logger } from "../utilities/logger.js"; import { resolveFileSources } from "../utilities/sourceFiles.js"; import { VERSION } from "../version.js"; @@ -37,6 +37,8 @@ export interface BundleOptions { jsxAutomatic?: boolean; watch?: boolean; plugins?: esbuild.Plugin[]; + /** Shared store directory for deduplicating chunk files via hardlinks */ + storeDir?: string; } export type BundleResult = { @@ -51,6 +53,8 @@ export type BundleResult = { indexControllerEntryPoint: string | undefined; initEntryPoint: string | undefined; stop: (() => Promise) | undefined; + /** Maps output file paths to their content hashes for deduplication */ + outputHashes: Record; }; export class BundleError extends Error { @@ -159,7 +163,8 @@ export async function bundleWorker(options: BundleOptions): Promise + result: esbuild.BuildResult<{ metafile: true; write: false }>, + storeDir?: string ): Promise | undefined> { const hasher = createHash("md5"); + const outputHashes: Record = {}; for (const outputFile of result.outputFiles) { hasher.update(outputFile.hash); - - await createFile(outputFile.path, outputFile.contents); + // Store the hash for each output file (keyed by path) + outputHashes[outputFile.path] = outputFile.hash; + + if (storeDir) { + // Use content-addressable store with esbuild's built-in hash for ALL files + await createFileWithStore(outputFile.path, outputFile.contents, storeDir, outputFile.hash); + } else { + await createFile(outputFile.path, outputFile.contents); + } } const files: Array<{ entry: string; out: string }> = []; @@ -308,6 +322,7 @@ export async function getBundleResultFromBuild( initEntryPoint, contentHash: hasher.digest("hex"), metafile: result.metafile, + outputHashes, }; } @@ -354,6 +369,7 @@ export async function createBuildManifestFromBundle({ target, envVars, sdkVersion, + storeDir, }: { bundle: BundleResult; destination: string; @@ -364,6 +380,7 @@ export async function createBuildManifestFromBundle({ target: BuildTarget; envVars?: Record; sdkVersion?: string; + storeDir?: string; }): Promise { const buildManifest: BuildManifest = { contentHash: bundle.contentHash, @@ -397,11 +414,12 @@ export async function createBuildManifestFromBundle({ otelImportHook: { include: resolvedConfig.instrumentedPackageNames ?? [], }, + outputHashes: bundle.outputHashes, }; if (!workerDir) { return buildManifest; } - return copyManifestToDir(buildManifest, destination, workerDir); + return copyManifestToDir(buildManifest, destination, workerDir, storeDir); } diff --git a/packages/cli-v3/src/build/manifests.ts b/packages/cli-v3/src/build/manifests.ts index 04fe2e3f6c..b676d10f1f 100644 --- a/packages/cli-v3/src/build/manifests.ts +++ b/packages/cli-v3/src/build/manifests.ts @@ -1,16 +1,25 @@ import { BuildManifest } from "@trigger.dev/core/v3/schemas"; -import { cp } from "node:fs/promises"; +import { cp, link, mkdir, readdir, readFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; import { logger } from "../utilities/logger.js"; export async function copyManifestToDir( manifest: BuildManifest, source: string, - destination: string + destination: string, + storeDir?: string ): Promise { - // Copy the dir in destination to workerDir - await cp(source, destination, { recursive: true }); + // Copy the dir from source to destination + // If storeDir is provided, create hardlinks for files that exist in the store + if (storeDir) { + await copyDirWithStore(source, destination, storeDir, manifest.outputHashes); + } else { + await cp(source, destination, { recursive: true }); + } - logger.debug("Copied manifest to dir", { source, destination }); + logger.debug("Copied manifest to dir", { source, destination, storeDir }); // Then update the manifest to point to the new workerDir const updatedManifest = { ...manifest }; @@ -37,3 +46,67 @@ export async function copyManifestToDir( return updatedManifest; } + +/** + * Sanitizes a hash to be safe for use as a filename. + * esbuild's hashes are base64-encoded and may contain `/` and `+` characters. + */ +function sanitizeHashForFilename(hash: string): string { + return hash.replace(/\//g, "_").replace(/\+/g, "-"); +} + +/** + * Computes a hash of file contents to use as content-addressable key. + * This is a fallback for when outputHashes is not available. + */ +async function computeFileHash(filePath: string): Promise { + const contents = await readFile(filePath); + return createHash("sha256").update(contents).digest("hex").slice(0, 16); +} + +/** + * Recursively copies a directory, using hardlinks for files that exist in the store. + * This preserves disk space savings from the content-addressable store. + * + * @param source - Source directory path + * @param destination - Destination directory path + * @param storeDir - Content-addressable store directory + * @param outputHashes - Optional map of file paths to their content hashes (from BuildManifest) + */ +async function copyDirWithStore( + source: string, + destination: string, + storeDir: string, + outputHashes?: Record +): Promise { + await mkdir(destination, { recursive: true }); + + const entries = await readdir(source, { withFileTypes: true }); + + for (const entry of entries) { + const sourcePath = join(source, entry.name); + const destPath = join(destination, entry.name); + + if (entry.isDirectory()) { + // Recursively copy subdirectories + await copyDirWithStore(sourcePath, destPath, storeDir, outputHashes); + } else if (entry.isFile()) { + // Try to get hash from manifest first, otherwise compute it + const contentHash = outputHashes?.[sourcePath] ?? (await computeFileHash(sourcePath)); + // Sanitize hash to be filesystem-safe (base64 can contain / and +) + const safeHash = sanitizeHashForFilename(contentHash); + const storePath = join(storeDir, safeHash); + + if (existsSync(storePath)) { + // Create hardlink to store file + await link(storePath, destPath); + } else { + // File wasn't in the store - copy normally + await cp(sourcePath, destPath); + } + } else if (entry.isSymbolicLink()) { + // Preserve symbolic links (e.g., node_modules links) + await cp(sourcePath, destPath, { verbatimSymlinks: true }); + } + } +} diff --git a/packages/cli-v3/src/dev/devSession.ts b/packages/cli-v3/src/dev/devSession.ts index 27e31b85ed..8b4068ff44 100644 --- a/packages/cli-v3/src/dev/devSession.ts +++ b/packages/cli-v3/src/dev/devSession.ts @@ -20,7 +20,12 @@ import { createExternalsBuildExtension, resolveAlwaysExternal } from "../build/e import { type DevCommandOptions } from "../commands/dev.js"; import { eventBus } from "../utilities/eventBus.js"; import { logger } from "../utilities/logger.js"; -import { clearTmpDirs, EphemeralDirectory, getTmpDir } from "../utilities/tempDirectories.js"; +import { + clearTmpDirs, + EphemeralDirectory, + getStoreDir, + getTmpDir, +} from "../utilities/tempDirectories.js"; import { startDevOutput } from "./devOutput.js"; import { startWorkerRuntime } from "./devSupervisor.js"; import { startMcpServer, stopMcpServer } from "./mcpServer.js"; @@ -53,6 +58,8 @@ export async function startDevSession({ }: DevSessionOptions): Promise { clearTmpDirs(rawConfig.workingDir); const destination = getTmpDir(rawConfig.workingDir, "build", keepTmpFiles); + // Create shared store directory for deduplicating chunk files across rebuilds + const storeDir = getStoreDir(rawConfig.workingDir); const runtime = await startWorkerRuntime({ name, @@ -102,6 +109,7 @@ export async function startDevSession({ workerDir: workerDir?.path, environment: "dev", target: "dev", + storeDir, }); logger.debug("Created build manifest from bundle", { buildManifest }); @@ -131,7 +139,13 @@ export async function startDevSession({ } async function updateBuild(build: esbuild.BuildResult, workerDir: EphemeralDirectory) { - const bundle = await getBundleResultFromBuild("dev", rawConfig.workingDir, rawConfig, build); + const bundle = await getBundleResultFromBuild( + "dev", + rawConfig.workingDir, + rawConfig, + build, + storeDir + ); if (bundle) { await updateBundle({ ...bundle, stop: undefined }, workerDir); @@ -190,6 +204,7 @@ export async function startDevSession({ jsxFactory: rawConfig.build.jsx.factory, jsxFragment: rawConfig.build.jsx.fragment, jsxAutomatic: rawConfig.build.jsx.automatic, + storeDir, }); await updateBundle(bundleResult); diff --git a/packages/cli-v3/src/utilities/fileSystem.ts b/packages/cli-v3/src/utilities/fileSystem.ts index 2de037582c..c7a62497fd 100644 --- a/packages/cli-v3/src/utilities/fileSystem.ts +++ b/packages/cli-v3/src/utilities/fileSystem.ts @@ -16,6 +16,59 @@ export async function createFile( return path; } +/** + * Sanitizes a hash to be safe for use as a filename. + * esbuild's hashes are base64-encoded and may contain `/` and `+` characters. + */ +function sanitizeHashForFilename(hash: string): string { + return hash.replace(/\//g, "_").replace(/\+/g, "-"); +} + +/** + * Creates a file using a content-addressable store for deduplication. + * Files are stored by their content hash, so identical content is only stored once. + * The build directory gets a hardlink to the stored file. + * + * @param filePath - The destination path for the file + * @param contents - The file contents to write + * @param storeDir - The shared store directory for deduplication + * @param contentHash - The content hash (e.g., from esbuild's outputFile.hash) + * @returns The destination file path + */ +export async function createFileWithStore( + filePath: string, + contents: string | NodeJS.ArrayBufferView, + storeDir: string, + contentHash: string +): Promise { + // Sanitize hash to be filesystem-safe (base64 can contain / and +) + const safeHash = sanitizeHashForFilename(contentHash); + // Store files by their content hash for true content-addressable storage + const storePath = pathModule.join(storeDir, safeHash); + + // Ensure build directory exists + await fsModule.mkdir(pathModule.dirname(filePath), { recursive: true }); + + // Remove existing file at destination if it exists (hardlinks fail on existing files) + if (fsSync.existsSync(filePath)) { + await fsModule.unlink(filePath); + } + + // Check if content already exists in store by hash + if (fsSync.existsSync(storePath)) { + // Create hardlink from build path to store path + await fsModule.link(storePath, filePath); + return filePath; + } + + // Write to store first (using hash as filename) + await fsModule.writeFile(storePath, contents); + // Create hardlink in build directory (with original filename) + await fsModule.link(storePath, filePath); + + return filePath; +} + export function isDirectory(configPath: string) { try { return fs.statSync(configPath).isDirectory(); diff --git a/packages/cli-v3/src/utilities/tempDirectories.ts b/packages/cli-v3/src/utilities/tempDirectories.ts index 8d36d419a5..1de3a8d09b 100644 --- a/packages/cli-v3/src/utilities/tempDirectories.ts +++ b/packages/cli-v3/src/utilities/tempDirectories.ts @@ -58,3 +58,15 @@ export function clearTmpDirs(projectRoot: string | undefined) { // This sometimes fails on Windows with EBUSY } } + +/** + * Gets the shared store directory for content-addressable build outputs. + * This directory persists across rebuilds and is used to deduplicate + * identical chunk files between build versions. + */ +export function getStoreDir(projectRoot: string | undefined): string { + projectRoot ??= process.cwd(); + const storeDir = path.join(projectRoot, ".trigger", "tmp", "store"); + fs.mkdirSync(storeDir, { recursive: true }); + return storeDir; +} diff --git a/packages/core/src/v3/schemas/build.ts b/packages/core/src/v3/schemas/build.ts index ee33bb7efb..8eb97f0f35 100644 --- a/packages/core/src/v3/schemas/build.ts +++ b/packages/core/src/v3/schemas/build.ts @@ -68,6 +68,8 @@ export const BuildManifest = z.object({ exclude: z.array(z.string()).optional(), }) .optional(), + /** Maps output file paths to their content hashes for deduplication during dev */ + outputHashes: z.record(z.string()).optional(), }); export type BuildManifest = z.infer; diff --git a/references/d3-chat/src/trigger/chat.ts b/references/d3-chat/src/trigger/chat.ts index d0a2f2d8dd..da07621046 100644 --- a/references/d3-chat/src/trigger/chat.ts +++ b/references/d3-chat/src/trigger/chat.ts @@ -199,8 +199,6 @@ export const todoChat = schemaTask({ run: async ({ input, userId }, { signal }) => { metadata.set("user_id", userId); - logger.info("todoChat: starting", { input, userId }); - const system = ` You are a SQL (postgres) expert who can turn natural language descriptions for a todo app into a SQL query which can then be executed against a SQL database. Here is the schema: From df5ef35616e10c236cb5c46b385df1e42acc7885 Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Tue, 2 Dec 2025 16:21:16 +0000 Subject: [PATCH 2/2] fix a few things --- packages/cli-v3/src/build/manifests.ts | 20 +++++++++-------- packages/cli-v3/src/utilities/fileSystem.ts | 24 ++++++++++++++++++--- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/packages/cli-v3/src/build/manifests.ts b/packages/cli-v3/src/build/manifests.ts index b676d10f1f..8b1da98ceb 100644 --- a/packages/cli-v3/src/build/manifests.ts +++ b/packages/cli-v3/src/build/manifests.ts @@ -4,6 +4,7 @@ import { createHash } from "node:crypto"; import { existsSync } from "node:fs"; import { join } from "node:path"; import { logger } from "../utilities/logger.js"; +import { sanitizeHashForFilename } from "../utilities/fileSystem.js"; export async function copyManifestToDir( manifest: BuildManifest, @@ -47,14 +48,6 @@ export async function copyManifestToDir( return updatedManifest; } -/** - * Sanitizes a hash to be safe for use as a filename. - * esbuild's hashes are base64-encoded and may contain `/` and `+` characters. - */ -function sanitizeHashForFilename(hash: string): string { - return hash.replace(/\//g, "_").replace(/\+/g, "-"); -} - /** * Computes a hash of file contents to use as content-addressable key. * This is a fallback for when outputHashes is not available. @@ -99,7 +92,16 @@ async function copyDirWithStore( if (existsSync(storePath)) { // Create hardlink to store file - await link(storePath, destPath); + // Fall back to copy if hardlink fails (e.g., on Windows or cross-device) + try { + await link(storePath, destPath); + } catch (linkError) { + try { + await cp(storePath, destPath); + } catch (copyError) { + throw linkError; // Rethrow original error if copy also fails + } + } } else { // File wasn't in the store - copy normally await cp(sourcePath, destPath); diff --git a/packages/cli-v3/src/utilities/fileSystem.ts b/packages/cli-v3/src/utilities/fileSystem.ts index c7a62497fd..28178881b1 100644 --- a/packages/cli-v3/src/utilities/fileSystem.ts +++ b/packages/cli-v3/src/utilities/fileSystem.ts @@ -20,7 +20,7 @@ export async function createFile( * Sanitizes a hash to be safe for use as a filename. * esbuild's hashes are base64-encoded and may contain `/` and `+` characters. */ -function sanitizeHashForFilename(hash: string): string { +export function sanitizeHashForFilename(hash: string): string { return hash.replace(/\//g, "_").replace(/\+/g, "-"); } @@ -57,14 +57,32 @@ export async function createFileWithStore( // Check if content already exists in store by hash if (fsSync.existsSync(storePath)) { // Create hardlink from build path to store path - await fsModule.link(storePath, filePath); + // Fall back to copy if hardlink fails (e.g., on Windows or cross-device) + try { + await fsModule.link(storePath, filePath); + } catch (linkError) { + try { + await fsModule.copyFile(storePath, filePath); + } catch (copyError) { + throw linkError; // Rethrow original error if copy also fails + } + } return filePath; } // Write to store first (using hash as filename) await fsModule.writeFile(storePath, contents); // Create hardlink in build directory (with original filename) - await fsModule.link(storePath, filePath); + // Fall back to copy if hardlink fails (e.g., on Windows or cross-device) + try { + await fsModule.link(storePath, filePath); + } catch (linkError) { + try { + await fsModule.copyFile(storePath, filePath); + } catch (copyError) { + throw linkError; // Rethrow original error if copy also fails + } + } return filePath; }