diff --git a/packages/amp/package.json b/packages/amp/package.json index 9d6ca7f..af59a46 100644 --- a/packages/amp/package.json +++ b/packages/amp/package.json @@ -58,6 +58,7 @@ "effect": "^3.19.15" }, "dependencies": { + "jiti": "^2.6.1", "viem": "^2.44.4" } } diff --git a/packages/amp/src/config.ts b/packages/amp/src/config.ts new file mode 100644 index 0000000..a439d8d --- /dev/null +++ b/packages/amp/src/config.ts @@ -0,0 +1,295 @@ +import * as FileSystem from "@effect/platform/FileSystem" +import * as Path from "@effect/platform/Path" +import type * as Cause from "effect/Cause" +import * as Context from "effect/Context" +import * as Data from "effect/Data" +import * as Effect from "effect/Effect" +import * as Either from "effect/Either" +import { identity } from "effect/Function" +import * as Layer from "effect/Layer" +import * as Match from "effect/Match" +import type * as Option from "effect/Option" +import * as Predicate from "effect/Predicate" +import * as Schema from "effect/Schema" +import * as Stream from "effect/Stream" +import * as fs from "node:fs" +import * as path from "node:path" +import * as ManifestBuilder from "./manifest-builder/service.ts" +import * as Models from "./Models.ts" + +export class ModuleContext { + public definitionPath: string + + constructor(definitionPath: string) { + this.definitionPath = definitionPath + } + + /** + * Reads a file relative to the directory of the dataset definition. + */ + functionSource(relativePath: string): Models.FunctionSource { + const baseDir = path.dirname(path.resolve(this.definitionPath)) + const fullPath = path.resolve(baseDir, relativePath) + if (!fullPath.startsWith(baseDir + path.sep)) { + throw new Error(`Invalid path: directory traversal not allowed`) + } + + let source: string + try { + source = fs.readFileSync(fullPath, "utf8") + } catch (err: any) { + throw new Error( + `Failed to read function source at ${fullPath}: ${err.message}`, + { cause: err } + ) + } + + const func = Models.FunctionSource.make({ + source, + filename: path.basename(fullPath) + }) + return func + } +} + +export class ConfigLoaderError extends Data.TaggedError("ConfigLoaderError")<{ + readonly cause?: unknown + readonly message?: string +}> {} + +export class ConfigLoader extends Context.Tag("Amp/ConfigLoader") Effect.Effect + + /** + * Finds a config file in the given directory by checking for known config file names. + */ + readonly find: (cwd?: string) => Effect.Effect, ConfigLoaderError> + + /** + * Loads and builds a dataset configuration from a file. + */ + readonly build: (file: string) => Effect.Effect + + /** + * Watches a config file for changes and emits built manifests. + */ + readonly watch: (file: string, options?: { + readonly onError?: (cause: Cause.Cause) => Effect.Effect + }) => Stream.Stream +}>() {} + +const make = Effect.gen(function*() { + const path = yield* Path.Path + const fs = yield* FileSystem.FileSystem + const builder = yield* ManifestBuilder.ManifestBuilder + + const decodeDatasetConfig = Schema.decodeUnknown(Models.DatasetConfig) + + const jiti = yield* Effect.tryPromise({ + try: () => + import("jiti").then(({ createJiti }) => + createJiti(import.meta.url, { + moduleCache: false, + tryNative: false + }) + ), + catch: (cause) => new ConfigLoaderError({ cause }) + }).pipe(Effect.cached) + + const loadTypeScript = Effect.fnUntraced(function*(file: string) { + return yield* Effect.tryMapPromise(jiti, { + try: (jiti) => + jiti.import<(context: ModuleContext) => Models.DatasetConfig>(file, { + default: true + }), + catch: identity + }).pipe( + Effect.map((callback) => callback(new ModuleContext(file))), + Effect.flatMap(decodeDatasetConfig), + Effect.mapError((cause) => + new ConfigLoaderError({ + cause, + message: `Failed to load config file ${file}` + }) + ) + ) + }) + + const loadJavaScript = Effect.fnUntraced(function*(file: string) { + return yield* Effect.tryPromise({ + try: () => + import(file).then( + (module) => module.default as (context: ModuleContext) => Models.DatasetConfig + ), + catch: identity + }).pipe( + Effect.map((callback) => callback(new ModuleContext(file))), + Effect.flatMap(decodeDatasetConfig), + Effect.mapError((cause) => + new ConfigLoaderError({ + cause, + message: `Failed to load config file ${file}` + }) + ) + ) + }) + + const loadJson = Effect.fnUntraced(function*(file: string) { + return yield* Effect.tryMap(fs.readFileString(file), { + try: (content) => JSON.parse(content), + catch: identity + }).pipe( + Effect.flatMap(decodeDatasetConfig), + Effect.mapError((cause) => + new ConfigLoaderError({ + cause, + message: `Failed to load config file ${file}` + }) + ) + ) + }) + + const fileMatcher = Match.type().pipe( + Match.when( + (_) => /\.(ts|mts|cts)$/.test(path.extname(_)), + (_) => loadTypeScript(_) + ), + Match.when( + (_) => /\.(js|mjs|cjs)$/.test(path.extname(_)), + (_) => loadJavaScript(_) + ), + Match.when( + (_) => /\.(json)$/.test(path.extname(_)), + (_) => loadJson(_) + ), + Match.orElse((_) => + new ConfigLoaderError({ + message: `Unsupported file extension ${path.extname(_)}` + }) + ) + ) + + const load = Effect.fnUntraced(function*(file: string) { + const resolved = path.resolve(file) + return yield* fileMatcher(resolved) + }) + + const build = Effect.fnUntraced(function*(file: string) { + const config = yield* load(file) + return yield* builder.build(config).pipe( + Effect.mapError( + (cause) => + new ConfigLoaderError({ + cause, + message: `Failed to build config file ${file}` + }) + ) + ) + }) + + const CANDIDATE_CONFIG_FILES = [ + "amp.config.ts", + "amp.config.mts", + "amp.config.cts", + "amp.config.js", + "amp.config.mjs", + "amp.config.cjs", + "amp.config.json" + ] + + const find = Effect.fnUntraced(function*(cwd: string = ".") { + const baseCwd = path.resolve(".") + const resolvedCwd = path.resolve(cwd) + if (resolvedCwd !== baseCwd && !resolvedCwd.startsWith(baseCwd + path.sep)) { + return yield* new ConfigLoaderError({ + message: "Invalid directory path: directory traversal not allowed" + }) + } + const candidates = CANDIDATE_CONFIG_FILES.map((fileName) => { + const filePath = path.resolve(cwd, fileName) + return fs.exists(filePath).pipe( + Effect.flatMap((exists) => exists ? Effect.succeed(filePath) : Effect.fail("not found")) + ) + }) + return yield* Effect.firstSuccessOf(candidates).pipe(Effect.option) + }) + + const watch = (file: string, options?: { + readonly onError?: ( + cause: Cause.Cause + ) => Effect.Effect + }): Stream.Stream< + ManifestBuilder.ManifestBuildResult, + ConfigLoaderError | E, + R + > => { + const baseCwd = path.resolve(".") + const resolved = path.resolve(file) + if (resolved !== baseCwd && !resolved.startsWith(baseCwd + path.sep)) { + return Stream.fail( + new ConfigLoaderError({ + message: "Invalid file path: directory traversal not allowed" + }) + ) + } + const open = load(resolved).pipe( + Effect.tapErrorCause(options?.onError ?? (() => Effect.void)), + Effect.either + ) + + const updates = fs.watch(resolved).pipe( + Stream.buffer({ capacity: 1, strategy: "sliding" }), + Stream.mapError( + (cause) => + new ConfigLoaderError({ + cause, + message: "Failed to watch config file" + }) + ), + Stream.filter(Predicate.isTagged("Update")), + Stream.mapEffect(() => open) + ) + + const build = (config: Models.DatasetConfig) => + builder.build(config).pipe( + Effect.mapError( + (cause) => + new ConfigLoaderError({ + cause, + message: `Failed to build config file ${file}` + }) + ), + Effect.tapErrorCause(options?.onError ?? (() => Effect.void)), + Effect.either + ) + + return Stream.fromEffect(open).pipe( + Stream.concat(updates), + Stream.filterMap(Either.getRight), + Stream.changesWith(DatasetConfigEquivalence), + Stream.mapEffect(build), + Stream.filterMap(Either.getRight), + Stream.changesWith((a, b) => + DatasetDerivedEquivalence(a.manifest, b.manifest) && + DatasetMetadataEquivalence(a.metadata, b.metadata) + ) + ) as Stream.Stream< + ManifestBuilder.ManifestBuildResult, + ConfigLoaderError | E, + R + > + } + + return { load, find, watch, build } +}) + +export const layer = Layer.effect(ConfigLoader, make).pipe( + Layer.provide(ManifestBuilder.layer) +) + +const DatasetConfigEquivalence = Schema.equivalence(Models.DatasetConfig) +const DatasetDerivedEquivalence = Schema.equivalence(Models.DatasetDerived) +const DatasetMetadataEquivalence = Schema.equivalence(Models.DatasetMetadata) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ef2af37..d14849c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -79,16 +79,19 @@ importers: version: 5.9.3 vite-tsconfig-paths: specifier: ^6.0.4 - version: 6.0.4(typescript@5.9.3)(vite@7.3.1(@types/node@25.0.10)(yaml@2.8.2)) + version: 6.0.4(typescript@5.9.3)(vite@7.3.1(@types/node@25.0.10)(jiti@2.6.1)(yaml@2.8.2)) vitest: specifier: ^4.0.18 - version: 4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(yaml@2.8.2) + version: 4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(jiti@2.6.1)(yaml@2.8.2) vitest-mock-express: specifier: ^2.2.0 version: 2.2.0 packages/amp: dependencies: + jiti: + specifier: ^2.6.1 + version: 2.6.1 viem: specifier: ^2.44.4 version: 2.44.4(typescript@5.9.3) @@ -122,7 +125,7 @@ importers: version: 25.0.10 vitest: specifier: ^4.0.18 - version: 4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(yaml@2.8.2) + version: 4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(jiti@2.6.1)(yaml@2.8.2) scratchpad: dependencies: @@ -1563,6 +1566,10 @@ packages: resolution: {integrity: sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==} engines: {node: '>=8'} + jiti@2.6.1: + resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==} + hasBin: true + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -2440,7 +2447,7 @@ snapshots: '@effect/vitest@0.27.0(effect@3.19.15)(vitest@4.0.18)': dependencies: effect: 3.19.15 - vitest: 4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(yaml@2.8.2) + vitest: 4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(jiti@2.6.1)(yaml@2.8.2) '@effect/workflow@0.16.0(@effect/experimental@0.58.0(@effect/platform@0.94.2(effect@3.19.15))(effect@3.19.15))(@effect/platform@0.94.2(effect@3.19.15))(@effect/rpc@0.73.0(@effect/platform@0.94.2(effect@3.19.15))(effect@3.19.15))(effect@3.19.15)': dependencies: @@ -2916,7 +2923,7 @@ snapshots: obug: 2.1.1 std-env: 3.10.0 tinyrainbow: 3.0.3 - vitest: 4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(yaml@2.8.2) + vitest: 4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(jiti@2.6.1)(yaml@2.8.2) '@vitest/expect@4.0.18': dependencies: @@ -2927,13 +2934,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.0.3 - '@vitest/mocker@4.0.18(vite@7.3.1(@types/node@25.0.10)(yaml@2.8.2))': + '@vitest/mocker@4.0.18(vite@7.3.1(@types/node@25.0.10)(jiti@2.6.1)(yaml@2.8.2))': dependencies: '@vitest/spy': 4.0.18 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.3.1(@types/node@25.0.10)(yaml@2.8.2) + vite: 7.3.1(@types/node@25.0.10)(jiti@2.6.1)(yaml@2.8.2) '@vitest/pretty-format@4.0.18': dependencies: @@ -2961,7 +2968,7 @@ snapshots: sirv: 3.0.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vitest: 4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(yaml@2.8.2) + vitest: 4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(jiti@2.6.1)(yaml@2.8.2) '@vitest/utils@4.0.18': dependencies: @@ -3445,6 +3452,8 @@ snapshots: html-escaper: 2.0.2 istanbul-lib-report: 3.0.1 + jiti@2.6.1: {} + js-tokens@4.0.0: {} js-tokens@9.0.1: {} @@ -3909,18 +3918,18 @@ snapshots: - utf-8-validate - zod - vite-tsconfig-paths@6.0.4(typescript@5.9.3)(vite@7.3.1(@types/node@25.0.10)(yaml@2.8.2)): + vite-tsconfig-paths@6.0.4(typescript@5.9.3)(vite@7.3.1(@types/node@25.0.10)(jiti@2.6.1)(yaml@2.8.2)): dependencies: debug: 4.4.3 globrex: 0.1.2 tsconfck: 3.1.6(typescript@5.9.3) optionalDependencies: - vite: 7.3.1(@types/node@25.0.10)(yaml@2.8.2) + vite: 7.3.1(@types/node@25.0.10)(jiti@2.6.1)(yaml@2.8.2) transitivePeerDependencies: - supports-color - typescript - vite@7.3.1(@types/node@25.0.10)(yaml@2.8.2): + vite@7.3.1(@types/node@25.0.10)(jiti@2.6.1)(yaml@2.8.2): dependencies: esbuild: 0.27.2 fdir: 6.5.0(picomatch@4.0.3) @@ -3931,16 +3940,17 @@ snapshots: optionalDependencies: '@types/node': 25.0.10 fsevents: 2.3.3 + jiti: 2.6.1 yaml: 2.8.2 vitest-mock-express@2.2.0: dependencies: '@types/express': 4.17.25 - vitest@4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(yaml@2.8.2): + vitest@4.0.18(@types/node@25.0.10)(@vitest/ui@4.0.18)(jiti@2.6.1)(yaml@2.8.2): dependencies: '@vitest/expect': 4.0.18 - '@vitest/mocker': 4.0.18(vite@7.3.1(@types/node@25.0.10)(yaml@2.8.2)) + '@vitest/mocker': 4.0.18(vite@7.3.1(@types/node@25.0.10)(jiti@2.6.1)(yaml@2.8.2)) '@vitest/pretty-format': 4.0.18 '@vitest/runner': 4.0.18 '@vitest/snapshot': 4.0.18 @@ -3957,7 +3967,7 @@ snapshots: tinyexec: 1.0.2 tinyglobby: 0.2.15 tinyrainbow: 3.0.3 - vite: 7.3.1(@types/node@25.0.10)(yaml@2.8.2) + vite: 7.3.1(@types/node@25.0.10)(jiti@2.6.1)(yaml@2.8.2) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 25.0.10