From 0a61b27be5b2d1ae767b89566437bc49992ffe5c Mon Sep 17 00:00:00 2001 From: Mikhail Kumachev Date: Mon, 25 Aug 2025 14:42:31 +0200 Subject: [PATCH 1/5] Prototype runtime parameters. --- core/actions/assertion.ts | 4 ++++ core/actions/data_preparation.ts | 4 ++++ core/actions/incremental_table.ts | 4 ++++ core/actions/operation.ts | 4 ++++ core/actions/table.ts | 7 +++++- core/actions/test.ts | 6 ++++- core/actions/view.ts | 14 ++++++++++- core/compilers.ts | 3 ++- core/contextables.ts | 2 ++ core/main_test.ts | 40 +++++++++++++++++++++++++++++++ core/session.ts | 6 ++++- protos/core.proto | 16 +++++++++++++ 12 files changed, 105 insertions(+), 5 deletions(-) diff --git a/core/actions/assertion.ts b/core/actions/assertion.ts index 353e394b4..4629ee598 100644 --- a/core/actions/assertion.ts +++ b/core/actions/assertion.ts @@ -364,6 +364,10 @@ export class AssertionContext implements IActionContext { return this.resolve(ref); } + public param(paramName: string): string { + return "" + } + public resolve(ref: Resolvable | string[], ...rest: string[]) { return this.assertion.session.resolve(ref, ...rest); } diff --git a/core/actions/data_preparation.ts b/core/actions/data_preparation.ts index c5bb03cab..52408adc8 100644 --- a/core/actions/data_preparation.ts +++ b/core/actions/data_preparation.ts @@ -454,6 +454,10 @@ export class DataPreparationContext implements ITableContext { return this.resolve(ref); } + public param(paramName: string): string { + return "" + } + public resolve(ref: Resolvable | string[], ...rest: string[]) { return this.dataPreparation.session.resolve(ref, ...rest); } diff --git a/core/actions/incremental_table.ts b/core/actions/incremental_table.ts index 605df68ab..4722eabb0 100644 --- a/core/actions/incremental_table.ts +++ b/core/actions/incremental_table.ts @@ -663,6 +663,10 @@ export class IncrementalTableContext implements ITableContext { return this.incrementalTable.session.finalizeName(this.incrementalTable.getTarget().name); } + public param(paramName: string): string { + return "" + } + public ref(ref: Resolvable | string[], ...rest: string[]): string { ref = toResolvable(ref, rest); if (!resolvableAsTarget(ref)) { diff --git a/core/actions/operation.ts b/core/actions/operation.ts index a03354c9e..309d2e811 100644 --- a/core/actions/operation.ts +++ b/core/actions/operation.ts @@ -425,6 +425,10 @@ export class OperationContext implements IActionContext { return this.resolve(ref); } + public param(paramName: string): string { + return "" + } + public resolve(ref: Resolvable | string[], ...rest: string[]) { return this.operation.session.resolve(ref, ...rest); } diff --git a/core/actions/table.ts b/core/actions/table.ts index 3f8f77117..904e4581f 100644 --- a/core/actions/table.ts +++ b/core/actions/table.ts @@ -93,7 +93,8 @@ export class Table extends ActionBuilder { type: "table", enumType: dataform.TableType.TABLE, disabled: false, - tags: [] + tags: [], + inputParams: [], }); /** @hidden */ @@ -611,6 +612,10 @@ export class TableContext implements ITableContext { return this.table.session.finalizeName(this.table.getTarget().name); } + public param(paramName: string): string { + return "" + } + public ref(ref: Resolvable | string[], ...rest: string[]): string { ref = toResolvable(ref, rest); if (!resolvableAsTarget(ref)) { diff --git a/core/actions/test.ts b/core/actions/test.ts index df781c9c9..d1035d8b4 100644 --- a/core/actions/test.ts +++ b/core/actions/test.ts @@ -221,7 +221,11 @@ class RefReplacingContext implements ITableContext { public ref(ref: Resolvable | string[], ...rest: string[]) { return this.resolve(ref, ...rest); } - + + public param(paramName: string): string { + return "" + } + public resolve(ref: Resolvable | string[], ...rest: string[]) { const target = resolvableAsTarget(toResolvable(ref, rest)); if (!this.testContext.test.contextableInputs.has(targetStringifier.stringify(target))) { diff --git a/core/actions/view.ts b/core/actions/view.ts index 50ac1018f..f8e87eebc 100644 --- a/core/actions/view.ts +++ b/core/actions/view.ts @@ -106,7 +106,8 @@ export class View extends ActionBuilder { type: "view", enumType: dataform.TableType.VIEW, disabled: false, - tags: [] + tags: [], + inputParams: [], }); /** @hidden */ @@ -486,6 +487,12 @@ export class View extends ActionBuilder { return dataform.Target.create(this.proto.target); } + public addInputParam(paramName: string) { + if (!this.proto.inputParams.includes(paramName)) { + this.proto.inputParams.push(paramName); + } + } + /** @hidden */ public compile() { const context = new ViewContext(this); @@ -650,6 +657,11 @@ export class ViewContext implements ITableContext { return this.resolve(ref); } + public param(paramName: string): string { + this.view.addInputParam(paramName); + return `\{${paramName}\}`; + } + public resolve(ref: Resolvable | string[], ...rest: string[]) { return this.view.session.resolve(ref, ...rest); } diff --git a/core/compilers.ts b/core/compilers.ts index 69c8f7704..1f8416b99 100644 --- a/core/compilers.ts +++ b/core/compilers.ts @@ -11,7 +11,8 @@ const CONTEXT_FUNCTIONS = [ "when", "incremental", "schema", - "database" + "database", + "param", ] .map(name => `const ${name} = ctx.${name} ? ctx.${name}.bind(ctx) : undefined;`) .join("\n"); diff --git a/core/contextables.ts b/core/contextables.ts index 7ad6d9bbd..5c16d11b5 100644 --- a/core/contextables.ts +++ b/core/contextables.ts @@ -67,6 +67,8 @@ export interface IActionContext { * Returns the database of this dataset, if applicable. */ database: () => string; + + param: (paramName: string) => string; } /** diff --git a/core/main_test.ts b/core/main_test.ts index e77602898..fbc681bec 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -52,6 +52,46 @@ suite("@dataform/core", ({ afterEach }) => { const tmpDirFixture = new TmpDirFixture(afterEach); suite("session", () => { + test("param resolved correctly", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/view.sqlx"), + ` +config { type: "view" } +SELECT 1 AS col WHERE \${param("myvar")} == 1` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ + { + canonicalTarget: { + database: "defaultProject", + name: "view", + schema: "defaultDataset" + }, + disabled: false, + enumType: "VIEW", + fileName: "definitions/view.sqlx", + hermeticity: "NON_HERMETIC", + query: "\n\nSELECT 1 AS col WHERE {myvar} == 1", + inputParams: ["myvar"], + target: { + database: "defaultProject", + name: "view", + schema: "defaultDataset" + }, + type: "view", + } + ]); + }); + suite("resolve succeeds", () => { [ WorkflowSettingsTemplates.bigquery, diff --git a/core/session.ts b/core/session.ts index f8c9ce2a6..964fb1153 100644 --- a/core/session.ts +++ b/core/session.ts @@ -58,6 +58,8 @@ export class Session { public graphErrors: dataform.IGraphErrors; + private inputParams: Set = new Set(); + constructor( rootDir?: string, projectConfig?: dataform.ProjectConfig, @@ -466,7 +468,8 @@ export class Session { ), graphErrors: this.graphErrors, dataformCoreVersion, - targets: this.actions.map(action => action.getTarget()) + targets: this.actions.map(action => action.getTarget()), + inputParams: Array.from(this.inputParams).sort(), }); this.fullyQualifyDependencies( @@ -545,6 +548,7 @@ export class Session { actions.forEach(action => { try { const compiledChunk = action.compile(); + compiledChunk.inputParams.forEach(param => this.inputParams.add(param)); compiledChunks.push(compiledChunk as any); } catch (e) { this.compileError(e, action.getFileName(), action.getTarget()); diff --git a/protos/core.proto b/protos/core.proto index 93249aee8..32c93c972 100644 --- a/protos/core.proto +++ b/protos/core.proto @@ -165,6 +165,8 @@ message Table { // Generated. string file_name = 18; + repeated string input_params = 38; + reserved 1, 2, 7, 12, 16; } @@ -186,6 +188,8 @@ message Operation { // Generated. string file_name = 7; + repeated string input_params = 15; + reserved 1, 2, 4, 5; } @@ -210,6 +214,8 @@ message Assertion { // Generated. string file_name = 7; + repeated string input_params = 16; + reserved 1, 2, 4, 5, 6; } @@ -225,6 +231,8 @@ message Declaration { ActionDescriptor action_descriptor = 3; + repeated string input_params = 6; + // Generated. string file_name = 4; } @@ -235,6 +243,8 @@ message Test { string test_query = 2; string expected_output_query = 3; + repeated string input_params = 6; + // Generated. string file_name = 4; } @@ -253,6 +263,8 @@ message Notebook { bool disabled = 6; string notebook_contents = 7; + + repeated string input_params = 8; } message NotebookRuntimeOptions { @@ -298,6 +310,8 @@ message DataPreparation { LoadConfiguration load = 16; + repeated string input_params = 14; + reserved 7, 10; } @@ -337,6 +351,8 @@ message CompiledGraph { repeated Target targets = 11; reserved 5, 6; + + repeated string input_params = 14; } message CoreExecutionRequest { From d0fbdcb11267a4a5935a4fb52dba4a7a581c9fc3 Mon Sep 17 00:00:00 2001 From: karo Date: Tue, 2 Sep 2025 13:05:14 +0200 Subject: [PATCH 2/5] changed param to dynamicVar, added dynamicVars[] in actions and protos --- core/actions/assertion.ts | 14 +++++++++++--- core/actions/data_preparation.ts | 14 +++++++++++--- core/actions/declaration.ts | 4 +++- core/actions/incremental_table.ts | 15 +++++++++++---- core/actions/notebook.ts | 4 +++- core/actions/operation.ts | 14 +++++++++++--- core/actions/table.ts | 12 +++++++++--- core/actions/test.ts | 14 +++++++++++--- core/actions/view.ts | 14 +++++++------- 9 files changed, 77 insertions(+), 28 deletions(-) diff --git a/core/actions/assertion.ts b/core/actions/assertion.ts index 4629ee598..48226cf9f 100644 --- a/core/actions/assertion.ts +++ b/core/actions/assertion.ts @@ -85,7 +85,9 @@ export class Assertion extends ActionBuilder { /** * @hidden Stores the generated proto for the compiled graph. */ - private proto = dataform.Assertion.create(); + private proto = dataform.Assertion.create({ + dynamicVars: [], + }); /** @hidden We delay contextification until the final compile step, so hold these here for now. */ private contextableQuery: AContextable; @@ -334,6 +336,11 @@ export class Assertion extends ActionBuilder { VerifyProtoErrorBehaviour.SHOW_DOCS_LINK ); } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } /** @@ -364,8 +371,9 @@ export class AssertionContext implements IActionContext { return this.resolve(ref); } - public param(paramName: string): string { - return "" + public dynamicVar(varName: string): string { + this.assertion.addInputDynamicVar(varName); + return `\{${varName}\}`; } public resolve(ref: Resolvable | string[], ...rest: string[]) { diff --git a/core/actions/data_preparation.ts b/core/actions/data_preparation.ts index 52408adc8..feaa16075 100644 --- a/core/actions/data_preparation.ts +++ b/core/actions/data_preparation.ts @@ -26,7 +26,9 @@ export class DataPreparation extends ActionBuilder { // We delay contextification until the final compile step, so hold these here for now. public contextableQuery: Contextable; - private proto = dataform.DataPreparation.create(); + private proto = dataform.DataPreparation.create({ + dynamicVars: [], + }); constructor( session?: Session, @@ -424,6 +426,11 @@ export class DataPreparation extends ActionBuilder { } return columnName; } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } export class DataPreparationContext implements ITableContext { @@ -454,8 +461,9 @@ export class DataPreparationContext implements ITableContext { return this.resolve(ref); } - public param(paramName: string): string { - return "" + public dynamicVar(varName: string): string { + this.dataPreparation.addInputDynamicVar(varName); + return `\{${varName}\}`; } public resolve(ref: Resolvable | string[], ...rest: string[]) { diff --git a/core/actions/declaration.ts b/core/actions/declaration.ts index f2ef9553a..db59aa2c8 100644 --- a/core/actions/declaration.ts +++ b/core/actions/declaration.ts @@ -66,7 +66,9 @@ export class Declaration extends ActionBuilder { /** * @hidden Stores the generated proto for the compiled graph. */ - private proto = dataform.Declaration.create(); + private proto = dataform.Declaration.create({ + dynamicVars: [], + }); /** @hidden */ constructor(session?: Session, unverifiedConfig?: any, filename?: string) { diff --git a/core/actions/incremental_table.ts b/core/actions/incremental_table.ts index 4722eabb0..053c627e0 100644 --- a/core/actions/incremental_table.ts +++ b/core/actions/incremental_table.ts @@ -88,7 +88,8 @@ export class IncrementalTable extends ActionBuilder { type: "incremental", enumType: dataform.TableType.INCREMENTAL, disabled: false, - tags: [] + tags: [], + dynamicVars: [], }); /** @hidden */ @@ -490,7 +491,7 @@ export class IncrementalTable extends ActionBuilder { /** @hidden */ public compile() { - const context = new IncrementalTableContext(this); + const context = new IncrementalTableContext(this);f const incrementalContext = new IncrementalTableContext(this, true); this.proto.query = context.apply(this.contextableQuery); @@ -647,6 +648,11 @@ export class IncrementalTable extends ActionBuilder { throw new Error(`OnSchemaChange value "${onSchemaChange}" is not supported`); } } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } /** @@ -663,8 +669,9 @@ export class IncrementalTableContext implements ITableContext { return this.incrementalTable.session.finalizeName(this.incrementalTable.getTarget().name); } - public param(paramName: string): string { - return "" + public dynamicVar(varName: string): string { + this.incrementalTable.addInputDynamicVar(varName); + return `\{${varName}\}`; } public ref(ref: Resolvable | string[], ...rest: string[]): string { diff --git a/core/actions/notebook.ts b/core/actions/notebook.ts index 082f342b9..2837c0bf5 100644 --- a/core/actions/notebook.ts +++ b/core/actions/notebook.ts @@ -58,7 +58,9 @@ export class Notebook extends ActionBuilder { /** * @hidden Stores the generated proto for the compiled graph. */ - private proto = dataform.Notebook.create(); + private proto = dataform.Notebook.create({ + dynamicVars: [], + }); /** @hidden */ constructor(session?: Session, unverifiedConfig?: any, configPath?: string) { diff --git a/core/actions/operation.ts b/core/actions/operation.ts index 309d2e811..09de864df 100644 --- a/core/actions/operation.ts +++ b/core/actions/operation.ts @@ -85,7 +85,9 @@ export class Operation extends ActionBuilder { /** * @hidden Stores the generated proto for the compiled graph. */ - private proto = dataform.Operation.create(); + private proto = dataform.Operation.create({ + dynamicVars: [], + }); /** @hidden We delay contextification until the final compile step, so hold these here for now. */ private contextableQueries: Contextable; @@ -395,6 +397,11 @@ export class Operation extends ActionBuilder { VerifyProtoErrorBehaviour.SHOW_DOCS_LINK ); } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } /** @@ -425,8 +432,9 @@ export class OperationContext implements IActionContext { return this.resolve(ref); } - public param(paramName: string): string { - return "" + public dynamicVar(varName: string): string { + this.operation.addInputDynamicVar(varName); + return `\{${varName}\}`; } public resolve(ref: Resolvable | string[], ...rest: string[]) { diff --git a/core/actions/table.ts b/core/actions/table.ts index 904e4581f..15268394d 100644 --- a/core/actions/table.ts +++ b/core/actions/table.ts @@ -94,7 +94,7 @@ export class Table extends ActionBuilder { enumType: dataform.TableType.TABLE, disabled: false, tags: [], - inputParams: [], + dynamicVars: [], }); /** @hidden */ @@ -596,6 +596,11 @@ export class Table extends ActionBuilder { return config; } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } /** @@ -612,8 +617,9 @@ export class TableContext implements ITableContext { return this.table.session.finalizeName(this.table.getTarget().name); } - public param(paramName: string): string { - return "" + public dynamicVar(varName: string): string { + this.table.addInputDynamicVar(varName); + return `\{${varName}\}`; } public ref(ref: Resolvable | string[], ...rest: string[]): string { diff --git a/core/actions/test.ts b/core/actions/test.ts index d1035d8b4..4eae7ddf0 100644 --- a/core/actions/test.ts +++ b/core/actions/test.ts @@ -80,7 +80,9 @@ export class Test extends ActionBuilder { /** * @hidden Stores the generated proto for the compiled graph. */ - private proto = dataform.Test.create(); + private proto = dataform.Test.create({ + dynamicVars: [], + }); /** @hidden */ constructor(session?: Session, config?: ITestConfig) { @@ -192,6 +194,11 @@ export class Test extends ActionBuilder { VerifyProtoErrorBehaviour.SUGGEST_REPORTING_TO_DATAFORM_TEAM ); } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } /** @hidden */ @@ -222,8 +229,9 @@ class RefReplacingContext implements ITableContext { return this.resolve(ref, ...rest); } - public param(paramName: string): string { - return "" + public dynamicVar(varName: string): string { + this.testContext.addInputDynamicVar(varName); + return `\{${varName}\}`; } public resolve(ref: Resolvable | string[], ...rest: string[]) { diff --git a/core/actions/view.ts b/core/actions/view.ts index f8e87eebc..a8c416cc4 100644 --- a/core/actions/view.ts +++ b/core/actions/view.ts @@ -107,7 +107,7 @@ export class View extends ActionBuilder { enumType: dataform.TableType.VIEW, disabled: false, tags: [], - inputParams: [], + dynamicVars: [], }); /** @hidden */ @@ -487,9 +487,9 @@ export class View extends ActionBuilder { return dataform.Target.create(this.proto.target); } - public addInputParam(paramName: string) { - if (!this.proto.inputParams.includes(paramName)) { - this.proto.inputParams.push(paramName); + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); } } @@ -657,9 +657,9 @@ export class ViewContext implements ITableContext { return this.resolve(ref); } - public param(paramName: string): string { - this.view.addInputParam(paramName); - return `\{${paramName}\}`; + public dynamicVar(varName: string): string { + this.view.addInputDynamicVar(varName); + return `\{${varName}\}`; } public resolve(ref: Resolvable | string[], ...rest: string[]) { From 45e58e67390aa21225657d721309e3fa0d339c92 Mon Sep 17 00:00:00 2001 From: karo Date: Tue, 2 Sep 2025 14:59:53 +0200 Subject: [PATCH 3/5] updated rest of files to match {dynamicVar} naming convention --- core/compilers.ts | 2 +- core/contextables.ts | 2 +- core/main_test.ts | 6 +++--- core/session.ts | 6 +++--- protos/core.proto | 16 ++++++++-------- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/core/compilers.ts b/core/compilers.ts index 1f8416b99..b8883eb52 100644 --- a/core/compilers.ts +++ b/core/compilers.ts @@ -12,7 +12,7 @@ const CONTEXT_FUNCTIONS = [ "incremental", "schema", "database", - "param", + "dynamicVar", ] .map(name => `const ${name} = ctx.${name} ? ctx.${name}.bind(ctx) : undefined;`) .join("\n"); diff --git a/core/contextables.ts b/core/contextables.ts index 5c16d11b5..78dcf6131 100644 --- a/core/contextables.ts +++ b/core/contextables.ts @@ -68,7 +68,7 @@ export interface IActionContext { */ database: () => string; - param: (paramName: string) => string; + dynamicVar: (varName: string) => string; } /** diff --git a/core/main_test.ts b/core/main_test.ts index fbc681bec..381dec2f4 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -52,7 +52,7 @@ suite("@dataform/core", ({ afterEach }) => { const tmpDirFixture = new TmpDirFixture(afterEach); suite("session", () => { - test("param resolved correctly", () => { + test("dynamic variable resolved correctly", () => { const projectDir = tmpDirFixture.createNewTmpDir(); fs.writeFileSync( path.join(projectDir, "workflow_settings.yaml"), @@ -63,7 +63,7 @@ suite("@dataform/core", ({ afterEach }) => { path.join(projectDir, "definitions/view.sqlx"), ` config { type: "view" } -SELECT 1 AS col WHERE \${param("myvar")} == 1` +SELECT 1 AS col WHERE \${dynamicVar("myvar")} == 1` ); const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); @@ -81,7 +81,7 @@ SELECT 1 AS col WHERE \${param("myvar")} == 1` fileName: "definitions/view.sqlx", hermeticity: "NON_HERMETIC", query: "\n\nSELECT 1 AS col WHERE {myvar} == 1", - inputParams: ["myvar"], + dynamicVars: ["myvar"], target: { database: "defaultProject", name: "view", diff --git a/core/session.ts b/core/session.ts index 964fb1153..5b1d92af8 100644 --- a/core/session.ts +++ b/core/session.ts @@ -58,7 +58,7 @@ export class Session { public graphErrors: dataform.IGraphErrors; - private inputParams: Set = new Set(); + private dynamicVars: Set = new Set(); constructor( rootDir?: string, @@ -469,7 +469,7 @@ export class Session { graphErrors: this.graphErrors, dataformCoreVersion, targets: this.actions.map(action => action.getTarget()), - inputParams: Array.from(this.inputParams).sort(), + dynamicVars: Array.from(this.dynamicVars).sort(), }); this.fullyQualifyDependencies( @@ -548,7 +548,7 @@ export class Session { actions.forEach(action => { try { const compiledChunk = action.compile(); - compiledChunk.inputParams.forEach(param => this.inputParams.add(param)); + compiledChunk.dynamicVars.forEach(dynamicVar => this.dynamicVars.add(dynamicVar)); compiledChunks.push(compiledChunk as any); } catch (e) { this.compileError(e, action.getFileName(), action.getTarget()); diff --git a/protos/core.proto b/protos/core.proto index 32c93c972..c431ff2e6 100644 --- a/protos/core.proto +++ b/protos/core.proto @@ -165,7 +165,7 @@ message Table { // Generated. string file_name = 18; - repeated string input_params = 38; + repeated string dynamic_vars = 38; reserved 1, 2, 7, 12, 16; } @@ -188,7 +188,7 @@ message Operation { // Generated. string file_name = 7; - repeated string input_params = 15; + repeated string dynamic_vars = 15; reserved 1, 2, 4, 5; } @@ -214,7 +214,7 @@ message Assertion { // Generated. string file_name = 7; - repeated string input_params = 16; + repeated string dynamic_vars = 16; reserved 1, 2, 4, 5, 6; } @@ -231,7 +231,7 @@ message Declaration { ActionDescriptor action_descriptor = 3; - repeated string input_params = 6; + repeated string dynamic_vars = 6; // Generated. string file_name = 4; @@ -243,7 +243,7 @@ message Test { string test_query = 2; string expected_output_query = 3; - repeated string input_params = 6; + repeated string dynamic_vars = 6; // Generated. string file_name = 4; @@ -264,7 +264,7 @@ message Notebook { string notebook_contents = 7; - repeated string input_params = 8; + repeated string dynamic_vars = 8; } message NotebookRuntimeOptions { @@ -310,7 +310,7 @@ message DataPreparation { LoadConfiguration load = 16; - repeated string input_params = 14; + repeated string dynamic_vars = 14; reserved 7, 10; } @@ -352,7 +352,7 @@ message CompiledGraph { reserved 5, 6; - repeated string input_params = 14; + repeated string dynamic_vars = 14; } message CoreExecutionRequest { From cb1a0efd0ae8e7708c7ca58859690e14be504653 Mon Sep 17 00:00:00 2001 From: karo Date: Thu, 4 Sep 2025 10:27:21 +0200 Subject: [PATCH 4/5] draft to convert default values from workflow_settings.ts to protos/core.proto - ProjectConfig --- core/workflow_settings.ts | 4 ++++ protos/core.proto | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/core/workflow_settings.ts b/core/workflow_settings.ts index f182d19c8..36de631b0 100644 --- a/core/workflow_settings.ts +++ b/core/workflow_settings.ts @@ -133,6 +133,10 @@ export function workflowSettingsAsProjectConfig( workflowSettings.defaultNotebookRuntimeOptions.runtimeTemplateName; } } + if (workflowSettings.dynamicVars) { + // logic of converting dynamicVars from WorkflowSettings to ProjectConfig + // projectConfig.dynamicVars = workflowSettings.dynamicVars; + } projectConfig.warehouse = "bigquery"; return projectConfig; } diff --git a/protos/core.proto b/protos/core.proto index c431ff2e6..0f3002c4a 100644 --- a/protos/core.proto +++ b/protos/core.proto @@ -18,6 +18,18 @@ message ProjectConfig { string assertion_schema = 5; map vars = 14; + enum type { + STRING = 1, + INT64 = 2, + BOOL = 3, + DATE = 4, + DOUBLE = 5, + } + message dynamicVar { + type var_type = 1; + string var_value = 2; + } + map dynamic_vars = 19 string database_suffix = 15; string schema_suffix = 7; From e4bfdc6a30d869ae94e7307289ac98e1c037f9a2 Mon Sep 17 00:00:00 2001 From: karo Date: Thu, 4 Sep 2025 16:01:12 +0200 Subject: [PATCH 5/5] added <> --- core/session.ts | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/core/session.ts b/core/session.ts index 5b1d92af8..77c6f93c0 100644 --- a/core/session.ts +++ b/core/session.ts @@ -541,7 +541,16 @@ export class Session { private getTablePrefixWithUnderscore() { return !!this.projectConfig.tablePrefix ? `${this.projectConfig.tablePrefix}_` : ""; } - + // function to check, if variable was provided in workflow_settings.yaml by user + private static isUserProvidedDynamicVar( + projectConfig: dataform.ProjectConfig, + varName: string + ): boolean { + return ( + projectConfig.vars !== undefined && + Object.prototype.hasOwnProperty.call(projectConfig.dynamicVars, varName) + ); + } private compileGraphChunk(actions: Array): T[] { const compiledChunks: T[] = []; @@ -549,12 +558,22 @@ export class Session { try { const compiledChunk = action.compile(); compiledChunk.dynamicVars.forEach(dynamicVar => this.dynamicVars.add(dynamicVar)); + compiledChunk.dynamicVars.forEach(dynamicVar => { + if (!Session.isUserProvidedDynamicVar(this.canonicalProjectConfig, dynamicVar)) { + this.compileError( + new Error( + `The dynamic variable '${dynamicVar}' used in '${action.getFileName()}' is not defined in the workflow settings.` + ), + action.getFileName(), + action.getTarget() + ); + } + }); compiledChunks.push(compiledChunk as any); } catch (e) { this.compileError(e, action.getFileName(), action.getTarget()); } }); - return compiledChunks; }