diff --git a/core/actions/assertion.ts b/core/actions/assertion.ts index 353e394b4..48226cf9f 100644 --- a/core/actions/assertion.ts +++ b/core/actions/assertion.ts @@ -85,7 +85,9 @@ export class Assertion extends ActionBuilder { /** * @hidden Stores the generated proto for the compiled graph. */ - private proto = dataform.Assertion.create(); + private proto = dataform.Assertion.create({ + dynamicVars: [], + }); /** @hidden We delay contextification until the final compile step, so hold these here for now. */ private contextableQuery: AContextable; @@ -334,6 +336,11 @@ export class Assertion extends ActionBuilder { VerifyProtoErrorBehaviour.SHOW_DOCS_LINK ); } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } /** @@ -364,6 +371,11 @@ export class AssertionContext implements IActionContext { return this.resolve(ref); } + public dynamicVar(varName: string): string { + this.assertion.addInputDynamicVar(varName); + return `\{${varName}\}`; + } + public resolve(ref: Resolvable | string[], ...rest: string[]) { return this.assertion.session.resolve(ref, ...rest); } diff --git a/core/actions/data_preparation.ts b/core/actions/data_preparation.ts index c5bb03cab..feaa16075 100644 --- a/core/actions/data_preparation.ts +++ b/core/actions/data_preparation.ts @@ -26,7 +26,9 @@ export class DataPreparation extends ActionBuilder { // We delay contextification until the final compile step, so hold these here for now. public contextableQuery: Contextable; - private proto = dataform.DataPreparation.create(); + private proto = dataform.DataPreparation.create({ + dynamicVars: [], + }); constructor( session?: Session, @@ -424,6 +426,11 @@ export class DataPreparation extends ActionBuilder { } return columnName; } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } export class DataPreparationContext implements ITableContext { @@ -454,6 +461,11 @@ export class DataPreparationContext implements ITableContext { return this.resolve(ref); } + public dynamicVar(varName: string): string { + this.dataPreparation.addInputDynamicVar(varName); + return `\{${varName}\}`; + } + public resolve(ref: Resolvable | string[], ...rest: string[]) { return this.dataPreparation.session.resolve(ref, ...rest); } diff --git a/core/actions/declaration.ts b/core/actions/declaration.ts index f2ef9553a..db59aa2c8 100644 --- a/core/actions/declaration.ts +++ b/core/actions/declaration.ts @@ -66,7 +66,9 @@ export class Declaration extends ActionBuilder { /** * @hidden Stores the generated proto for the compiled graph. */ - private proto = dataform.Declaration.create(); + private proto = dataform.Declaration.create({ + dynamicVars: [], + }); /** @hidden */ constructor(session?: Session, unverifiedConfig?: any, filename?: string) { diff --git a/core/actions/incremental_table.ts b/core/actions/incremental_table.ts index 605df68ab..053c627e0 100644 --- a/core/actions/incremental_table.ts +++ b/core/actions/incremental_table.ts @@ -88,7 +88,8 @@ export class IncrementalTable extends ActionBuilder { type: "incremental", enumType: dataform.TableType.INCREMENTAL, disabled: false, - tags: [] + tags: [], + dynamicVars: [], }); /** @hidden */ @@ -490,7 +491,7 @@ export class IncrementalTable extends ActionBuilder { /** @hidden */ public compile() { - const context = new IncrementalTableContext(this); + const context = new IncrementalTableContext(this);f const incrementalContext = new IncrementalTableContext(this, true); this.proto.query = context.apply(this.contextableQuery); @@ -647,6 +648,11 @@ export class IncrementalTable extends ActionBuilder { throw new Error(`OnSchemaChange value "${onSchemaChange}" is not supported`); } } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } /** @@ -663,6 +669,11 @@ export class IncrementalTableContext implements ITableContext { return this.incrementalTable.session.finalizeName(this.incrementalTable.getTarget().name); } + public dynamicVar(varName: string): string { + this.incrementalTable.addInputDynamicVar(varName); + return `\{${varName}\}`; + } + public ref(ref: Resolvable | string[], ...rest: string[]): string { ref = toResolvable(ref, rest); if (!resolvableAsTarget(ref)) { diff --git a/core/actions/notebook.ts b/core/actions/notebook.ts index 082f342b9..2837c0bf5 100644 --- a/core/actions/notebook.ts +++ b/core/actions/notebook.ts @@ -58,7 +58,9 @@ export class Notebook extends ActionBuilder { /** * @hidden Stores the generated proto for the compiled graph. */ - private proto = dataform.Notebook.create(); + private proto = dataform.Notebook.create({ + dynamicVars: [], + }); /** @hidden */ constructor(session?: Session, unverifiedConfig?: any, configPath?: string) { diff --git a/core/actions/operation.ts b/core/actions/operation.ts index a03354c9e..09de864df 100644 --- a/core/actions/operation.ts +++ b/core/actions/operation.ts @@ -85,7 +85,9 @@ export class Operation extends ActionBuilder { /** * @hidden Stores the generated proto for the compiled graph. */ - private proto = dataform.Operation.create(); + private proto = dataform.Operation.create({ + dynamicVars: [], + }); /** @hidden We delay contextification until the final compile step, so hold these here for now. */ private contextableQueries: Contextable; @@ -395,6 +397,11 @@ export class Operation extends ActionBuilder { VerifyProtoErrorBehaviour.SHOW_DOCS_LINK ); } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } /** @@ -425,6 +432,11 @@ export class OperationContext implements IActionContext { return this.resolve(ref); } + public dynamicVar(varName: string): string { + this.operation.addInputDynamicVar(varName); + return `\{${varName}\}`; + } + public resolve(ref: Resolvable | string[], ...rest: string[]) { return this.operation.session.resolve(ref, ...rest); } diff --git a/core/actions/table.ts b/core/actions/table.ts index 3f8f77117..15268394d 100644 --- a/core/actions/table.ts +++ b/core/actions/table.ts @@ -93,7 +93,8 @@ export class Table extends ActionBuilder { type: "table", enumType: dataform.TableType.TABLE, disabled: false, - tags: [] + tags: [], + dynamicVars: [], }); /** @hidden */ @@ -595,6 +596,11 @@ export class Table extends ActionBuilder { return config; } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } /** @@ -611,6 +617,11 @@ export class TableContext implements ITableContext { return this.table.session.finalizeName(this.table.getTarget().name); } + public dynamicVar(varName: string): string { + this.table.addInputDynamicVar(varName); + return `\{${varName}\}`; + } + public ref(ref: Resolvable | string[], ...rest: string[]): string { ref = toResolvable(ref, rest); if (!resolvableAsTarget(ref)) { diff --git a/core/actions/test.ts b/core/actions/test.ts index df781c9c9..4eae7ddf0 100644 --- a/core/actions/test.ts +++ b/core/actions/test.ts @@ -80,7 +80,9 @@ export class Test extends ActionBuilder { /** * @hidden Stores the generated proto for the compiled graph. */ - private proto = dataform.Test.create(); + private proto = dataform.Test.create({ + dynamicVars: [], + }); /** @hidden */ constructor(session?: Session, config?: ITestConfig) { @@ -192,6 +194,11 @@ export class Test extends ActionBuilder { VerifyProtoErrorBehaviour.SUGGEST_REPORTING_TO_DATAFORM_TEAM ); } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } } /** @hidden */ @@ -221,7 +228,12 @@ class RefReplacingContext implements ITableContext { public ref(ref: Resolvable | string[], ...rest: string[]) { return this.resolve(ref, ...rest); } - + + public dynamicVar(varName: string): string { + this.testContext.addInputDynamicVar(varName); + return `\{${varName}\}`; + } + public resolve(ref: Resolvable | string[], ...rest: string[]) { const target = resolvableAsTarget(toResolvable(ref, rest)); if (!this.testContext.test.contextableInputs.has(targetStringifier.stringify(target))) { diff --git a/core/actions/view.ts b/core/actions/view.ts index 50ac1018f..a8c416cc4 100644 --- a/core/actions/view.ts +++ b/core/actions/view.ts @@ -106,7 +106,8 @@ export class View extends ActionBuilder { type: "view", enumType: dataform.TableType.VIEW, disabled: false, - tags: [] + tags: [], + dynamicVars: [], }); /** @hidden */ @@ -486,6 +487,12 @@ export class View extends ActionBuilder { return dataform.Target.create(this.proto.target); } + public addInputDynamicVar(varName: string) { + if (!this.proto.dynamicVars.includes(varName)) { + this.proto.dynamicVars.push(varName); + } + } + /** @hidden */ public compile() { const context = new ViewContext(this); @@ -650,6 +657,11 @@ export class ViewContext implements ITableContext { return this.resolve(ref); } + public dynamicVar(varName: string): string { + this.view.addInputDynamicVar(varName); + return `\{${varName}\}`; + } + public resolve(ref: Resolvable | string[], ...rest: string[]) { return this.view.session.resolve(ref, ...rest); } diff --git a/core/compilers.ts b/core/compilers.ts index 69c8f7704..b8883eb52 100644 --- a/core/compilers.ts +++ b/core/compilers.ts @@ -11,7 +11,8 @@ const CONTEXT_FUNCTIONS = [ "when", "incremental", "schema", - "database" + "database", + "dynamicVar", ] .map(name => `const ${name} = ctx.${name} ? ctx.${name}.bind(ctx) : undefined;`) .join("\n"); diff --git a/core/contextables.ts b/core/contextables.ts index 7ad6d9bbd..78dcf6131 100644 --- a/core/contextables.ts +++ b/core/contextables.ts @@ -67,6 +67,8 @@ export interface IActionContext { * Returns the database of this dataset, if applicable. */ database: () => string; + + dynamicVar: (varName: string) => string; } /** diff --git a/core/main_test.ts b/core/main_test.ts index e77602898..381dec2f4 100644 --- a/core/main_test.ts +++ b/core/main_test.ts @@ -52,6 +52,46 @@ suite("@dataform/core", ({ afterEach }) => { const tmpDirFixture = new TmpDirFixture(afterEach); suite("session", () => { + test("dynamic variable resolved correctly", () => { + const projectDir = tmpDirFixture.createNewTmpDir(); + fs.writeFileSync( + path.join(projectDir, "workflow_settings.yaml"), + VALID_WORKFLOW_SETTINGS_YAML + ); + fs.mkdirSync(path.join(projectDir, "definitions")); + fs.writeFileSync( + path.join(projectDir, "definitions/view.sqlx"), + ` +config { type: "view" } +SELECT 1 AS col WHERE \${dynamicVar("myvar")} == 1` + ); + + const result = runMainInVm(coreExecutionRequestFromPath(projectDir)); + + expect(result.compile.compiledGraph.graphErrors.compilationErrors).deep.equals([]); + expect(asPlainObject(result.compile.compiledGraph.tables)).deep.equals([ + { + canonicalTarget: { + database: "defaultProject", + name: "view", + schema: "defaultDataset" + }, + disabled: false, + enumType: "VIEW", + fileName: "definitions/view.sqlx", + hermeticity: "NON_HERMETIC", + query: "\n\nSELECT 1 AS col WHERE {myvar} == 1", + dynamicVars: ["myvar"], + target: { + database: "defaultProject", + name: "view", + schema: "defaultDataset" + }, + type: "view", + } + ]); + }); + suite("resolve succeeds", () => { [ WorkflowSettingsTemplates.bigquery, diff --git a/core/session.ts b/core/session.ts index f8c9ce2a6..77c6f93c0 100644 --- a/core/session.ts +++ b/core/session.ts @@ -58,6 +58,8 @@ export class Session { public graphErrors: dataform.IGraphErrors; + private dynamicVars: Set = new Set(); + constructor( rootDir?: string, projectConfig?: dataform.ProjectConfig, @@ -466,7 +468,8 @@ export class Session { ), graphErrors: this.graphErrors, dataformCoreVersion, - targets: this.actions.map(action => action.getTarget()) + targets: this.actions.map(action => action.getTarget()), + dynamicVars: Array.from(this.dynamicVars).sort(), }); this.fullyQualifyDependencies( @@ -538,19 +541,39 @@ export class Session { private getTablePrefixWithUnderscore() { return !!this.projectConfig.tablePrefix ? `${this.projectConfig.tablePrefix}_` : ""; } - + // function to check, if variable was provided in workflow_settings.yaml by user + private static isUserProvidedDynamicVar( + projectConfig: dataform.ProjectConfig, + varName: string + ): boolean { + return ( + projectConfig.vars !== undefined && + Object.prototype.hasOwnProperty.call(projectConfig.dynamicVars, varName) + ); + } private compileGraphChunk(actions: Array): T[] { const compiledChunks: T[] = []; actions.forEach(action => { try { const compiledChunk = action.compile(); + compiledChunk.dynamicVars.forEach(dynamicVar => this.dynamicVars.add(dynamicVar)); + compiledChunk.dynamicVars.forEach(dynamicVar => { + if (!Session.isUserProvidedDynamicVar(this.canonicalProjectConfig, dynamicVar)) { + this.compileError( + new Error( + `The dynamic variable '${dynamicVar}' used in '${action.getFileName()}' is not defined in the workflow settings.` + ), + action.getFileName(), + action.getTarget() + ); + } + }); compiledChunks.push(compiledChunk as any); } catch (e) { this.compileError(e, action.getFileName(), action.getTarget()); } }); - return compiledChunks; } diff --git a/core/workflow_settings.ts b/core/workflow_settings.ts index f182d19c8..36de631b0 100644 --- a/core/workflow_settings.ts +++ b/core/workflow_settings.ts @@ -133,6 +133,10 @@ export function workflowSettingsAsProjectConfig( workflowSettings.defaultNotebookRuntimeOptions.runtimeTemplateName; } } + if (workflowSettings.dynamicVars) { + // logic of converting dynamicVars from WorkflowSettings to ProjectConfig + // projectConfig.dynamicVars = workflowSettings.dynamicVars; + } projectConfig.warehouse = "bigquery"; return projectConfig; } diff --git a/protos/core.proto b/protos/core.proto index 93249aee8..0f3002c4a 100644 --- a/protos/core.proto +++ b/protos/core.proto @@ -18,6 +18,18 @@ message ProjectConfig { string assertion_schema = 5; map vars = 14; + enum type { + STRING = 1, + INT64 = 2, + BOOL = 3, + DATE = 4, + DOUBLE = 5, + } + message dynamicVar { + type var_type = 1; + string var_value = 2; + } + map dynamic_vars = 19 string database_suffix = 15; string schema_suffix = 7; @@ -165,6 +177,8 @@ message Table { // Generated. string file_name = 18; + repeated string dynamic_vars = 38; + reserved 1, 2, 7, 12, 16; } @@ -186,6 +200,8 @@ message Operation { // Generated. string file_name = 7; + repeated string dynamic_vars = 15; + reserved 1, 2, 4, 5; } @@ -210,6 +226,8 @@ message Assertion { // Generated. string file_name = 7; + repeated string dynamic_vars = 16; + reserved 1, 2, 4, 5, 6; } @@ -225,6 +243,8 @@ message Declaration { ActionDescriptor action_descriptor = 3; + repeated string dynamic_vars = 6; + // Generated. string file_name = 4; } @@ -235,6 +255,8 @@ message Test { string test_query = 2; string expected_output_query = 3; + repeated string dynamic_vars = 6; + // Generated. string file_name = 4; } @@ -253,6 +275,8 @@ message Notebook { bool disabled = 6; string notebook_contents = 7; + + repeated string dynamic_vars = 8; } message NotebookRuntimeOptions { @@ -298,6 +322,8 @@ message DataPreparation { LoadConfiguration load = 16; + repeated string dynamic_vars = 14; + reserved 7, 10; } @@ -337,6 +363,8 @@ message CompiledGraph { repeated Target targets = 11; reserved 5, 6; + + repeated string dynamic_vars = 14; } message CoreExecutionRequest {