diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 7ff5519eb9..7cd636e33d 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,5 +9,6 @@ ### CLI ### Bundles +- Fixed an issue where `allow_duplicate_names` field on the pipeline definition was ignored by the bundle ([#3274](https://github.com/databricks/cli/pull/3274)) ### API Changes diff --git a/acceptance/bundle/deploy/pipeline/allow-duplicate-names/databricks.yml.tmpl b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/databricks.yml.tmpl new file mode 100644 index 0000000000..bdab53c9b5 --- /dev/null +++ b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: acc-bundle-deploy-pipeline-duplicate-names-$UNIQUE_NAME + +resources: + pipelines: + pipeline_one: + name: test-pipeline-same-name-$UNIQUE_NAME + allow_duplicate_names: true + libraries: + - file: + path: "./foo.py" diff --git a/acceptance/bundle/deploy/pipeline/allow-duplicate-names/out.test.toml b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/out.test.toml new file mode 100644 index 0000000000..58ed268930 --- /dev/null +++ b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/deploy/pipeline/allow-duplicate-names/output.txt b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/output.txt new file mode 100644 index 0000000000..4e3ff9f204 --- /dev/null +++ b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/output.txt @@ -0,0 +1,54 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests +{ + "body": { + "libraries": [ + { + "file": { + "path": "/some-script.py" + } + } + ], + "name": "test-pipeline-same-name-[UNIQUE_NAME]" + }, + "method": "POST", + "path": "/api/2.0/pipelines" +} +{ + "body": { + "allow_duplicate_names": true, + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-same-name-[UNIQUE_NAME]" + }, + "method": "POST", + "path": "/api/2.0/pipelines" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete pipeline pipeline_one + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! + +>>> [CLI] pipelines delete [UUID] diff --git a/acceptance/bundle/deploy/pipeline/allow-duplicate-names/pipeline.json.tmpl b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/pipeline.json.tmpl new file mode 100644 index 0000000000..f7d584ed0e --- /dev/null +++ b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/pipeline.json.tmpl @@ -0,0 +1,10 @@ +{ + "name": "test-pipeline-same-name-$UNIQUE_NAME", + "libraries": [ + { + "file": { + "path": "/some-script.py" + } + } + ] +} diff --git a/acceptance/bundle/deploy/pipeline/allow-duplicate-names/script b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/script new file mode 100644 index 0000000000..6db327deec --- /dev/null +++ b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/script @@ -0,0 +1,22 @@ +envsubst < databricks.yml.tmpl > databricks.yml +envsubst < pipeline.json.tmpl > pipeline.json +touch foo.py + +cleanup() { + trace $CLI bundle destroy --auto-approve + trace $CLI pipelines delete ${PIPELINE_ID} + rm out.requests.txt +} +trap cleanup EXIT + +# Create a pre-existing pipeline: +PIPELINE_ID=$($CLI pipelines create --json @pipeline.json | jq -r .pipeline_id) +export PIPELINE_ID + +# Deploy the bundle that has a pipeline with the same name: +trace $CLI bundle deploy + +print_requests() { + jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt +} +trace print_requests diff --git a/acceptance/bundle/deploy/pipeline/allow-duplicate-names/test.toml b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/test.toml new file mode 100644 index 0000000000..c8db125e67 --- /dev/null +++ b/acceptance/bundle/deploy/pipeline/allow-duplicate-names/test.toml @@ -0,0 +1,3 @@ +Cloud = true +Ignore = ["foo.py","pipeline.json"] +RecordRequests = true diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline.go b/bundle/deploy/terraform/tfdyn/convert_pipeline.go index 53a9868649..d2df60fa28 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline.go @@ -21,7 +21,7 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err return dyn.InvalidValue, err } - vout, err = dyn.DropKeys(vout, []string{"allow_duplicate_names", "dry_run"}) + vout, err = dyn.DropKeys(vout, []string{"dry_run"}) if err != nil { return dyn.InvalidValue, err } diff --git a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go index 45eb0b902e..ed6bd70a08 100644 --- a/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_pipeline_test.go @@ -116,6 +116,7 @@ func TestConvertPipeline(t *testing.T) { }, }, }, + "allow_duplicate_names": true, "cluster": []any{ map[string]any{ "label": "default", diff --git a/bundle/internal/schema/main.go b/bundle/internal/schema/main.go index d78f57b8a1..1f3c0c03ca 100644 --- a/bundle/internal/schema/main.go +++ b/bundle/internal/schema/main.go @@ -115,7 +115,6 @@ func removePipelineFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Sche // Even though DABs supports this field, TF provider does not. Thus, we // should not expose it to the user. delete(s.Properties, "dry_run") - delete(s.Properties, "allow_duplicate_names") // These fields are only meant to be set by the DABs client (ie the CLI) // and thus should not be exposed to the user. These are used to annotate diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 4ebfbe4b6b..592ffccbfd 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -990,6 +990,10 @@ { "type": "object", "properties": { + "allow_duplicate_names": { + "description": "If false, deployment will fail if name conflicts with that of another pipeline.", + "$ref": "#/$defs/bool" + }, "budget_policy_id": { "description": "Budget policy of this pipeline.", "$ref": "#/$defs/string", diff --git a/experimental/python/databricks/bundles/pipelines/_models/pipeline.py b/experimental/python/databricks/bundles/pipelines/_models/pipeline.py index a700046063..6ca71f4dae 100644 --- a/experimental/python/databricks/bundles/pipelines/_models/pipeline.py +++ b/experimental/python/databricks/bundles/pipelines/_models/pipeline.py @@ -59,6 +59,11 @@ class Pipeline(Resource): """""" + allow_duplicate_names: VariableOrOptional[bool] = None + """ + If false, deployment will fail if name conflicts with that of another pipeline. + """ + budget_policy_id: VariableOrOptional[str] = None """ :meta private: [EXPERIMENTAL] @@ -212,6 +217,11 @@ def as_dict(self) -> "PipelineDict": class PipelineDict(TypedDict, total=False): """""" + allow_duplicate_names: VariableOrOptional[bool] + """ + If false, deployment will fail if name conflicts with that of another pipeline. + """ + budget_policy_id: VariableOrOptional[str] """ :meta private: [EXPERIMENTAL]