Preserve allow_duplicate_names field in the pipeline resource

NEXT_CHANGELOG.md

@@ -9,5 +9,6 @@
 ### CLI
 
 ### Bundles
+- Fixed an issue where `allow_duplicate_names` field on the pipeline definition was ignored by the bundle ([#3274](https://github.com/databricks/cli/pull/3274))
 
 ### API Changes

acceptance/bundle/deploy/pipeline/allow-duplicate-names/databricks.yml.tmpl

@@ -0,0 +1,11 @@
+bundle:
+  name: acc-bundle-deploy-pipeline-duplicate-names-$UNIQUE_NAME
+
+resources:
+  pipelines:
+    pipeline_one:
+      name: test-pipeline-same-name-$UNIQUE_NAME
+      allow_duplicate_names: true
+      libraries:
+        - file:
+            path: "./foo.py"

acceptance/bundle/deploy/pipeline/allow-duplicate-names/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = true
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/deploy/pipeline/allow-duplicate-names/output.txt

@@ -0,0 +1,54 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests
+{
+  "body": {
+    "libraries": [
+      {
+        "file": {
+          "path": "/some-script.py"
+        }
+      }
+    ],
+    "name": "test-pipeline-same-name-[UNIQUE_NAME]"
+  },
+  "method": "POST",
+  "path": "/api/2.0/pipelines"
+}
+{
+  "body": {
+    "allow_duplicate_names": true,
+    "channel": "CURRENT",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/state/metadata.json"
+    },
+    "edition": "ADVANCED",
+    "libraries": [
+      {
+        "file": {
+          "path": "/Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/files/foo.py"
+        }
+      }
+    ],
+    "name": "test-pipeline-same-name-[UNIQUE_NAME]"
+  },
+  "method": "POST",
+  "path": "/api/2.0/pipelines"
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete pipeline pipeline_one
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!
+
+>>> [CLI] pipelines delete [UUID]

acceptance/bundle/deploy/pipeline/allow-duplicate-names/pipeline.json.tmpl

@@ -0,0 +1,10 @@
+{
+    "name": "test-pipeline-same-name-$UNIQUE_NAME",
+    "libraries": [
+        {
+            "file": {
+                "path": "/some-script.py"
+            }
+        }
+    ]
+}

acceptance/bundle/deploy/pipeline/allow-duplicate-names/script

@@ -0,0 +1,22 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+envsubst < pipeline.json.tmpl > pipeline.json
+touch foo.py
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve
+  trace $CLI pipelines delete ${PIPELINE_ID}
+  rm out.requests.txt
+}
+trap cleanup EXIT
+
+# Create a pre-existing pipeline:
+PIPELINE_ID=$($CLI pipelines create --json @pipeline.json | jq -r .pipeline_id)
+export PIPELINE_ID
+
+# Deploy the bundle that has a pipeline with the same name:
+trace $CLI bundle deploy
+
+print_requests() {
+    jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt
+}
+trace print_requests

acceptance/bundle/deploy/pipeline/allow-duplicate-names/test.toml

@@ -0,0 +1,3 @@
+Cloud = true
+Ignore = ["foo.py","pipeline.json"]
+RecordRequests = true

bundle/deploy/terraform/tfdyn/convert_pipeline.go

@@ -21,7 +21,7 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err
 		return dyn.InvalidValue, err
 	}
 
-	vout, err = dyn.DropKeys(vout, []string{"allow_duplicate_names", "dry_run"})
+	vout, err = dyn.DropKeys(vout, []string{"dry_run"})
 	if err != nil {
 		return dyn.InvalidValue, err
 	}

bundle/deploy/terraform/tfdyn/convert_pipeline_test.go

@@ -116,6 +116,7 @@ func TestConvertPipeline(t *testing.T) {
 				},
 			},
 		},
+		"allow_duplicate_names": true,
 		"cluster": []any{
 			map[string]any{
 				"label":       "default",

bundle/internal/schema/main.go

@@ -115,7 +115,6 @@ func removePipelineFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Sche
 		// Even though DABs supports this field, TF provider does not. Thus, we
 		// should not expose it to the user.
 		delete(s.Properties, "dry_run")
-		delete(s.Properties, "allow_duplicate_names")
 
 		// These fields are only meant to be set by the DABs client (ie the CLI)
 		// and thus should not be exposed to the user. These are used to annotate

experimental/python/databricks/bundles/pipelines/_models/pipeline.py

@@ -59,6 +59,11 @@
 class Pipeline(Resource):
     """"""
 
+    allow_duplicate_names: VariableOrOptional[bool] = None
+    """
+    If false, deployment will fail if name conflicts with that of another pipeline.
+    """
+
     budget_policy_id: VariableOrOptional[str] = None
     """
     :meta private: [EXPERIMENTAL]
@@ -212,6 +217,11 @@ def as_dict(self) -> "PipelineDict":
 class PipelineDict(TypedDict, total=False):
     """"""
 
+    allow_duplicate_names: VariableOrOptional[bool]
+    """
+    If false, deployment will fail if name conflicts with that of another pipeline.
+    """
+
     budget_policy_id: VariableOrOptional[str]
     """
     :meta private: [EXPERIMENTAL]