Preserve allow_duplicate_names field in the pipeline resource (#3274)

## Changes
<!-- Brief summary of your changes that is easy to understand -->
- Fixed pipeline resource conversion to preserve `allow_duplicate_names`
configuration field
- Updated schema and codegen to reflect the `allow_duplicate_names`
field

## Why
<!-- Why are these changes needed? Provide the context that the reviewer
might be missing.
For example, were there any decisions behind the change that are not
reflected in the code itself? -->
The `allow_duplicate_names` field was being dropped from pipeline
resources during conversion, preventing users from configuring pipelines
that allow duplicate names. This change ensures the field is properly
preserved and passed through to the Terraform Provider

## Tests
<!-- How have you tested the changes? -->
- Added a new acceptance test

<!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->

Author: anton-107

NEXT_CHANGELOG.md

@@ -9,5 +9,6 @@
 ### CLI
 
 ### Bundles
+- Fixed an issue where `allow_duplicate_names` field on the pipeline definition was ignored by the bundle ([#3274](https://github.com/databricks/cli/pull/3274))
 
 ### API Changes

acceptance/bundle/deploy/pipeline/allow-duplicate-names/databricks.yml.tmpl

@@ -0,0 +1,11 @@
+bundle:
+  name: acc-bundle-deploy-pipeline-duplicate-names-$UNIQUE_NAME
+
+resources:
+  pipelines:
+    pipeline_one:
+      name: test-pipeline-same-name-$UNIQUE_NAME
+      allow_duplicate_names: true
+      libraries:
+        - file:
+            path: "./foo.py"

acceptance/bundle/deploy/pipeline/allow-duplicate-names/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = true
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/deploy/pipeline/allow-duplicate-names/output.txt

@@ -0,0 +1,54 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> print_requests
+{
+  "body": {
+    "libraries": [
+      {
+        "file": {
+          "path": "/some-script.py"
+        }
+      }
+    ],
+    "name": "test-pipeline-same-name-[UNIQUE_NAME]"
+  },
+  "method": "POST",
+  "path": "/api/2.0/pipelines"
+}
+{
+  "body": {
+    "allow_duplicate_names": true,
+    "channel": "CURRENT",
+    "deployment": {
+      "kind": "BUNDLE",
+      "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/state/metadata.json"
+    },
+    "edition": "ADVANCED",
+    "libraries": [
+      {
+        "file": {
+          "path": "/Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default/files/foo.py"
+        }
+      }
+    ],
+    "name": "test-pipeline-same-name-[UNIQUE_NAME]"
+  },
+  "method": "POST",
+  "path": "/api/2.0/pipelines"
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete pipeline pipeline_one
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/acc-bundle-deploy-pipeline-duplicate-names-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!
+
+>>> [CLI] pipelines delete [UUID]

acceptance/bundle/deploy/pipeline/allow-duplicate-names/pipeline.json.tmpl

@@ -0,0 +1,10 @@
+{
+    "name": "test-pipeline-same-name-$UNIQUE_NAME",
+    "libraries": [
+        {
+            "file": {
+                "path": "/some-script.py"
+            }
+        }
+    ]
+}

acceptance/bundle/deploy/pipeline/allow-duplicate-names/script

@@ -0,0 +1,22 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+envsubst < pipeline.json.tmpl > pipeline.json
+touch foo.py
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve
+  trace $CLI pipelines delete ${PIPELINE_ID}
+  rm out.requests.txt
+}
+trap cleanup EXIT
+
+# Create a pre-existing pipeline:
+PIPELINE_ID=$($CLI pipelines create --json @pipeline.json | jq -r .pipeline_id)
+export PIPELINE_ID
+
+# Deploy the bundle that has a pipeline with the same name:
+trace $CLI bundle deploy
+
+print_requests() {
+    jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt
+}
+trace print_requests

acceptance/bundle/deploy/pipeline/allow-duplicate-names/test.toml

@@ -0,0 +1,3 @@
+Cloud = true
+Ignore = ["foo.py","pipeline.json"]
+RecordRequests = true

bundle/deploy/terraform/tfdyn/convert_pipeline.go

@@ -21,7 +21,7 @@ func convertPipelineResource(ctx context.Context, vin dyn.Value) (dyn.Value, err
 		return dyn.InvalidValue, err
 	}
 
-	vout, err = dyn.DropKeys(vout, []string{"allow_duplicate_names", "dry_run"})
+	vout, err = dyn.DropKeys(vout, []string{"dry_run"})
 	if err != nil {
 		return dyn.InvalidValue, err
 	}

bundle/deploy/terraform/tfdyn/convert_pipeline_test.go

@@ -116,6 +116,7 @@ func TestConvertPipeline(t *testing.T) {
 				},
 			},
 		},
+		"allow_duplicate_names": true,
 		"cluster": []any{
 			map[string]any{
 				"label":       "default",

bundle/internal/schema/main.go

@@ -115,7 +115,6 @@ func removePipelineFields(typ reflect.Type, s jsonschema.Schema) jsonschema.Sche
 		// Even though DABs supports this field, TF provider does not. Thus, we
 		// should not expose it to the user.
 		delete(s.Properties, "dry_run")
-		delete(s.Properties, "allow_duplicate_names")
 
 		// These fields are only meant to be set by the DABs client (ie the CLI)
 		// and thus should not be exposed to the user. These are used to annotate