[Python] Support pipelines (#2574)

## Changes
Add support for pipelines into `experimental/python`.

Only the `name` property is supported, and the generated code will be
added as a follow-up.

## Why
It allows using pipelines in addition to jobs.

## Tests
Unit and acceptance tests

---------

Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com>

Author: kanterov

acceptance/bundle/python/mutator-ordering/script

@@ -1,4 +1,4 @@
-UV_ARGS="${UV_ARGS//\[\DATABRICKS_BUNDLES_WHEEL\]/$DATABRICKS_BUNDLES_WHEEL}"
+echo "$DATABRICKS_BUNDLES_WHEEL" > "requirements-latest.txt"
 
 # after mutators are applied, we expect to record location of the last mutator that had any effect
 

acceptance/bundle/python/pipelines-support/databricks.yml

@@ -0,0 +1,16 @@
+bundle:
+  name: my_project
+
+sync: { paths: [ ] } # don't need to copy files
+
+experimental:
+  python:
+    resources:
+      - "resources:load_resources"
+    mutators:
+      - "mutators:update_pipeline"
+
+resources:
+  pipelines:
+    my_pipeline_1:
+      name: "My Pipeline 1"

acceptance/bundle/python/pipelines-support/mutators.py

@@ -0,0 +1,11 @@
+from dataclasses import replace
+
+from databricks.bundles.core import pipeline_mutator
+from databricks.bundles.pipelines import Pipeline
+
+
+@pipeline_mutator
+def update_pipeline(pipeline: Pipeline) -> Pipeline:
+    assert isinstance(pipeline.name, str)
+
+    return replace(pipeline, name=f"{pipeline.name} (updated)")

acceptance/bundle/python/pipelines-support/output.txt

@@ -0,0 +1,34 @@
+
+>>> uv run [UV_ARGS] -q [CLI] bundle validate --output json
+{
+  "experimental": {
+    "python": {
+      "mutators": [
+        "mutators:update_pipeline"
+      ],
+      "resources": [
+        "resources:load_resources"
+      ]
+    }
+  },
+  "resources": {
+    "pipelines": {
+      "my_pipeline_1": {
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json"
+        },
+        "name": "My Pipeline 1 (updated)",
+        "permissions": []
+      },
+      "my_pipeline_2": {
+        "deployment": {
+          "kind": "BUNDLE",
+          "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json"
+        },
+        "name": "My Pipeline 2 (updated)",
+        "permissions": []
+      }
+    }
+  }
+}

acceptance/bundle/python/pipelines-support/resources.py

@@ -0,0 +1,9 @@
+from databricks.bundles.core import Resources
+
+
+def load_resources() -> Resources:
+    resources = Resources()
+
+    resources.add_pipeline("my_pipeline_2", {"name": "My Pipeline 2"})
+
+    return resources

acceptance/bundle/python/pipelines-support/script

@@ -0,0 +1,6 @@
+echo "$DATABRICKS_BUNDLES_WHEEL" > "requirements-latest.txt"
+
+trace uv run $UV_ARGS -q $CLI bundle validate --output json | \
+  jq "pick(.experimental.python, .resources)"
+
+rm -fr .databricks __pycache__

acceptance/bundle/python/pipelines-support/test.toml

@@ -0,0 +1,8 @@
+Local = true
+Cloud = false # tests don't interact with APIs
+
+[EnvMatrix]
+UV_ARGS = [
+    # pipelines are only supported in the latest version of the wheel
+    "--with-requirements requirements-latest.txt --no-cache",
+]

acceptance/bundle/python/resolve-variable/script

@@ -1,4 +1,4 @@
-UV_ARGS="${UV_ARGS//\[\DATABRICKS_BUNDLES_WHEEL\]/$DATABRICKS_BUNDLES_WHEEL}"
+echo "$DATABRICKS_BUNDLES_WHEEL" > "requirements-latest.txt"
 
 trace uv run $UV_ARGS -q $CLI bundle validate --output json | \
   jq "pick(.experimental.python, .resources)"

acceptance/bundle/python/resource-loading/script

@@ -1,4 +1,4 @@
-UV_ARGS="${UV_ARGS//\[\DATABRICKS_BUNDLES_WHEEL\]/$DATABRICKS_BUNDLES_WHEEL}"
+echo "$DATABRICKS_BUNDLES_WHEEL" > "requirements-latest.txt"
 
 # each job should record location where add_job function was called
 

acceptance/bundle/python/restricted-execution/script

@@ -1,4 +1,4 @@
-UV_ARGS="${UV_ARGS//\[\DATABRICKS_BUNDLES_WHEEL\]/$DATABRICKS_BUNDLES_WHEEL}"
+echo "$DATABRICKS_BUNDLES_WHEEL" > "requirements-latest.txt"
 
 export SOME_ENV_VAR="value_from_env"