pipelines: dry-run new command (#3218)

alyssa-db · web-flow · commit 825ec29bb8b7 · 2025-07-11T14:11:57.000Z
## Changes Dry run copied from cmd/pipelines/run.go that only validates run. ## Why We want to move the `bundle run --validate-only` flag from bundle/run.go to be a separate command as `pipelines dry-run`. This allows to add more flags in the future. ## Tests Acceptance tests in follow-up: #3219
diff --git a/acceptance/pipelines/dry-run/dry-run-pipeline/databricks.yml b/acceptance/pipelines/dry-run/dry-run-pipeline/databricks.yml
@@ -0,0 +1,6 @@
+bundle:
+  name: test-pipeline-run
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline
diff --git a/acceptance/pipelines/dry-run/dry-run-pipeline/out.test.toml b/acceptance/pipelines/dry-run/dry-run-pipeline/out.test.toml
@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
diff --git a/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt b/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt
@@ -0,0 +1,18 @@
+
+>>> [PIPELINES] deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Dry running pipeline, should have validate_only set to true
+>>> [PIPELINES] dry-run my_pipeline
+Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
+
+{
+  "body": {
+    "validate_only": true
+  },
+  "method": "POST",
+  "path": "/api/2.0/pipelines/[UUID]/updates"
+}
diff --git a/acceptance/pipelines/dry-run/dry-run-pipeline/script b/acceptance/pipelines/dry-run/dry-run-pipeline/script
@@ -0,0 +1,10 @@
+print_requests() {
+    jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt
+    rm out.requests.txt
+}
+
+trace $PIPELINES deploy
+rm out.requests.txt
+title "Dry running pipeline, should have validate_only set to true"
+trace $PIPELINES dry-run my_pipeline
+print_requests
diff --git a/acceptance/pipelines/dry-run/dry-run-pipeline/test.toml b/acceptance/pipelines/dry-run/dry-run-pipeline/test.toml
@@ -0,0 +1 @@
+RecordRequests = true
diff --git a/acceptance/pipelines/dry-run/no-wait/databricks.yml b/acceptance/pipelines/dry-run/no-wait/databricks.yml
@@ -0,0 +1,6 @@
+bundle:
+  name: test-pipeline-run-flags
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-flags
diff --git a/acceptance/pipelines/dry-run/no-wait/out.test.toml b/acceptance/pipelines/dry-run/no-wait/out.test.toml
@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
diff --git a/acceptance/pipelines/dry-run/no-wait/output.txt b/acceptance/pipelines/dry-run/no-wait/output.txt
@@ -0,0 +1,12 @@
+
+>>> [PIPELINES] deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Dry running pipeline with --no-wait flag
+>>> [PIPELINES] dry-run my_pipeline --no-wait
+Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
+
+<EOL>
diff --git a/acceptance/pipelines/dry-run/no-wait/script b/acceptance/pipelines/dry-run/no-wait/script
@@ -0,0 +1,5 @@
+trace $PIPELINES deploy
+title "Dry running pipeline with --no-wait flag"
+trace $PIPELINES dry-run my_pipeline --no-wait
+# This printf is here to fix the whitespace linter error (ensures exactly one newline at end)
+printf "<EOL>\n"
diff --git a/acceptance/pipelines/dry-run/restart/databricks.yml b/acceptance/pipelines/dry-run/restart/databricks.yml
@@ -0,0 +1,6 @@
+bundle:
+  name: test-pipeline-run-flags
+resources:
+  pipelines:
+    my_pipeline:
+      name: test-pipeline-flags
diff --git a/acceptance/pipelines/dry-run/restart/out.test.toml b/acceptance/pipelines/dry-run/restart/out.test.toml
@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]
diff --git a/acceptance/pipelines/dry-run/restart/output.txt b/acceptance/pipelines/dry-run/restart/output.txt
@@ -0,0 +1,24 @@
+
+>>> [PIPELINES] deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Dry running pipeline with --restart flag, should stop the current pipeline and start a new run
+>>> [PIPELINES] dry-run my_pipeline --restart
+Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
+
+
+>>> print_requests
+{
+  "method": "POST",
+  "path": "/api/2.0/pipelines/[UUID]/stop"
+}
+{
+  "body": {
+    "validate_only": true
+  },
+  "method": "POST",
+  "path": "/api/2.0/pipelines/[UUID]/updates"
+}
diff --git a/acceptance/pipelines/dry-run/restart/script b/acceptance/pipelines/dry-run/restart/script
@@ -0,0 +1,11 @@
+print_requests() {
+    jq --sort-keys 'select(.method != "GET" and (.path | contains("/pipelines")))' < out.requests.txt
+    rm out.requests.txt
+}
+
+trace $PIPELINES deploy
+rm out.requests.txt
+
+title "Dry running pipeline with --restart flag, should stop the current pipeline and start a new run"
+trace $PIPELINES dry-run my_pipeline --restart
+trace print_requests
diff --git a/acceptance/pipelines/dry-run/restart/test.toml b/acceptance/pipelines/dry-run/restart/test.toml
@@ -0,0 +1 @@
+RecordRequests = true
diff --git a/acceptance/pipelines/install-pipelines-cli/output.txt b/acceptance/pipelines/install-pipelines-cli/output.txt
@@ -14,6 +14,7 @@ Available Commands:
   completion  Generate the autocompletion script for the specified shell
   deploy      Deploy pipelines
   destroy     Destroy a pipelines project
+  dry-run     Validate correctness of the pipeline's graph
   help        Help about any command
   init        Initialize a new pipelines project
   run         Run a pipeline
@@ -55,6 +56,7 @@ Available Commands:
   completion  Generate the autocompletion script for the specified shell
   deploy      Deploy pipelines
   destroy     Destroy a pipelines project
+  dry-run     Validate correctness of the pipeline's graph
   help        Help about any command
   init        Initialize a new pipelines project
   run         Run a pipeline
diff --git a/cmd/pipelines/dry_run.go b/cmd/pipelines/dry_run.go
@@ -0,0 +1,151 @@
+// Copied from cmd/bundle/run.go and adapted for pipelines use.
+// Consider if changes made here should be made to the bundle counterpart as well.
+package pipelines
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/bundle/deploy/terraform"
+	"github.com/databricks/cli/bundle/phases"
+	"github.com/databricks/cli/bundle/resources"
+	"github.com/databricks/cli/bundle/run"
+	"github.com/databricks/cli/bundle/run/output"
+	"github.com/databricks/cli/bundle/statemgmt"
+	"github.com/databricks/cli/cmd/bundle/utils"
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/flags"
+	"github.com/databricks/cli/libs/logdiag"
+	"github.com/spf13/cobra"
+	"golang.org/x/exp/maps"
+)
+
+func dryRunCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "dry-run [KEY]",
+		Short: "Validate correctness of the pipeline's graph",
+		Long: `Validates correctness of the pipeline's graph, identified by KEY. Does not materialize or publish any datasets.
+The KEY is the unique identifier of the pipeline to run.`,
+	}
+
+	var noWait bool
+	var restart bool
+	cmd.Flags().BoolVar(&noWait, "no-wait", false, "Don't wait for the run to complete.")
+	cmd.Flags().BoolVar(&restart, "restart", false, "Restart the run if it is already running.")
+
+	cmd.RunE = func(cmd *cobra.Command, args []string) error {
+		ctx := logdiag.InitContext(cmd.Context())
+		cmd.SetContext(ctx)
+
+		b := utils.ConfigureBundleWithVariables(cmd)
+		if b == nil || logdiag.HasError(ctx) {
+			return root.ErrAlreadyPrinted
+		}
+
+		phases.Initialize(ctx, b)
+		if logdiag.HasError(ctx) {
+			return root.ErrAlreadyPrinted
+		}
+
+		key, _, err := resolveRunArgument(ctx, b, args)
+		if err != nil {
+			return err
+		}
+
+		if !b.DirectDeployment {
+			bundle.ApplySeqContext(ctx, b,
+				terraform.Interpolate(),
+				terraform.Write(),
+			)
+			if logdiag.HasError(ctx) {
+				return root.ErrAlreadyPrinted
+			}
+		}
+
+		bundle.ApplySeqContext(ctx, b,
+			statemgmt.StatePull(),
+			statemgmt.Load(statemgmt.ErrorOnEmptyState),
+		)
+		if logdiag.HasError(ctx) {
+			return root.ErrAlreadyPrinted
+		}
+
+		runner, err := keyToRunner(b, key)
+		if err != nil {
+			return err
+		}
+
+		// For dry-run, we only validate the graph without materializing
+		runOptions := run.Options{
+			Pipeline: run.PipelineOptions{
+				ValidateOnly: true,
+			},
+			NoWait: noWait,
+		}
+
+		var output output.RunOutput
+		if restart {
+			output, err = runner.Restart(ctx, &runOptions)
+		} else {
+			output, err = runner.Run(ctx, &runOptions)
+		}
+		if err != nil {
+			return err
+		}
+
+		if output != nil {
+			switch root.OutputType(cmd) {
+			case flags.OutputText:
+				resultString, err := output.String()
+				if err != nil {
+					return err
+				}
+				_, err = cmd.OutOrStdout().Write([]byte(resultString))
+				if err != nil {
+					return err
+				}
+			case flags.OutputJSON:
+				b, err := json.MarshalIndent(output, "", "  ")
+				if err != nil {
+					return err
+				}
+				_, err = cmd.OutOrStdout().Write(b)
+				if err != nil {
+					return err
+				}
+				_, _ = cmd.OutOrStdout().Write([]byte{'\n'})
+			default:
+				return fmt.Errorf("unknown output type %s", root.OutputType(cmd))
+			}
+		}
+		return nil
+	}
+
+	cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+		b := root.MustConfigureBundle(cmd)
+		if logdiag.HasError(cmd.Context()) {
+			return nil, cobra.ShellCompDirectiveError
+		}
+
+		// No completion in the context of a bundle.
+		// Source and destination paths are taken from bundle configuration.
+		if b == nil {
+			return nil, cobra.ShellCompDirectiveNoFileComp
+		}
+
+		if len(args) == 0 {
+			completions := resources.Completions(b, run.IsRunnable)
+			return maps.Keys(completions), cobra.ShellCompDirectiveNoFileComp
+		} else {
+			// If we know the resource to run, we can complete additional positional arguments.
+			runner, err := keyToRunner(b, args[0])
+			if err != nil {
+				return nil, cobra.ShellCompDirectiveError
+			}
+			return runner.CompleteArgs(args[1:], toComplete)
+		}
+	}
+
+	return cmd
+}
diff --git a/cmd/pipelines/pipelines.go b/cmd/pipelines/pipelines.go
@@ -13,6 +13,7 @@ func New(ctx context.Context) *cobra.Command {
 	cli.AddCommand(initCommand())
 	cli.AddCommand(deployCommand())
 	cli.AddCommand(runCommand())
+	cli.AddCommand(dryRunCommand())
 	cli.AddCommand(authCommand())
 	cli.AddCommand(destroyCommand())
 	cli.AddCommand(versionCommand())
diff --git a/cmd/pipelines/run.go b/cmd/pipelines/run.go
@@ -1,4 +1,5 @@
 // Copied from cmd/bundle/run.go and adapted for pipelines use.
+// Consider if changes made here should be made to the bundle counterpart as well.
 package pipelines
 
 import (
