Check for running resources with --fail-on-active-runs before any mutative operation during deploy (#2743)

## Changes
Check for running resources with `--fail-on-active-runs` before any
mutative operation during deploy
Fixes #2671 

## Why
We should check for running resources and fail if there are any earlier
before any mutative operations such as artifacts folder clean up happens

## Tests
Added acceptance test

<!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->

Author: andrewnester

NEXT_CHANGELOG.md

@@ -9,5 +9,6 @@
 ### CLI
 
 ### Bundles
+* Check for running resources with --fail-on-active-runs before any mutative operation during deploy ([#2743](https://github.com/databricks/cli/pull/2743))
 
 ### API Changes

acceptance/bundle/deploy/fail-on-active-runs/databricks.yml

@@ -0,0 +1,11 @@
+bundle:
+  name: fail-on-active-runs
+
+resources:
+  jobs:
+    my_job:
+      name: My Job
+      tasks:
+      - task_key: my_notebook
+        notebook_task:
+          notebook_path: my_notebook.py

acceptance/bundle/deploy/fail-on-active-runs/my_notebook.py

@@ -0,0 +1,2 @@
+# Databricks notebook source
+1 + 1

acceptance/bundle/deploy/fail-on-active-runs/output.txt

@@ -0,0 +1,23 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/fail-on-active-runs/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> errcode [CLI] bundle deploy --fail-on-active-runs
+Error: job 1 is running
+
+
+Exit code: 1
+
+=== Expecting only 1 delete request to artifact_path/.internal folder from the first deploy
+>>> jq -s .[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal")) out.requests.txt
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace/delete",
+  "body": {
+    "path": "/Workspace/Users/[USERNAME]/.bundle/fail-on-active-runs/default/artifacts/.internal",
+    "recursive": true
+  }
+}

acceptance/bundle/deploy/fail-on-active-runs/script

@@ -0,0 +1,10 @@
+# We deploy the bundle first to create a job
+trace $CLI bundle deploy
+
+# We deploy the bundle again to check that the deploy is failing if the job is running
+trace errcode $CLI bundle deploy --fail-on-active-runs
+
+title "Expecting only 1 delete request to artifact_path/.internal folder from the first deploy"
+trace jq -s '.[] | select(.path=="/api/2.0/workspace/delete") | select(.body.path | test(".*/artifacts/.internal"))' out.requests.txt
+
+rm out.requests.txt

acceptance/bundle/deploy/fail-on-active-runs/test.toml

@@ -0,0 +1,18 @@
+RecordRequests = true
+
+[[Server]]
+Pattern = "GET /api/2.2/jobs/runs/list"
+Response.Body = '''
+{
+  "next_page_token": null,
+  "runs": [
+    {
+      "id": 1234567890,
+      "job_id": 1234567890,
+      "status": {
+        "state": "RUNNING"
+      }
+    }
+  ]
+}
+'''

bundle/phases/deploy.go

@@ -192,6 +192,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
 		terraform.CheckDashboardsModifiedRemotely(),
 		deploy.StatePull(),
 		mutator.ValidateGitDetails(),
+		terraform.CheckRunningResource(),
 		artifacts.CleanUp(),
 		// libraries.CheckForSameNameLibraries() needs to be run after we expand glob references so we
 		// know what are the actual library paths.
@@ -209,7 +210,6 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand
 		permissions.ApplyWorkspaceRootPermissions(),
 		terraform.Interpolate(),
 		terraform.Write(),
-		terraform.CheckRunningResource(),
 		terraform.Plan(terraform.PlanGoal("deploy")),
 	)