From 32c16ff2059eb3a096832659792753cf9289b229 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 2 Jan 2026 14:40:45 +0100 Subject: [PATCH 1/9] plan v2 add remote_matches_config test implement remote_already_set and update the test WIP linter handle nil update update structaccess: support key/value syntax modify structaccess.Get() to return nil on missing values update update tests update fix dashboards update migrate tests dashboard tag comparison fix linter fix clean up test pipelines run_as use IsEqual, fix run_as clean up update undo version change, will be done in follow up to reduce PR size --- .../whl_dynamic/out.plan_update.direct.json | 60 +++--- .../deploy/readplan/basic/out.plan_skip.json | 27 +-- .../bundle/migrate/basic/out.plan_update.json | 87 ++++---- .../dashboards/out.plan_after_migrate.json | 20 +- .../default-python/out.plan_after_deploy.json | 67 +++--- .../out.plan_after_migration.json | 77 +++---- .../bundle/migrate/default-python/output.txt | 154 ++++++-------- acceptance/bundle/migrate/runas/out.plan.json | 14 +- .../id_chain/out.plan_skip.direct.json | 135 ++++++------ .../id_chain/out.plan_update.direct.json | 190 ++++++++--------- .../out.plan_delete.direct.json | 57 ++--- .../out.plan_delete.direct.json | 27 +-- .../out.plan_update.direct.json | 74 ++++--- .../out.plan_noop.direct.json | 27 +-- .../out.plan_update.direct.json | 71 ++++--- .../remote_pipeline/out.plan_skip.direct.json | 36 ++-- .../out.plan_.direct.json | 87 ++++---- .../update-and-resize/out.plan_.direct.json | 44 ++-- .../out.plan.direct.json | 39 ++-- .../out.plan.direct.json | 9 +- .../detect-change/out.plan.direct.json | 46 ++++ .../detect-change/out.plan.terraform.json | 9 + .../dashboards/detect-change/output.txt | 14 ++ .../resources/dashboards/detect-change/script | 1 + .../dashboards/simple/out.plan.direct.json | 20 +- .../change_privilege/out.plan2.direct.json | 22 +- .../grants/volumes/out.plan2.direct.json | 31 ++- .../delete_task/out.plan_update.direct.json | 68 +++--- .../out.plan.direct.json | 35 ++- .../out.plan_post_update.direct.json | 35 +-- .../jobs/remote_matches_config/databricks.yml | 18 ++ .../out.plan.direct.json | 79 +++++++ .../out.plan.terraform.json | 9 + .../jobs/remote_matches_config/out.test.toml | 5 + .../jobs/remote_matches_config/output.txt | 26 +++ .../jobs/remote_matches_config/script | 22 ++ .../jobs/remote_matches_config/test.toml | 2 + .../jobs/update/out.plan_skip.direct.json | 27 +-- .../jobs/update/out.plan_update.direct.json | 38 ++-- .../out.plan_skip.direct.json | 27 +-- .../out.plan_update.direct.json | 38 ++-- .../basic/out.second-plan.direct.json | 50 +++-- .../catalog-name/out.second-plan.direct.json | 43 +++- .../name-change/out.second-plan.direct.json | 38 +++- .../out.second-plan.direct.json | 35 ++- .../schema-name/out.second-plan.direct.json | 43 +++- .../table-prefix/out.second-plan.direct.json | 43 +++- .../update/ai-gateway/out.plan.direct.json | 11 +- .../out.plan.direct.json | 22 +- .../update/config/out.plan.direct.json | 11 +- .../email-notifications/out.plan.direct.json | 11 +- .../update/tags/out.plan.direct.json | 11 +- .../jobs/added_remotely/out.plan.direct.json | 39 ++-- .../out.plan.direct.txt | 27 +-- .../local/out.plan_update.direct.json | 43 ++-- .../out.plan_restore.direct.json | 59 +++--- .../out.destroy.direct.txt | 2 +- .../update/out.plan_delete_all.direct.json | 27 +-- .../update/out.plan_delete_one.direct.json | 43 ++-- .../update/out.plan_post_create.direct.json | 27 +-- .../jobs/update/out.plan_restore.direct.json | 27 +-- .../update/out.plan_set_empty.direct.json | 27 +-- .../jobs/update/out.plan_update.direct.json | 38 ++-- .../update/out.plan_delete_all.direct.json | 9 +- .../update/out.plan_delete_one.direct.json | 25 +-- .../update/out.plan_restore.direct.json | 9 +- .../update/out.plan_update.direct.json | 20 +- .../out.plan_recreate.direct.json | 49 ++++- .../out.plan_recreate.direct.json | 38 +++- .../secret_scopes/basic/out.plan2.direct.json | 28 +-- .../volumes/change-name/out.plan.direct.json | 21 +- .../out.plan_after_deploy_dev.direct.json | 67 +++--- .../out.plan_after_deploy_prod.direct.json | 45 ++-- .../out.plan_after_deploy_dev.direct.json | 36 ++-- .../out.plan_after_deploy_prod.direct.json | 36 ++-- bundle/deployplan/action.go | 23 +- bundle/deployplan/plan.go | 24 +-- bundle/direct/apply.go | 4 +- bundle/direct/bundle_plan.go | 200 ++++++++++-------- bundle/direct/dresources/adapter.go | 170 ++++++--------- bundle/direct/dresources/alert.go | 5 +- bundle/direct/dresources/all_test.go | 29 +-- bundle/direct/dresources/app.go | 4 +- bundle/direct/dresources/cluster.go | 42 ++-- bundle/direct/dresources/dashboard.go | 43 ++-- bundle/direct/dresources/database_catalog.go | 2 +- bundle/direct/dresources/database_instance.go | 2 +- bundle/direct/dresources/experiment.go | 4 +- bundle/direct/dresources/grants.go | 2 +- bundle/direct/dresources/job.go | 2 +- bundle/direct/dresources/model.go | 4 +- .../dresources/model_serving_endpoint.go | 4 +- bundle/direct/dresources/permissions.go | 2 +- bundle/direct/dresources/pipeline.go | 23 +- bundle/direct/dresources/registered_model.go | 4 +- bundle/direct/dresources/schema.go | 4 +- bundle/direct/dresources/secret_scope.go | 2 +- bundle/direct/dresources/secret_scope_acls.go | 4 +- bundle/direct/dresources/sql_warehouse.go | 2 +- .../dresources/synced_database_table.go | 2 +- bundle/direct/dresources/volume.go | 4 +- bundle/phases/deploy.go | 2 +- bundle/phases/destroy.go | 2 +- cmd/bundle/deployment/migrate.go | 11 +- libs/calladapt/calladapt.go | 5 + libs/calladapt/calladapt_test.go | 2 +- 106 files changed, 2034 insertions(+), 1533 deletions(-) create mode 100644 acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json create mode 100644 acceptance/bundle/resources/dashboards/detect-change/out.plan.terraform.json create mode 100644 acceptance/bundle/resources/jobs/remote_matches_config/databricks.yml create mode 100644 acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json create mode 100644 acceptance/bundle/resources/jobs/remote_matches_config/out.plan.terraform.json create mode 100644 acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml create mode 100644 acceptance/bundle/resources/jobs/remote_matches_config/output.txt create mode 100755 acceptance/bundle/resources/jobs/remote_matches_config/script create mode 100644 acceptance/bundle/resources/jobs/remote_matches_config/test.toml diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json index a74a25da14..4fa3ff4c7c 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json @@ -150,36 +150,38 @@ } }, "changes": { - "local": { - "environments[0].spec.dependencies[0]": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - }, - "tasks[task_key='TestTask'].for_each_task.task.libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - }, - "tasks[task_key='TestTask'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "environments[0].spec.dependencies[0]": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" + }, + "tasks[task_key='TestTask'].for_each_task.task.libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" + }, + "tasks[task_key='TestTask'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json b/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json index ceeadda03b..5a9799ccba 100644 --- a/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json +++ b/acceptance/bundle/deploy/readplan/basic/out.plan_skip.json @@ -29,19 +29,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/migrate/basic/out.plan_update.json b/acceptance/bundle/migrate/basic/out.plan_update.json index 5975123f29..e1bf515833 100644 --- a/acceptance/bundle/migrate/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/basic/out.plan_update.json @@ -61,30 +61,31 @@ } }, "changes": { - "local": { - "name": { - "action": "update", - "old": "Test Migration Job", - "new": "Test Migrated Job" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='main'].notebook_task.source": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "name": { + "action": "update", + "old": "Test Migration Job", + "new": "Test Migrated Job", + "remote": "Test Migration Job" + }, + "tasks[task_key='main'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default", + "remote": "WORKSPACE" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -177,23 +178,22 @@ "state": "IDLE" }, "changes": { - "local": { - "name": { - "action": "update", - "old": "Test Migration Pipeline", - "new": "Test Migrated Pipeline" - }, - "tags['myjob_name']": { - "action": "update", - "old": "Test Migration Job", - "new": "Test Migrated Job" - } + "name": { + "action": "update", + "old": "Test Migration Pipeline", + "new": "Test Migrated Pipeline", + "remote": "Test Migration Pipeline" }, - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[UUID]" + }, + "tags['myjob_name']": { + "action": "update", + "old": "Test Migration Job", + "new": "Test Migrated Job", + "remote": "Test Migration Job" } } }, @@ -212,11 +212,10 @@ "volume_type": "MANAGED" }, "changes": { - "remote": { - "storage_location": { - "action": "skip", - "reason": "server_side_default" - } + "storage_location": { + "action": "skip", + "reason": "server_side_default", + "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" } } } diff --git a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json index 421d9acd4a..21b7337f65 100644 --- a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json +++ b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json @@ -20,18 +20,16 @@ "warehouse_id": "123456" }, "changes": { - "local": { - "etag": { - "action": "skip", - "old": "[NUMID]" - } + "etag": { + "action": "skip", + "old": "[NUMID]", + "remote": "[NUMID]" }, - "remote": { - "serialized_dashboard": { - "action": "skip", - "old": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n", - "new": "{\"pages\":[{\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"name\":\"02724bf2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n" - } + "serialized_dashboard": { + "action": "skip", + "old": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n", + "new": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n", + "remote": "{\"pages\":[{\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"name\":\"02724bf2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n" } } } diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json index 0b24dab3a5..725d20b834 100644 --- a/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json +++ b/acceptance/bundle/migrate/default-python/out.plan_after_deploy.json @@ -217,35 +217,37 @@ } }, "changes": { - "local": { - "tasks[task_key='notebook_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - }, - "tasks[task_key='python_wheel_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='notebook_task'].notebook_task.source": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default", + "remote": "WORKSPACE" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -289,11 +291,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[UUID]" } } } diff --git a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json index 6873ef820c..401878a1b1 100644 --- a/acceptance/bundle/migrate/default-python/out.plan_after_migration.json +++ b/acceptance/bundle/migrate/default-python/out.plan_after_migration.json @@ -217,45 +217,37 @@ } }, "changes": { - "local": { - "tasks[task_key='notebook_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - }, - "tasks[task_key='python_wheel_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='notebook_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" - }, - "tasks[task_key='notebook_task'].notebook_task.source": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='python_wheel_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default", + "remote": "WORKSPACE" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -299,11 +291,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[UUID]" } } } diff --git a/acceptance/bundle/migrate/default-python/output.txt b/acceptance/bundle/migrate/default-python/output.txt index 08881f2c5f..af1e30a08b 100644 --- a/acceptance/bundle/migrate/default-python/output.txt +++ b/acceptance/bundle/migrate/default-python/output.txt @@ -74,53 +74,44 @@ Building python_artifact... >>> jq .plan[] | .changes ../out.plan_after_migration.json { - "local": { - "tasks[task_key='notebook_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" - }, - "tasks[task_key='python_wheel_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='notebook_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - }, - "tasks[task_key='notebook_task'].notebook_task.source": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='python_wheel_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default", + "remote": "WORKSPACE" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[UUID]" } } @@ -157,52 +148,43 @@ Building python_artifact... >>> jq .plan[] | .changes ../out.plan_after_migration.json { - "local": { - "tasks[task_key='notebook_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" - }, - "tasks[task_key='python_wheel_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='notebook_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - }, - "tasks[task_key='notebook_task'].notebook_task.source": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='python_wheel_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default", + "remote": "WORKSPACE" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[UUID]" } } diff --git a/acceptance/bundle/migrate/runas/out.plan.json b/acceptance/bundle/migrate/runas/out.plan.json index 1aafc2d84b..c487f92147 100644 --- a/acceptance/bundle/migrate/runas/out.plan.json +++ b/acceptance/bundle/migrate/runas/out.plan.json @@ -35,12 +35,14 @@ "state": "IDLE" }, "changes": { - "remote": { - "run_as": { - "action": "skip", - "old": { - "service_principal_name": "[UUID]" - } + "run_as": { + "action": "skip", + "reason": "override", + "old": { + "service_principal_name": "[UUID]" + }, + "new": { + "service_principal_name": "[UUID]" } } } diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json b/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json index a8adc7c6de..63b73ce495 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_skip.direct.json @@ -30,19 +30,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -78,19 +79,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -126,19 +128,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -174,19 +177,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -222,19 +226,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json b/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json index 70909c3bad..352aa0cd9a 100644 --- a/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/id_chain/out.plan_update.direct.json @@ -46,26 +46,26 @@ } }, "changes": { - "local": { - "description": { - "action": "update", - "old": "aa_desc", - "new": "aa_new_desc" - } + "description": { + "action": "update", + "old": "aa_desc", + "new": "aa_new_desc", + "remote": "aa_desc" }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -117,26 +117,26 @@ } }, "changes": { - "local": { - "description": { - "action": "update", - "old": "prefix [NUMID]", - "new": "new_prefix [NUMID]" - } + "description": { + "action": "update", + "old": "prefix [NUMID]", + "new": "new_prefix [NUMID]", + "remote": "prefix [NUMID]" }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -188,26 +188,26 @@ } }, "changes": { - "local": { - "description": { - "action": "update", - "old": "prefix [NUMID]", - "new": "new_prefix [NUMID]" - } + "description": { + "action": "update", + "old": "prefix [NUMID]", + "new": "new_prefix [NUMID]", + "remote": "prefix [NUMID]" }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -259,26 +259,26 @@ } }, "changes": { - "local": { - "description": { - "action": "update", - "old": "prefix [NUMID]", - "new": "new_prefix [NUMID]" - } + "description": { + "action": "update", + "old": "prefix [NUMID]", + "new": "new_prefix [NUMID]", + "remote": "prefix [NUMID]" }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -330,26 +330,26 @@ } }, "changes": { - "local": { - "description": { - "action": "update", - "old": "prefix [NUMID]", - "new": "new_prefix [NUMID]" - } + "description": { + "action": "update", + "old": "prefix [NUMID]", + "new": "new_prefix [NUMID]", + "remote": "prefix [NUMID]" }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json index 6ff5a47afb..759cffcb8b 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_bar/out.plan_delete.direct.json @@ -77,32 +77,39 @@ } }, "changes": { - "local": { - "tasks": { - "action": "update", - "old": [ - { - "run_job_task": { - "job_id": [BAR_ID] - }, - "task_key": "job_task" - } - ] - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "tasks": { + "action": "update", + "old": [ + { + "run_job_task": { + "job_id": [BAR_ID] + }, + "task_key": "job_task" + } + ], + "remote": [ + { + "run_job_task": { + "job_id": [BAR_ID] + }, + "task_key": "job_task" + } + ] + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json index 2153e79826..88d4e3516e 100644 --- a/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json +++ b/acceptance/bundle/resource_deps/job_id_delete_foo/out.plan_delete.direct.json @@ -29,19 +29,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, diff --git a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json index 3e6340104a..a949f6947a 100644 --- a/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/jobs_update_remote/out.plan_update.direct.json @@ -36,19 +36,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -123,28 +124,31 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "job_clusters[0].new_cluster.num_workers": { - "action": "update", - "old": 0 - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "trigger.periodic.unit": { - "action": "update", - "old": "DAYS", - "new": "HOURS" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "job_clusters[0].new_cluster.num_workers": { + "action": "update", + "old": 0, + "new": 0 + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "trigger.periodic.unit": { + "action": "update", + "old": "DAYS", + "new": "DAYS", + "remote": "HOURS" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json index 93e5973a4a..8eb8f9c0f4 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_noop.direct.json @@ -36,19 +36,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, diff --git a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json index 47fd5d4e09..43116fc6eb 100644 --- a/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json +++ b/acceptance/bundle/resource_deps/pipelines_recreate/out.plan_update.direct.json @@ -32,7 +32,7 @@ } }, "remote_state": { - "created_time": [UNIX_TIME_MILLIS], + "created_time": [UNIX_TIME_MILLIS][0], "creator_user_name": "[USERNAME]", "job_id": [BAR_ID], "run_as_user_name": "[USERNAME]", @@ -55,26 +55,26 @@ } }, "changes": { - "local": { - "description": { - "action": "update", - "old": "depends on foo id [FOO_ID]", - "new": "depends on foo id ${resources.pipelines.foo.id}" - } + "description": { + "action": "update", + "old": "depends on foo id [FOO_ID]", + "new": "depends on foo id ${resources.pipelines.foo.id}", + "remote": "depends on foo id [FOO_ID]" }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -92,13 +92,32 @@ "storage": "dbfs:/my-new-storage" } }, + "remote_state": { + "creator_user_name": "[USERNAME]", + "last_modified": [UNIX_TIME_MILLIS][1], + "name": "pipeline foo", + "pipeline_id": "[FOO_ID]", + "run_as_user_name": "[USERNAME]", + "spec": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[FOO_ID]", + "name": "pipeline foo", + "storage": "dbfs:/my-storage" + }, + "state": "IDLE" + }, "changes": { - "local": { - "storage": { - "action": "recreate", - "old": "dbfs:/my-storage", - "new": "dbfs:/my-new-storage" - } + "storage": { + "action": "recreate", + "reason": "field_triggers", + "old": "dbfs:/my-storage", + "new": "dbfs:/my-new-storage", + "remote": "dbfs:/my-storage" } } } diff --git a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json index be3c8d2cef..eb20610a13 100644 --- a/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json +++ b/acceptance/bundle/resource_deps/remote_pipeline/out.plan_skip.direct.json @@ -26,11 +26,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[UUID]" } } }, @@ -62,11 +61,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[UUID]" } } }, @@ -98,11 +96,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[UUID]" } } }, @@ -134,11 +131,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[UUID]" } } } diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json index 11fa40f2bf..b729ddf057 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize-autoscale/out.plan_.direct.json @@ -45,18 +45,17 @@ "spark_version": "13.3.x-snapshot-scala2.12" }, "changes": { - "local": { - "autoscale": { - "action": "update", - "new": { - "max_workers": 4, - "min_workers": 2 - } - }, - "num_workers": { - "action": "update", - "old": 2 + "autoscale": { + "action": "update", + "new": { + "max_workers": 4, + "min_workers": 2 } + }, + "num_workers": { + "action": "update", + "old": 2, + "remote": 2 } } } @@ -94,17 +93,17 @@ "spark_version": "13.3.x-snapshot-scala2.12" }, "changes": { - "local": { - "autoscale.max_workers": { - "action": "update", - "old": 4, - "new": 5 - }, - "autoscale.min_workers": { - "action": "update", - "old": 2, - "new": 3 - } + "autoscale.max_workers": { + "action": "update", + "old": 4, + "new": 5, + "remote": 4 + }, + "autoscale.min_workers": { + "action": "update", + "old": 2, + "new": 3, + "remote": 2 } } } @@ -143,17 +142,17 @@ "state": "RUNNING" }, "changes": { - "local": { - "autoscale.max_workers": { - "action": "resize", - "old": 5, - "new": 6 - }, - "autoscale.min_workers": { - "action": "resize", - "old": 3, - "new": 4 - } + "autoscale.max_workers": { + "action": "resize", + "old": 5, + "new": 6, + "remote": 5 + }, + "autoscale.min_workers": { + "action": "resize", + "old": 3, + "new": 4, + "remote": 3 } } } @@ -189,18 +188,20 @@ "state": "RUNNING" }, "changes": { - "local": { - "autoscale": { - "action": "resize", - "old": { - "max_workers": 6, - "min_workers": 4 - } + "autoscale": { + "action": "resize", + "old": { + "max_workers": 6, + "min_workers": 4 }, - "num_workers": { - "action": "resize", - "new": 3 + "remote": { + "max_workers": 6, + "min_workers": 4 } + }, + "num_workers": { + "action": "resize", + "new": 3 } } } diff --git a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json index 85a1aec302..de1207f700 100644 --- a/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json +++ b/acceptance/bundle/resources/clusters/deploy/update-and-resize/out.plan_.direct.json @@ -51,12 +51,11 @@ "spark_version": "13.3.x-snapshot-scala2.12" }, "changes": { - "local": { - "num_workers": { - "action": "update", - "old": 2, - "new": 3 - } + "num_workers": { + "action": "update", + "old": 2, + "new": 3, + "remote": 2 } } } @@ -95,12 +94,11 @@ "state": "RUNNING" }, "changes": { - "local": { - "num_workers": { - "action": "resize", - "old": 3, - "new": 4 - } + "num_workers": { + "action": "resize", + "old": 3, + "new": 4, + "remote": 3 } } } @@ -139,17 +137,17 @@ "state": "RUNNING" }, "changes": { - "local": { - "num_workers": { - "action": "resize", - "old": 4, - "new": 5 - }, - "spark_conf['spark.executor.memory']": { - "action": "update", - "old": "2g", - "new": "4g" - } + "num_workers": { + "action": "resize", + "old": 4, + "new": 5, + "remote": 4 + }, + "spark_conf['spark.executor.memory']": { + "action": "update", + "old": "2g", + "new": "4g", + "remote": "2g" } } } diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json index 2a6766a1e0..d87fcba0bc 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json @@ -20,26 +20,29 @@ "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" }, "changes": { - "local": { - "etag": { - "action": "skip", - "old": [ETAG] - } + "dataset_catalog": { + "action": "skip", + "old": "main", + "new": "main" + }, + "dataset_schema": { + "action": "skip", + "old": "default", + "new": "default" + }, + "etag": { + "action": "skip", + "old": [ETAG], + "remote": [ETAG] + }, + "serialized_dashboard": { + "action": "skip", + "old": "{\n \"pages\": [\n {\n \"name\": \"test_page\",\n \"displayName\": \"Test Page\",\n \"pageType\": \"PAGE_TYPE_CANVAS\"\n }\n ],\n \"datasets\": [\n {\n \"name\": \"bf8f76f4\",\n \"displayName\": \"Test Dataset\",\n \"queryLines\": [\n \"SELECT 1\\n\"\n ],\n \"catalog\": \"foobar\",\n \"schema\": \"foobar\"\n }\n ]\n}\n", + "new": "{\n \"pages\": [\n {\n \"name\": \"test_page\",\n \"displayName\": \"Test Page\",\n \"pageType\": \"PAGE_TYPE_CANVAS\"\n }\n ],\n \"datasets\": [\n {\n \"name\": \"bf8f76f4\",\n \"displayName\": \"Test Dataset\",\n \"queryLines\": [\n \"SELECT 1\\n\"\n ],\n \"catalog\": \"foobar\",\n \"schema\": \"foobar\"\n }\n ]\n}\n", + "remote": "{\"datasets\":[{\"catalog\":\"main\",\"displayName\":\"Test Dataset\",\"name\":\"bf8f76f4\",\"queryLines\":[\"SELECT 1\\n\"],\"schema\":\"default\"}],\"pages\":[{\"displayName\":\"Test Page\",\"name\":\"test_page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n" }, "remote": { - "dataset_catalog": { - "action": "skip", - "old": "main" - }, - "dataset_schema": { - "action": "skip", - "old": "default" - }, - "serialized_dashboard": { - "action": "skip", - "old": "[SERIALIZED_FIXTURE_OLD]", - "new": "[SERIALIZED_FIXTURE_NEW]" - } + "serialized_dashboard": null } } } diff --git a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json index 56fdcd680f..ae0bc8f97a 100644 --- a/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/delete-trashed-out-of-band/out.plan.direct.json @@ -16,11 +16,10 @@ } }, "changes": { - "local": { - "etag": { - "action": "skip", - "old": [ETAG] - } + "etag": { + "action": "update", + "reason": "remote_already_set", + "old": [ETAG] } } } diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json new file mode 100644 index 0000000000..5317d4ddd1 --- /dev/null +++ b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json @@ -0,0 +1,46 @@ +{ + "plan_version": 1, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.dashboards.file_reference": { + "action": "update", + "new_state": { + "value": { + "display_name": "test-dashboard-[UNIQUE_NAME]", + "embed_credentials": false, + "parent_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/resources", + "serialized_dashboard": "{\n \"pages\": [\n {\n \"displayName\": \"New Page\",\n \"layout\": [\n {\n \"position\": {\n \"height\": 2,\n \"width\": 6,\n \"x\": 0,\n \"y\": 0\n },\n \"widget\": {\n \"name\": \"82eb9107\",\n \"textbox_spec\": \"# I'm a title\"\n }\n },\n {\n \"position\": {\n \"height\": 2,\n \"width\": 6,\n \"x\": 0,\n \"y\": 2\n },\n \"widget\": {\n \"name\": \"ffa6de4f\",\n \"textbox_spec\": \"Text\"\n }\n }\n ],\n \"name\": \"fdd21a3c\"\n }\n ]\n}\n", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + }, + "remote_state": { + "create_time": "[TIMESTAMP]", + "dashboard_id": "[DASHBOARD_ID]", + "display_name": "test-dashboard-[UNIQUE_NAME]", + "embed_credentials": false, + "etag": [ETAG], + "lifecycle_state": "ACTIVE", + "parent_path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/resources", + "path": "/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/resources/test-dashboard-[UNIQUE_NAME].lvdash.json", + "serialized_dashboard": "{}\n", + "update_time": "[TIMESTAMP]", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + }, + "changes": { + "etag": { + "action": "update", + "old": [ETAG], + "remote": [ETAG] + }, + "serialized_dashboard": { + "action": "skip", + "old": "{\n \"pages\": [\n {\n \"displayName\": \"New Page\",\n \"layout\": [\n {\n \"position\": {\n \"height\": 2,\n \"width\": 6,\n \"x\": 0,\n \"y\": 0\n },\n \"widget\": {\n \"name\": \"82eb9107\",\n \"textbox_spec\": \"# I'm a title\"\n }\n },\n {\n \"position\": {\n \"height\": 2,\n \"width\": 6,\n \"x\": 0,\n \"y\": 2\n },\n \"widget\": {\n \"name\": \"ffa6de4f\",\n \"textbox_spec\": \"Text\"\n }\n }\n ],\n \"name\": \"fdd21a3c\"\n }\n ]\n}\n", + "new": "{\n \"pages\": [\n {\n \"displayName\": \"New Page\",\n \"layout\": [\n {\n \"position\": {\n \"height\": 2,\n \"width\": 6,\n \"x\": 0,\n \"y\": 0\n },\n \"widget\": {\n \"name\": \"82eb9107\",\n \"textbox_spec\": \"# I'm a title\"\n }\n },\n {\n \"position\": {\n \"height\": 2,\n \"width\": 6,\n \"x\": 0,\n \"y\": 2\n },\n \"widget\": {\n \"name\": \"ffa6de4f\",\n \"textbox_spec\": \"Text\"\n }\n }\n ],\n \"name\": \"fdd21a3c\"\n }\n ]\n}\n", + "remote": "{}\n" + } + } + } + } +} diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.plan.terraform.json b/acceptance/bundle/resources/dashboards/detect-change/out.plan.terraform.json new file mode 100644 index 0000000000..70e76d8731 --- /dev/null +++ b/acceptance/bundle/resources/dashboards/detect-change/out.plan.terraform.json @@ -0,0 +1,9 @@ +{ + "plan_version": 1, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.dashboards.file_reference": { + "action": "update" + } + } +} diff --git a/acceptance/bundle/resources/dashboards/detect-change/output.txt b/acceptance/bundle/resources/dashboards/detect-change/output.txt index fa4361e30a..53179be516 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/output.txt +++ b/acceptance/bundle/resources/dashboards/detect-change/output.txt @@ -66,6 +66,20 @@ update dashboards.file_reference Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged +>>> [CLI] bundle plan -o json +Warning: dashboard "file_reference" has been modified remotely + at resources.dashboards.file_reference + in databricks.yml:10:7 + +This dashboard has been modified remotely since the last bundle deployment. +These modifications are untracked and will be overwritten on deploy. + +Make sure that the local dashboard definition matches what you intend to deploy +before proceeding with the deployment. + +Run `databricks bundle deploy --force` to bypass this error. + + >>> errcode [CLI] bundle deploy Error: dashboard "file_reference" has been modified remotely at resources.dashboards.file_reference diff --git a/acceptance/bundle/resources/dashboards/detect-change/script b/acceptance/bundle/resources/dashboards/detect-change/script index dd61a61235..773ca7041c 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/script +++ b/acceptance/bundle/resources/dashboards/detect-change/script @@ -34,6 +34,7 @@ $CLI lakeview update "${RESOURCE_ID}" --json "${DASHBOARD_JSON}" | jq '{lifecycl title "Try to redeploy the bundle and confirm that the out of band modification is detected:" trace $CLI bundle plan +trace $CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json trace errcode $CLI bundle deploy title "Redeploy the bundle with the --force flag and confirm that the out of band modification is ignored:" diff --git a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json index 83aa0a6408..e5cf69728f 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json @@ -20,18 +20,16 @@ "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" }, "changes": { - "local": { - "etag": { - "action": "skip", - "old": [ETAG] - } + "etag": { + "action": "skip", + "old": [ETAG], + "remote": [ETAG] }, - "remote": { - "serialized_dashboard": { - "action": "skip", - "old": "{ }\n", - "new": "{}\n" - } + "serialized_dashboard": { + "action": "skip", + "old": "{ }\n", + "new": "{ }\n", + "remote": "{}\n" } } } diff --git a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json index 3eb1e986fa..a6099e6f5c 100644 --- a/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/schemas/change_privilege/out.plan2.direct.json @@ -54,17 +54,17 @@ ] }, "changes": { - "local": { - "grants[0].privileges[0]": { - "action": "update", - "old": "CREATE_TABLE", - "new": "APPLY_TAG" - }, - "grants[0].privileges[1]": { - "action": "update", - "old": "USE_SCHEMA", - "new": "CREATE_TABLE" - } + "grants[0].privileges[0]": { + "action": "update", + "old": "CREATE_TABLE", + "new": "APPLY_TAG", + "remote": "CREATE_TABLE" + }, + "grants[0].privileges[1]": { + "action": "update", + "old": "USE_SCHEMA", + "new": "CREATE_TABLE", + "remote": "USE_SCHEMA" } } } diff --git a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json index 5aada9a77e..574e35bbc7 100644 --- a/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/volumes/out.plan2.direct.json @@ -37,11 +37,10 @@ "volume_type": "MANAGED" }, "changes": { - "remote": { - "storage_location": { - "action": "skip", - "reason": "server_side_default" - } + "storage_location": { + "action": "skip", + "reason": "server_side_default", + "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" } } }, @@ -82,17 +81,17 @@ ] }, "changes": { - "local": { - "grants[0].privileges[0]": { - "action": "update", - "old": "READ_VOLUME", - "new": "MANAGE" - }, - "grants[0].privileges[1]": { - "action": "update", - "old": "WRITE_VOLUME", - "new": "READ_VOLUME" - } + "grants[0].privileges[0]": { + "action": "update", + "old": "READ_VOLUME", + "new": "MANAGE", + "remote": "READ_VOLUME" + }, + "grants[0].privileges[1]": { + "action": "update", + "old": "WRITE_VOLUME", + "new": "READ_VOLUME", + "remote": "WRITE_VOLUME" } } } diff --git a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json index 8d1e451f98..8a1ab101d9 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json @@ -99,37 +99,49 @@ } }, "changes": { - "local": { - "tasks[task_key='TestTask1']": { - "action": "update", - "old": { - "existing_cluster_id": "0717-132531-5opeqon1", - "libraries": [ - { - "whl": "/Workspace/Users/foo@bar.com/mywheel.whl" - } - ], - "python_wheel_task": { - "entry_point": "run", - "package_name": "whl" - }, - "task_key": "TestTask1" - } - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" + "tasks[task_key='TestTask1']": { + "action": "update", + "old": { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/foo@bar.com/mywheel.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "whl" + }, + "task_key": "TestTask1" }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" + "remote": { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/foo@bar.com/mywheel.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "whl" + }, + "task_key": "TestTask1" } + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.plan.direct.json b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.plan.direct.json index 14ae4bc44d..319a62d25d 100644 --- a/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.plan.direct.json +++ b/acceptance/bundle/resources/jobs/on_failure_empty_slice/out.plan.direct.json @@ -1,22 +1,21 @@ { - "local": { - "email_notifications.on_failure": { - "action": "update", - "new": [] - } + "email_notifications.on_failure": { + "action": "update", + "new": [] }, - "remote": { - "tasks[task_key='usage_logs_grants'].notebook_task.source": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "tasks[task_key='usage_logs_grants'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default", + "remote": "WORKSPACE" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } diff --git a/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json b/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json index d334a1fc4f..94c1153333 100644 --- a/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json +++ b/acceptance/bundle/resources/jobs/remote_add_tag/out.plan_post_update.direct.json @@ -81,23 +81,24 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "tags['new_tag']": { - "action": "update", - "new": "new_value" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "tags['new_tag']": { + "action": "update", + "remote": "new_value" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/databricks.yml b/acceptance/bundle/resources/jobs/remote_matches_config/databricks.yml new file mode 100644 index 0000000000..04607fe613 --- /dev/null +++ b/acceptance/bundle/resources/jobs/remote_matches_config/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +resources: + jobs: + my_job: + name: "Test Job for Conflict Detection" + max_concurrent_runs: 1 + tags: + environment: dev + tasks: + - task_key: main_task + notebook_task: + notebook_path: /Users/{{workspace_user_name}}/test_notebook + new_cluster: + spark_version: 13.3.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 1 diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json new file mode 100644 index 0000000000..43270a30ad --- /dev/null +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.direct.json @@ -0,0 +1,79 @@ +{ + "plan_version": 1, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.jobs.my_job": { + "action": "skip", + "remote_state": { + "created_time": [UNIX_TIME_MILLIS], + "creator_user_name": "[USERNAME]", + "job_id": [JOB_ID], + "run_as_user_name": "[USERNAME]", + "settings": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "email_notifications": {}, + "format": "MULTI_TASK", + "max_concurrent_runs": 2, + "name": "Test Job for Conflict Detection", + "queue": { + "enabled": true + }, + "tags": { + "environment": "dev" + }, + "tasks": [ + { + "new_cluster": { + "node_type_id": "[NODE_TYPE_ID]", + "num_workers": 1, + "spark_version": "13.3.x-scala2.12" + }, + "notebook_task": { + "notebook_path": "/Users/{{workspace_user_name}}/test_notebook", + "source": "WORKSPACE" + }, + "task_key": "main_task" + } + ], + "timeout_seconds": 0, + "webhook_notifications": {} + } + }, + "changes": { + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "max_concurrent_runs": { + "action": "skip", + "reason": "remote_already_set", + "old": 1, + "new": 2, + "remote": 2 + }, + "tasks[task_key='main_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default", + "remote": "WORKSPACE" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + } + } + } + } +} diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.terraform.json b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.terraform.json new file mode 100644 index 0000000000..6861bfdb0c --- /dev/null +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.plan.terraform.json @@ -0,0 +1,9 @@ +{ + "plan_version": 1, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.jobs.my_job": { + "action": "skip" + } + } +} diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/resources/jobs/remote_matches_config/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/output.txt b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt new file mode 100644 index 0000000000..a067a14177 --- /dev/null +++ b/acceptance/bundle/resources/jobs/remote_matches_config/output.txt @@ -0,0 +1,26 @@ + +>>> [CLI] bundle plan +create jobs.my_job + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +=== Make local changes: update max_concurrent_runs to 2 +=== Make remote changes via CLI: add a new tag and set timeout + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> print_requests.py //jobs diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/script b/acceptance/bundle/resources/jobs/remote_matches_config/script new file mode 100755 index 0000000000..2799279c14 --- /dev/null +++ b/acceptance/bundle/resources/jobs/remote_matches_config/script @@ -0,0 +1,22 @@ +trace $CLI bundle plan +$CLI bundle deploy +job_id="$(read_id.py jobs my_job)" +echo "$job_id:JOB_ID" >> ACC_REPLS + +trace $CLI bundle plan | contains.py "0 to add, 0 to change, 0 to delete" + +title "Make local changes: update max_concurrent_runs to 2" +update_file.py databricks.yml "max_concurrent_runs: 1" "max_concurrent_runs: 2" + +title "Make remote changes via CLI: add a new tag and set timeout\n" +edit_resource.py jobs $job_id < out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +rm out.requests.txt +# XXX READPLAN +trace $CLI bundle deploy +trace print_requests.py //jobs diff --git a/acceptance/bundle/resources/jobs/remote_matches_config/test.toml b/acceptance/bundle/resources/jobs/remote_matches_config/test.toml new file mode 100644 index 0000000000..eb67288687 --- /dev/null +++ b/acceptance/bundle/resources/jobs/remote_matches_config/test.toml @@ -0,0 +1,2 @@ +RecordRequests = true +Ignore = [".databricks"] diff --git a/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json b/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json index 1e8a16999a..e117cce588 100644 --- a/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.plan_skip.direct.json @@ -45,19 +45,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json index be4135f7dd..84ef2a002d 100644 --- a/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/update/out.plan_update.direct.json @@ -76,26 +76,26 @@ } }, "changes": { - "local": { - "trigger.periodic.unit": { - "action": "update", - "old": "DAYS", - "new": "HOURS" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "trigger.periodic.unit": { + "action": "update", + "old": "DAYS", + "new": "HOURS", + "remote": "DAYS" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json b/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json index 02ed23f913..e9ff09da2e 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json +++ b/acceptance/bundle/resources/jobs/update_single_node/out.plan_skip.direct.json @@ -52,19 +52,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json index 73f0b94939..18de25f615 100644 --- a/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/update_single_node/out.plan_update.direct.json @@ -90,26 +90,26 @@ } }, "changes": { - "local": { - "trigger.periodic.unit": { - "action": "update", - "old": "DAYS", - "new": "HOURS" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "trigger.periodic.unit": { + "action": "update", + "old": "DAYS", + "new": "HOURS", + "remote": "DAYS" + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.json index 44c1439c36..2c0c32856c 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/basic/out.second-plan.direct.json @@ -11,13 +11,40 @@ "name": "[ENDPOINT_NAME_2]" } }, - "changes": { - "local": { - "name": { - "action": "recreate", - "old": "[ENDPOINT_NAME_1]", - "new": "[ENDPOINT_NAME_2]" + "remote_state": { + "endpoint_details": { + "creation_timestamp": [UNIX_TIME_MILLIS][0], + "creator": "[USERNAME]", + "description": "", + "id": "[ENDPOINT_ID_1]", + "last_updated_timestamp": [UNIX_TIME_MILLIS][0], + "name": "[ENDPOINT_NAME_1]", + "permission_level": "CAN_MANAGE", + "route_optimized": false, + "state": { + "config_update": "NOT_UPDATING", + "ready": "NOT_READY" } + }, + "endpoint_id": "[ENDPOINT_ID_1]" + }, + "changes": { + "description": { + "action": "skip", + "reason": "server_side_default", + "remote": "" + }, + "name": { + "action": "recreate", + "reason": "field_triggers", + "old": "[ENDPOINT_NAME_1]", + "new": "[ENDPOINT_NAME_2]", + "remote": "[ENDPOINT_NAME_1]" + }, + "route_optimized": { + "action": "skip", + "reason": "server_side_default", + "remote": false } } }, @@ -61,12 +88,11 @@ ] }, "changes": { - "local": { - "object_id": { - "action": "update", - "old": "/serving-endpoints/[ENDPOINT_ID_1]", - "new": "" - } + "object_id": { + "action": "update", + "old": "/serving-endpoints/[ENDPOINT_ID_1]", + "new": "", + "remote": "/serving-endpoints/[ENDPOINT_ID_1]" } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json index 80a82c107e..db01a87a95 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/catalog-name/out.second-plan.direct.json @@ -31,13 +31,44 @@ "name": "[ORIGINAL_ENDPOINT_ID]" } }, - "changes": { - "local": { - "config.auto_capture_config.catalog_name": { - "action": "recreate", - "old": "main", - "new": "other_catalog" + "remote_state": { + "endpoint_details": { + "config": { + "auto_capture_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "my_table" + }, + "served_entities": [ + { + "external_model": { + "name": "gpt-4o-mini", + "openai_config": { + "openai_api_key": "{{secrets/test-scope/openai-key}}" + }, + "provider": "openai", + "task": "llm/v1/chat" + }, + "name": "prod" + } + ] + }, + "creator": "[USERNAME]", + "id": "[UUID]", + "name": "[ORIGINAL_ENDPOINT_ID]", + "state": { + "config_update": "NOT_UPDATING" } + }, + "endpoint_id": "[UUID]" + }, + "changes": { + "config.auto_capture_config.catalog_name": { + "action": "recreate", + "reason": "field_triggers", + "old": "main", + "new": "other_catalog", + "remote": "main" } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json index d0918832c6..a56491fd32 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/name-change/out.second-plan.direct.json @@ -26,13 +26,39 @@ "name": "[NEW_ENDPOINT_ID]" } }, - "changes": { - "local": { - "name": { - "action": "recreate", - "old": "[ORIGINAL_ENDPOINT_ID]", - "new": "[NEW_ENDPOINT_ID]" + "remote_state": { + "endpoint_details": { + "config": { + "served_entities": [ + { + "external_model": { + "name": "gpt-4o-mini", + "openai_config": { + "openai_api_key": "{{secrets/test-scope/openai-key}}" + }, + "provider": "openai", + "task": "llm/v1/chat" + }, + "name": "prod" + } + ] + }, + "creator": "[USERNAME]", + "id": "[UUID]", + "name": "[ORIGINAL_ENDPOINT_ID]", + "state": { + "config_update": "NOT_UPDATING" } + }, + "endpoint_id": "[UUID]" + }, + "changes": { + "name": { + "action": "recreate", + "reason": "field_triggers", + "old": "[ORIGINAL_ENDPOINT_ID]", + "new": "[NEW_ENDPOINT_ID]", + "remote": "[ORIGINAL_ENDPOINT_ID]" } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json index 4217fca47a..85f1c0efcd 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/route-optimized/out.second-plan.direct.json @@ -23,13 +23,36 @@ "route_optimized": true } }, - "changes": { - "local": { - "route_optimized": { - "action": "recreate", - "old": false, - "new": true + "remote_state": { + "endpoint_details": { + "config": { + "served_entities": [ + { + "entity_name": "system.ai.llama_v3_2_1b_instruct", + "entity_version": "1", + "name": "llama", + "scale_to_zero_enabled": true, + "workload_size": "Small" + } + ] + }, + "creator": "[USERNAME]", + "id": "[UUID]", + "name": "[ORIGINAL_ENDPOINT_ID]", + "route_optimized": false, + "state": { + "config_update": "NOT_UPDATING" } + }, + "endpoint_id": "[UUID]" + }, + "changes": { + "route_optimized": { + "action": "recreate", + "reason": "field_triggers", + "old": false, + "new": true, + "remote": false } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json index 52111788ef..5d1fb654da 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/schema-name/out.second-plan.direct.json @@ -31,13 +31,44 @@ "name": "[ORIGINAL_ENDPOINT_ID]" } }, - "changes": { - "local": { - "config.auto_capture_config.schema_name": { - "action": "recreate", - "old": "default", - "new": "other_schema" + "remote_state": { + "endpoint_details": { + "config": { + "auto_capture_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "my_table" + }, + "served_entities": [ + { + "external_model": { + "name": "gpt-4o-mini", + "openai_config": { + "openai_api_key": "{{secrets/test-scope/openai-key}}" + }, + "provider": "openai", + "task": "llm/v1/chat" + }, + "name": "prod" + } + ] + }, + "creator": "[USERNAME]", + "id": "[UUID]", + "name": "[ORIGINAL_ENDPOINT_ID]", + "state": { + "config_update": "NOT_UPDATING" } + }, + "endpoint_id": "[UUID]" + }, + "changes": { + "config.auto_capture_config.schema_name": { + "action": "recreate", + "reason": "field_triggers", + "old": "default", + "new": "other_schema", + "remote": "default" } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json index bd4e2bc4f5..82113352d9 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/recreate/table-prefix/out.second-plan.direct.json @@ -31,13 +31,44 @@ "name": "[ORIGINAL_ENDPOINT_ID]" } }, - "changes": { - "local": { - "config.auto_capture_config.table_name_prefix": { - "action": "recreate", - "old": "my_table", - "new": "other_table" + "remote_state": { + "endpoint_details": { + "config": { + "auto_capture_config": { + "catalog_name": "main", + "schema_name": "default", + "table_name_prefix": "my_table" + }, + "served_entities": [ + { + "external_model": { + "name": "gpt-4o-mini", + "openai_config": { + "openai_api_key": "{{secrets/test-scope/openai-key}}" + }, + "provider": "openai", + "task": "llm/v1/chat" + }, + "name": "prod" + } + ] + }, + "creator": "[USERNAME]", + "id": "[UUID]", + "name": "[ORIGINAL_ENDPOINT_ID]", + "state": { + "config_update": "NOT_UPDATING" } + }, + "endpoint_id": "[UUID]" + }, + "changes": { + "config.auto_capture_config.table_name_prefix": { + "action": "recreate", + "reason": "field_triggers", + "old": "my_table", + "new": "other_table", + "remote": "my_table" } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json index 8adbb06a85..b5456d1890 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/ai-gateway/out.plan.direct.json @@ -63,12 +63,11 @@ "endpoint_id": "[UUID]" }, "changes": { - "local": { - "ai_gateway.inference_table_config.catalog_name": { - "action": "update", - "old": "first-inference-catalog", - "new": "second-inference-catalog" - } + "ai_gateway.inference_table_config.catalog_name": { + "action": "update", + "old": "first-inference-catalog", + "new": "second-inference-catalog", + "remote": "first-inference-catalog" } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json index a770d28573..ea5d8e0de0 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/both_gateway_and_tags/out.plan.direct.json @@ -75,17 +75,17 @@ "endpoint_id": "[UUID]" }, "changes": { - "local": { - "ai_gateway.inference_table_config.catalog_name": { - "action": "update", - "old": "first-inference-catalog", - "new": "second-inference-catalog" - }, - "tags[0].value": { - "action": "update", - "old": "my-team-one", - "new": "my-team-two" - } + "ai_gateway.inference_table_config.catalog_name": { + "action": "update", + "old": "first-inference-catalog", + "new": "second-inference-catalog", + "remote": "first-inference-catalog" + }, + "tags[0].value": { + "action": "update", + "old": "my-team-one", + "new": "my-team-two", + "remote": "my-team-one" } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json index 590657ce93..d25eaa7f18 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/config/out.plan.direct.json @@ -53,12 +53,11 @@ "endpoint_id": "[UUID]" }, "changes": { - "local": { - "config.served_entities[0].external_model.name": { - "action": "update", - "old": "gpt-4o-mini", - "new": "gpt-5o-mini" - } + "config.served_entities[0].external_model.name": { + "action": "update", + "old": "gpt-4o-mini", + "new": "gpt-5o-mini", + "remote": "gpt-4o-mini" } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json index f020836d77..151dfaa734 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/email-notifications/out.plan.direct.json @@ -63,12 +63,11 @@ "endpoint_id": "[UUID]" }, "changes": { - "local": { - "email_notifications.on_update_success[0]": { - "action": "update", - "old": "user1@example.com", - "new": "user2@example.com" - } + "email_notifications.on_update_success[0]": { + "action": "update", + "old": "user1@example.com", + "new": "user2@example.com", + "remote": "user1@example.com" } } } diff --git a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json index fd346a57b2..4813ab01d2 100644 --- a/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json +++ b/acceptance/bundle/resources/model_serving_endpoints/update/tags/out.plan.direct.json @@ -65,12 +65,11 @@ "endpoint_id": "[UUID]" }, "changes": { - "local": { - "tags[0].value": { - "action": "update", - "old": "my-team-one", - "new": "my-team-two" - } + "tags[0].value": { + "action": "update", + "old": "my-team-one", + "new": "my-team-two", + "remote": "my-team-one" } } } diff --git a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json index 658ea6d45d..e105dd1d5d 100644 --- a/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/added_remotely/out.plan.direct.json @@ -38,19 +38,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -95,13 +96,11 @@ ] }, "changes": { - "remote": { - "permissions[group_name='admin-team']": { - "action": "update", - "new": { - "group_name": "admin-team", - "permission_level": "CAN_MANAGE" - } + "permissions[group_name='admin-team']": { + "action": "update", + "remote": { + "group_name": "admin-team", + "permission_level": "CAN_MANAGE" } } } diff --git a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt index 2459d1f037..ede9576900 100644 --- a/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt +++ b/acceptance/bundle/resources/permissions/jobs/current_can_manage_run/out.plan.direct.txt @@ -29,19 +29,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, diff --git a/acceptance/bundle/resources/permissions/jobs/delete_one/local/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/jobs/delete_one/local/out.plan_update.direct.json index aec39602c5..2a80f06cec 100644 --- a/acceptance/bundle/resources/permissions/jobs/delete_one/local/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/delete_one/local/out.plan_update.direct.json @@ -38,19 +38,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -103,13 +104,15 @@ ] }, "changes": { - "local": { - "permissions[user_name='test-dabs-1@databricks.com']": { - "action": "update", - "old": { - "permission_level": "CAN_MANAGE_RUN", - "user_name": "test-dabs-1@databricks.com" - } + "permissions[user_name='test-dabs-1@databricks.com']": { + "action": "update", + "old": { + "permission_level": "CAN_MANAGE_RUN", + "user_name": "test-dabs-1@databricks.com" + }, + "remote": { + "permission_level": "CAN_MANAGE_RUN", + "user_name": "test-dabs-1@databricks.com" } } } diff --git a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json index f043db44e3..070ceebb30 100644 --- a/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/deleted_remotely/out.plan_restore.direct.json @@ -38,19 +38,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -91,20 +92,26 @@ ] }, "changes": { - "remote": { - "permissions[group_name='data-team']": { - "action": "update", - "old": { - "group_name": "data-team", - "permission_level": "CAN_MANAGE" - } + "permissions[group_name='data-team']": { + "action": "update", + "old": { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" }, - "permissions[user_name='viewer@example.com']": { - "action": "update", - "old": { - "permission_level": "CAN_VIEW", - "user_name": "viewer@example.com" - } + "new": { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" + } + }, + "permissions[user_name='viewer@example.com']": { + "action": "update", + "old": { + "permission_level": "CAN_VIEW", + "user_name": "viewer@example.com" + }, + "new": { + "permission_level": "CAN_VIEW", + "user_name": "viewer@example.com" } } } diff --git a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.destroy.direct.txt b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.destroy.direct.txt index 097e49e239..bf1ebcb737 100644 --- a/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.destroy.direct.txt +++ b/acceptance/bundle/resources/permissions/jobs/destroy_without_mgmtperms/without_permissions/out.destroy.direct.txt @@ -1,6 +1,6 @@ >>> errcode as-test-sp [CLI] bundle destroy --auto-approve -Warn: cannot read resources.jobs.foo id="[NUMID]": User [UUID] does not have View or Admin or Manage Run or Owner permissions on job [NUMID] +Warn: reading resources.jobs.foo id="[NUMID]": User [UUID] does not have View or Admin or Manage Run or Owner permissions on job [NUMID] The following resources will be deleted: delete resources.jobs.foo diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json index ebf2076c02..47b02fce7b 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_all.direct.json @@ -38,19 +38,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json index 77069f6e70..dc804ff0e5 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_delete_one.direct.json @@ -38,19 +38,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -95,13 +96,15 @@ ] }, "changes": { - "local": { - "permissions[group_name='data-team']": { - "action": "update", - "old": { - "group_name": "data-team", - "permission_level": "CAN_MANAGE" - } + "permissions[group_name='data-team']": { + "action": "update", + "old": { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" + }, + "remote": { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" } } } diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json index 72a88065dc..9e7ed25c81 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_post_create.direct.json @@ -38,19 +38,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json index 8a34b48444..70770d4c75 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_restore.direct.json @@ -38,19 +38,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json index abf8a80a98..03eebc5cee 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_set_empty.direct.json @@ -38,19 +38,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, diff --git a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json index a33735e356..c329b7a518 100644 --- a/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/jobs/update/out.plan_update.direct.json @@ -38,19 +38,20 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -99,12 +100,11 @@ ] }, "changes": { - "local": { - "permissions[user_name='viewer@example.com'].permission_level": { - "action": "update", - "old": "CAN_VIEW", - "new": "CAN_MANAGE" - } + "permissions[user_name='viewer@example.com'].permission_level": { + "action": "update", + "old": "CAN_VIEW", + "new": "CAN_MANAGE", + "remote": "CAN_VIEW" } } } diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json index 1ae475a411..f3afa77b12 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_all.direct.json @@ -26,11 +26,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[PIPELINE_ID]" } } }, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json index a977ab2afc..147caaac46 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_delete_one.direct.json @@ -26,11 +26,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[PIPELINE_ID]" } } }, @@ -75,13 +74,15 @@ ] }, "changes": { - "local": { - "permissions[group_name='data-team']": { - "action": "update", - "old": { - "group_name": "data-team", - "permission_level": "CAN_MANAGE" - } + "permissions[group_name='data-team']": { + "action": "update", + "old": { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" + }, + "remote": { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" } } } diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json index 1edf12dfda..81dc62178f 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_restore.direct.json @@ -26,11 +26,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[PIPELINE_ID]" } } }, diff --git a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json index 051dcf6551..0f4eff10f1 100644 --- a/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json +++ b/acceptance/bundle/resources/permissions/pipelines/update/out.plan_update.direct.json @@ -26,11 +26,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[PIPELINE_ID]" } } }, @@ -79,12 +78,11 @@ ] }, "changes": { - "local": { - "permissions[user_name='viewer@example.com'].permission_level": { - "action": "update", - "old": "CAN_VIEW", - "new": "CAN_MANAGE" - } + "permissions[user_name='viewer@example.com'].permission_level": { + "action": "update", + "old": "CAN_VIEW", + "new": "CAN_MANAGE", + "remote": "CAN_VIEW" } } } diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json index e23ed9f2b2..13f73a44cd 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-ingestion-definition/out.plan_recreate.direct.json @@ -30,13 +30,50 @@ "name": "test-pipeline-[UNIQUE_NAME]" } }, + "remote_state": { + "creator_user_name": "[USERNAME]", + "last_modified": [UNIX_TIME_MILLIS], + "name": "test-pipeline-[UNIQUE_NAME]", + "pipeline_id": "[PIPELINE_ID_1]", + "run_as_user_name": "[USERNAME]", + "spec": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[PIPELINE_ID_1]", + "ingestion_definition": { + "connection_name": "my_connection", + "objects": [ + {} + ] + }, + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]", + "storage": "dbfs:/pipelines/[PIPELINE_ID_1]" + }, + "state": "IDLE" + }, "changes": { - "local": { - "ingestion_definition.connection_name": { - "action": "recreate", - "old": "my_connection", - "new": "my_new_connection" - } + "ingestion_definition.connection_name": { + "action": "recreate", + "reason": "field_triggers", + "old": "my_connection", + "new": "my_new_connection", + "remote": "my_connection" + }, + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[PIPELINE_ID_1]" } } } diff --git a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json index efa3c6effe..8b5cfaed44 100644 --- a/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json +++ b/acceptance/bundle/resources/pipelines/recreate-keys/change-storage/out.plan_recreate.direct.json @@ -25,13 +25,39 @@ "storage": "dbfs:/pipelines/newcustom" } }, + "remote_state": { + "creator_user_name": "[USERNAME]", + "last_modified": [UNIX_TIME_MILLIS], + "name": "test-pipeline-[UNIQUE_NAME]", + "pipeline_id": "[PIPELINE_ID_1]", + "run_as_user_name": "[USERNAME]", + "spec": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/state/metadata.json" + }, + "edition": "ADVANCED", + "id": "[PIPELINE_ID_1]", + "libraries": [ + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/acc-[UNIQUE_NAME]/default/files/foo.py" + } + } + ], + "name": "test-pipeline-[UNIQUE_NAME]", + "storage": "dbfs:/pipelines/custom" + }, + "state": "IDLE" + }, "changes": { - "local": { - "storage": { - "action": "recreate", - "old": "dbfs:/pipelines/custom", - "new": "dbfs:/pipelines/newcustom" - } + "storage": { + "action": "recreate", + "reason": "field_triggers", + "old": "dbfs:/pipelines/custom", + "new": "dbfs:/pipelines/newcustom", + "remote": "dbfs:/pipelines/custom" } } } diff --git a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.json b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.json index 3e3eea6df1..71880f848b 100644 --- a/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.json +++ b/acceptance/bundle/resources/secret_scopes/basic/out.plan2.direct.json @@ -12,13 +12,17 @@ "scope_backend_type": "DATABRICKS" } }, + "remote_state": { + "backend_type": "DATABRICKS", + "name": "test-scope-[UNIQUE_NAME]-1" + }, "changes": { - "local": { - "scope": { - "action": "recreate", - "old": "test-scope-[UNIQUE_NAME]-1", - "new": "test-scope-[UNIQUE_NAME]-2" - } + "scope": { + "action": "recreate", + "reason": "field_triggers", + "old": "test-scope-[UNIQUE_NAME]-1", + "new": "test-scope-[UNIQUE_NAME]-2", + "remote": "test-scope-[UNIQUE_NAME]-1" } } }, @@ -62,12 +66,12 @@ ] }, "changes": { - "local": { - "scope_name": { - "action": "update_id", - "old": "test-scope-[UNIQUE_NAME]-1", - "new": "" - } + "scope_name": { + "action": "update_id", + "reason": "field_triggers", + "old": "test-scope-[UNIQUE_NAME]-1", + "new": "", + "remote": "test-scope-[UNIQUE_NAME]-1" } } } diff --git a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json index f301e4cbe9..dd2dc4e73a 100644 --- a/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json +++ b/acceptance/bundle/resources/volumes/change-name/out.plan.direct.json @@ -29,18 +29,17 @@ "volume_type": "MANAGED" }, "changes": { - "local": { - "name": { - "action": "update_id", - "old": "myvolume", - "new": "mynewvolume" - } + "name": { + "action": "update_id", + "reason": "field_triggers", + "old": "myvolume", + "new": "mynewvolume", + "remote": "myvolume" }, - "remote": { - "storage_location": { - "action": "skip", - "reason": "server_side_default" - } + "storage_location": { + "action": "skip", + "reason": "server_side_default", + "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]" } } } diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json index e25ead31e5..085a3374ea 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_dev.direct.json @@ -217,35 +217,37 @@ } }, "changes": { - "local": { - "tasks[task_key='notebook_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - }, - "tasks[task_key='python_wheel_task'].libraries[0].whl": { - "action": "update", - "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", - "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} }, - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='notebook_task'].notebook_task.source": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "tasks[task_key='notebook_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default", + "remote": "WORKSPACE" + }, + "tasks[task_key='python_wheel_task'].libraries[0].whl": { + "action": "update", + "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl", + "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl", + "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -289,11 +291,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[UUID]" } } } diff --git a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json index 09b3a9209a..4c86fe7dae 100644 --- a/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json +++ b/acceptance/bundle/templates/default-python/classic/out.plan_after_deploy_prod.direct.json @@ -116,23 +116,25 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='notebook_task'].notebook_task.source": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default", + "remote": "WORKSPACE" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, @@ -190,11 +192,10 @@ "state": "IDLE" }, "changes": { - "remote": { - "storage": { - "action": "skip", - "reason": "server_side_default" - } + "storage": { + "action": "skip", + "reason": "server_side_default", + "remote": "dbfs:/pipelines/[UUID]" } } }, diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json index d0dd56c193..8326e0d8a4 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_dev.direct.json @@ -105,23 +105,25 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='notebook_task'].notebook_task.source": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default", + "remote": "WORKSPACE" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, diff --git a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json index d5ca3f2686..a3dcc69d26 100644 --- a/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json +++ b/acceptance/bundle/templates/default-python/serverless/out.plan_after_deploy_prod.direct.json @@ -102,23 +102,25 @@ } }, "changes": { - "remote": { - "email_notifications": { - "action": "skip", - "reason": "server_side_default" - }, - "tasks[task_key='notebook_task'].notebook_task.source": { - "action": "skip", - "reason": "server_side_default" - }, - "timeout_seconds": { - "action": "skip", - "reason": "server_side_default" - }, - "webhook_notifications": { - "action": "skip", - "reason": "server_side_default" - } + "email_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} + }, + "tasks[task_key='notebook_task'].notebook_task.source": { + "action": "skip", + "reason": "server_side_default", + "remote": "WORKSPACE" + }, + "timeout_seconds": { + "action": "skip", + "reason": "server_side_default", + "remote": 0 + }, + "webhook_notifications": { + "action": "skip", + "reason": "server_side_default", + "remote": {} } } }, diff --git a/bundle/deployplan/action.go b/bundle/deployplan/action.go index 150aeba689..6b561c18c4 100644 --- a/bundle/deployplan/action.go +++ b/bundle/deployplan/action.go @@ -38,16 +38,25 @@ const ( ActionTypeDelete ) -const ActionTypeSkipString = "skip" +const ( + ActionTypeUndefinedString = "" + ActionTypeSkipString = "skip" + ActionTypeResizeString = "resize" + ActionTypeUpdateString = "update" + ActionTypeUpdateWithIDString = "update_id" + ActionTypeCreateString = "create" + ActionTypeRecreateString = "recreate" + ActionTypeDeleteString = "delete" +) var actionName = map[ActionType]string{ ActionTypeSkip: ActionTypeSkipString, - ActionTypeResize: "resize", - ActionTypeUpdate: "update", - ActionTypeUpdateWithID: "update_id", - ActionTypeCreate: "create", - ActionTypeRecreate: "recreate", - ActionTypeDelete: "delete", + ActionTypeResize: ActionTypeResizeString, + ActionTypeUpdate: ActionTypeUpdateString, + ActionTypeUpdateWithID: ActionTypeUpdateWithIDString, + ActionTypeCreate: ActionTypeCreateString, + ActionTypeRecreate: ActionTypeRecreateString, + ActionTypeDelete: ActionTypeDeleteString, } var nameToAction = map[string]ActionType{} diff --git a/bundle/deployplan/plan.go b/bundle/deployplan/plan.go index eaf2d8e0d1..a134f00012 100644 --- a/bundle/deployplan/plan.go +++ b/bundle/deployplan/plan.go @@ -68,7 +68,7 @@ type PlanEntry struct { Action string `json:"action,omitempty"` NewState *structvar.StructVarJSON `json:"new_state,omitempty"` RemoteState any `json:"remote_state,omitempty"` - Changes *Changes `json:"changes,omitempty"` + Changes Changes `json:"changes,omitempty"` } type DependsOnEntry struct { @@ -76,18 +76,24 @@ type DependsOnEntry struct { Label string `json:"label,omitempty"` } -type Changes struct { - Local map[string]ChangeDesc `json:"local,omitempty"` - Remote map[string]ChangeDesc `json:"remote,omitempty"` -} +type Changes map[string]*ChangeDesc type ChangeDesc struct { Action string `json:"action"` Reason string `json:"reason,omitempty"` Old any `json:"old,omitempty"` New any `json:"new,omitempty"` + Remote any `json:"remote,omitempty"` } +// Possible values for Reason field +const ( + ReasonServerSideDefault = "server_side_default" + ReasonAlias = "alias" + ReasonRemoteAlreadySet = "remote_already_set" + ReasonFieldTriggers = "field_triggers" +) + // HasChange checks if there are any changes for fields with the given prefix. // This function is path-aware and correctly handles path component boundaries. // For example: @@ -100,13 +106,7 @@ func (c *Changes) HasChange(fieldPath string) bool { return false } - for field := range c.Local { - if structpath.HasPrefix(field, fieldPath) { - return true - } - } - - for field := range c.Remote { + for field := range *c { if structpath.HasPrefix(field, fieldPath) { return true } diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 92625c82e8..782ddc44db 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -27,7 +27,7 @@ func (d *DeploymentUnit) Destroy(ctx context.Context, db *dstate.DeploymentState return d.Delete(ctx, db, entry.ID) } -func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, newState any, actionType deployplan.ActionType, changes *deployplan.Changes) error { +func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState, newState any, actionType deployplan.ActionType, changes deployplan.Changes) error { if actionType == deployplan.ActionTypeCreate { return d.Create(ctx, db, newState) } @@ -103,7 +103,7 @@ func (d *DeploymentUnit) Recreate(ctx context.Context, db *dstate.DeploymentStat return d.Create(ctx, db, newState) } -func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, id string, newState any, changes *deployplan.Changes) error { +func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, id string, newState any, changes deployplan.Changes) error { if !d.Adapter.HasDoUpdate() { return fmt.Errorf("internal error: DoUpdate not implemented for resource %s", d.ResourceKey) } diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 560fa87442..0ef95df2ec 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -55,7 +55,7 @@ func ValidatePlanAgainstState(statePath string, plan *deployplan.Plan) error { if os.IsNotExist(err) { return fmt.Errorf("plan has lineage %q but state file does not exist at %s; the state may have been deleted", plan.Lineage, statePath) } - return fmt.Errorf("failed to read state from %s: %w", statePath, err) + return fmt.Errorf("reading state from %s: %w", statePath, err) } // Validate that the plan's lineage matches the current state's lineage @@ -76,7 +76,7 @@ func ValidatePlanAgainstState(statePath string, plan *deployplan.Plan) error { func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks.WorkspaceClient, statePath string, plan *deployplan.Plan) error { err := b.StateDB.Open(statePath) if err != nil { - return fmt.Errorf("failed to read state from %s: %w", statePath, err) + return fmt.Errorf("reading state from %s: %w", statePath, err) } err = b.init(client) @@ -94,12 +94,12 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. adapter, err := b.getAdapterForKey(resourceKey) if err != nil { - return fmt.Errorf("cannot convert plan entry %s: %w", resourceKey, err) + return fmt.Errorf("converting plan entry %s: %w", resourceKey, err) } sv, err := entry.NewState.ToStructVar(adapter.StateType()) if err != nil { - return fmt.Errorf("cannot load plan entry %s: %w", resourceKey, err) + return fmt.Errorf("loading plan entry %s: %w", resourceKey, err) } b.StructVarCache.Store(resourceKey, sv) @@ -109,10 +109,10 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. return nil } -func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath string) (*deployplan.Plan, error) { +func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath string, migrateMode MigrateMode) (*deployplan.Plan, error) { err := b.StateDB.Open(statePath) if err != nil { - return nil, fmt.Errorf("failed to read state from %s: %w", statePath, err) + return nil, fmt.Errorf("reading state from %s: %w", statePath, err) } err = b.init(client) @@ -143,29 +143,35 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks entry, err := plan.WriteLockEntry(resourceKey) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: %w", resourceKey, err)) + logdiag.LogError(ctx, fmt.Errorf("%s: internal error: %w", errorPrefix, err)) return false } if entry == nil { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: node not in graph", resourceKey)) + logdiag.LogError(ctx, fmt.Errorf("%s: internal error: node not in graph", errorPrefix)) return false } defer plan.WriteUnlockEntry(resourceKey) if failedDependency != nil { + // TODO: this should be a warning logdiag.LogError(ctx, fmt.Errorf("%s: dependency failed: %s", errorPrefix, *failedDependency)) return false } adapter, err := b.getAdapterForKey(resourceKey) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + logdiag.LogError(ctx, fmt.Errorf("%s: getting adapter: %w", errorPrefix, err)) return false } if entry.Action == deployplan.ActionTypeDelete.String() { + if migrateMode { + logdiag.LogError(ctx, fmt.Errorf("%s: is planned for deletion, cannot migrate. Must perform deployment first", errorPrefix)) + return false + } + dbentry, hasEntry := b.StateDB.GetResourceEntry(resourceKey) if !hasEntry { logdiag.LogError(ctx, fmt.Errorf("%s: internal error, missing in state", errorPrefix)) @@ -178,7 +184,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // no such resource plan.RemoveEntry(resourceKey) } else { - log.Warnf(ctx, "cannot read %s id=%q: %s", resourceKey, dbentry.ID, err) + log.Warnf(ctx, "reading %s id=%q: %s", resourceKey, dbentry.ID, err) return false } } @@ -194,6 +200,11 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } + if migrateMode { + entry.Action = deployplan.ActionTypeUpdateString + return true + } + dbentry, hasEntry := b.StateDB.GetResourceEntry(resourceKey) if !hasEntry { entry.Action = deployplan.ActionTypeCreate.String() @@ -233,48 +244,53 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks if isResourceGone(err) { remoteState = nil } else { - logdiag.LogError(ctx, fmt.Errorf("%s: failed to read id=%q: %w", errorPrefix, dbentry.ID, err)) + logdiag.LogError(ctx, fmt.Errorf("%s: reading id=%q: %w", errorPrefix, dbentry.ID, err)) return false } } - localAction, localChangeMap := localChangesToTriggers(ctx, adapter, localDiff, remoteState) - if localAction == deployplan.ActionTypeRecreate { - entry.Action = localAction.String() - if len(localChangeMap) > 0 { - entry.Changes = &deployplan.Changes{ - Local: localChangeMap, - } - } - return true - } - // We have a choice whether to include remoteState or remoteStateComparable from below. // Including remoteState because in the near future remoteState is expected to become a superset struct of remoteStateComparable entry.RemoteState = remoteState - var remoteAction deployplan.ActionType - var remoteChangeMap map[string]deployplan.ChangeDesc + action := deployplan.ActionTypeSkip + var remoteDiff []structdiff.Change + var remoteStateComparable any - if remoteState == nil { - remoteAction = deployplan.ActionTypeCreate - } else { - remoteStateComparable, err := adapter.RemapState(remoteState) + if remoteState != nil { + remoteStateComparable, err = adapter.RemapState(remoteState) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: failed to interpret remote state id=%q: %w", errorPrefix, dbentry.ID, err)) + logdiag.LogError(ctx, fmt.Errorf("%s: interpreting remote state id=%q: %w", errorPrefix, dbentry.ID, err)) return false } - remoteDiff, err := structdiff.GetStructDiff(savedState, remoteStateComparable, adapter.KeyedSlices()) + remoteDiff, err = structdiff.GetStructDiff(remoteStateComparable, sv.Value, adapter.KeyedSlices()) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: diffing remote state: %w", errorPrefix, err)) return false } + } + + entry.Changes, err = prepareChanges(ctx, adapter, localDiff, remoteDiff, savedState, remoteState != nil) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + return false + } + + err = addPerFieldActions(ctx, adapter, entry.Changes, remoteState) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) + return false + } - remoteAction, remoteChangeMap = interpretOldStateVsRemoteState(ctx, adapter, remoteDiff, remoteState) + if remoteState == nil { + // Even if local action is "recreate" which is higher than "create", we should still pick "create" here + // because we know remote does not exist. + action = deployplan.ActionTypeCreate + } else { + action = getMaxAction(entry.Changes) } - action := max(localAction, remoteAction) if action == deployplan.ActionTypeSkip { // resource is not going to change, can use remoteState to resolve references b.RemoteStateCache.Store(resourceKey, remoteState) @@ -287,14 +303,6 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } entry.Action = action.String() - - if len(localChangeMap) > 0 || len(remoteChangeMap) > 0 { - entry.Changes = &deployplan.Changes{ - Local: localChangeMap, - Remote: remoteChangeMap, - } - } - return true }) @@ -311,68 +319,92 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return plan, nil } -func localChangesToTriggers(ctx context.Context, adapter *dresources.Adapter, diff []structdiff.Change, remoteState any) (deployplan.ActionType, map[string]deployplan.ChangeDesc) { - action := deployplan.ActionTypeSkip - var m map[string]deployplan.ChangeDesc +func getMaxAction(m map[string]*deployplan.ChangeDesc) deployplan.ActionType { + result := deployplan.ActionTypeSkip + for _, ch := range m { + result = max(result, deployplan.ActionTypeFromString(ch.Action)) + } + return result +} - for _, ch := range diff { - fieldAction, err := adapter.ClassifyChange(ch, remoteState, true) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("internal error: failed to classify change: %w", err)) - continue - } - if fieldAction > action { - action = fieldAction +func prepareChanges(ctx context.Context, adapter *dresources.Adapter, localDiff, remoteDiff []structdiff.Change, oldState any, hasRemote bool) (deployplan.Changes, error) { + m := make(deployplan.Changes) + + for _, ch := range localDiff { + e := deployplan.ChangeDesc{ + Old: ch.Old, + New: ch.New, } - if m == nil { - m = make(map[string]deployplan.ChangeDesc) + if hasRemote { + // by default, assume e.Remote is the same as config; if not the case it'll be ovewritten below + e.Remote = ch.New } - m[ch.Path.String()] = deployplan.ChangeDesc{ - Action: fieldAction.String(), - Old: ch.Old, - New: ch.New, + m[ch.Path.String()] = &e + } + + for _, ch := range remoteDiff { + entry := m[ch.Path.String()] + if entry == nil { + // we have difference for remoteState but not difference for localState + // from remoteDiff we can find out remote value (ch.Old) and new config value (ch.New) but we don't know oldState value + oldStateVal, err := structaccess.Get(oldState, ch.Path) + var notFound *structaccess.NotFoundError + if errors.As(err, ¬Found) { + oldStateVal = nil + } else if err != nil { + return nil, fmt.Errorf("accessing %q on %T: %w", ch.Path, oldState, err) + } + m[ch.Path.String()] = &deployplan.ChangeDesc{ + Old: oldStateVal, + New: ch.New, + Remote: ch.Old, + } + } else { + entry.Remote = ch.Old + if !structdiff.IsEqual(entry.New, ch.New) { + // this is not fatal (may result in unexpected drift or undetected change but not incorrect deploy), but good to log this + log.Warnf(ctx, "unexpected local and remote diffs (%T, %T); entry=%v ch=%v", entry.New, ch.New, entry, ch) + } } } - return action, m + return m, nil } -func interpretOldStateVsRemoteState(ctx context.Context, adapter *dresources.Adapter, diff []structdiff.Change, remoteState any) (deployplan.ActionType, map[string]deployplan.ChangeDesc) { - action := deployplan.ActionTypeSkip - m := make(map[string]deployplan.ChangeDesc) +func addPerFieldActions(ctx context.Context, adapter *dresources.Adapter, changes deployplan.Changes, remoteState any) error { + fieldTriggers := adapter.FieldTriggers() + + for pathString, ch := range changes { + path, err := structpath.Parse(pathString) + if err != nil { + return err + } - for _, ch := range diff { - if ch.Old == nil && ch.Path.IsDotString() { + if ch.New == nil && ch.Old == nil && ch.Remote != nil && path.IsDotString() { // The field was not set by us, but comes from the remote state. // This could either be server-side default or a policy. // In any case, this is not a change we should react to. // Note, we only consider struct fields here. Adding/removing elements to/from maps and slices should trigger updates. - m[ch.Path.String()] = deployplan.ChangeDesc{ - Action: deployplan.ActionTypeSkipString, - Reason: "server_side_default", - } - continue + ch.Action = deployplan.ActionTypeSkipString + ch.Reason = deployplan.ReasonServerSideDefault + } else if structdiff.IsEqual(ch.Remote, ch.New) { + ch.Action = deployplan.ActionTypeSkipString + ch.Reason = deployplan.ReasonRemoteAlreadySet + } else if action, ok := fieldTriggers[pathString]; ok { + // TODO: should we check prefixes instead? + ch.Action = action.String() + ch.Reason = deployplan.ReasonFieldTriggers + } else { + ch.Action = deployplan.ActionTypeUpdateString } - fieldAction, err := adapter.ClassifyChange(ch, remoteState, false) + + err = adapter.OverrideChangeDesc(ctx, path, ch, remoteState) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("internal error: failed to classify change: %w", err)) - continue - } - if fieldAction > action { - action = fieldAction - } - m[ch.Path.String()] = deployplan.ChangeDesc{ - Action: fieldAction.String(), - Old: ch.Old, - New: ch.New, + return fmt.Errorf("internal error: failed to classify change: %w", err) } } - if len(m) == 0 { - m = nil - } - - return action, m + return nil } // TODO: calling this "Local" is not right, it can resolve "id" and remote refrences for "skip" targets diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index 71279b8706..aeb5505280 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -9,10 +9,15 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/calladapt" - "github.com/databricks/cli/libs/structs/structdiff" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" ) +type ( + Changes = deployplan.Changes + ChangeDesc = deployplan.ChangeDesc +) + // IResource describes core methods for the resource implementation. // The resources don't actually implement this interface, but implement the same methods with signatures with concrete types. // The resources need to have all of the methods on IResource that are not marked [Optional]. @@ -45,18 +50,12 @@ type IResource interface { // Keys are field paths (e.g., "name", "catalog_name"). Values are actions. // Unspecified changed fields default to ActionTypeUpdate. // - // FieldTriggers(true) is applied on every change between state (last deployed config) - // and new state (current config) to determine action based on config changes. - // - // FieldTriggers(false) is called on every change between state and remote state to - // determine action based on remote drift. - // // Note: these functions are called once per resource implementation initialization, // not once per resource. - FieldTriggers(isLocal bool) map[string]deployplan.ActionType - // [Optional] ClassifyChange classifies a change using custom logic. - // The isLocal parameter indicates whether this is a local change (true) or remote change (false). - ClassifyChange(change structdiff.Change, remoteState any, isLocal bool) (deployplan.ActionType, error) + FieldTriggers() map[string]deployplan.ActionType + + // [Optional] OverrideChangeDesc can implement custom logic to update a given ChangeDesc; it is run last after built-in classifiers and field triggers. + OverrideChangeDesc(ctx context.Context, path *structpath.PathNode, changedesc *ChangeDesc, remoteState any) error // DoCreate creates a new resource from the newState. Returns id of the resource and optionally remote state. // If remote state is available as part of the operation, return it; otherwise return nil. @@ -65,8 +64,8 @@ type IResource interface { // [Optional] DoUpdate updates the resource. ID must not change as a result of this operation. Returns optionally remote state. // If remote state is available as part of the operation, return it; otherwise return nil. - // Example: func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, newState *catalog.CreateSchema, changes *deployplan.Changes) (*catalog.SchemaInfo, error) - DoUpdate(ctx context.Context, id string, newState any, changes *deployplan.Changes) (remoteState any, e error) + // Example: func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, newState *catalog.CreateSchema, changes Changes) (*catalog.SchemaInfo, error) + DoUpdate(ctx context.Context, id string, newState any, changes Changes) (remoteState any, e error) // [Optional] DoUpdateWithID performs an update that may result in resource having a new ID. Returns new id and optionally remote state. DoUpdateWithID(ctx context.Context, id string, newState any) (newID string, remoteState any, e error) @@ -97,16 +96,15 @@ type Adapter struct { doCreate *calladapt.BoundCaller // Optional: - doUpdate *calladapt.BoundCaller - doUpdateWithID *calladapt.BoundCaller - waitAfterCreate *calladapt.BoundCaller - waitAfterUpdate *calladapt.BoundCaller - classifyChange *calladapt.BoundCaller - doResize *calladapt.BoundCaller - - fieldTriggersLocal map[string]deployplan.ActionType - fieldTriggersRemote map[string]deployplan.ActionType - keyedSlices map[string]any + doUpdate *calladapt.BoundCaller + doUpdateWithID *calladapt.BoundCaller + waitAfterCreate *calladapt.BoundCaller + waitAfterUpdate *calladapt.BoundCaller + overrideChangeDesc *calladapt.BoundCaller + doResize *calladapt.BoundCaller + + fieldTriggers map[string]deployplan.ActionType + keyedSlices map[string]any } func NewAdapter(typedNil any, client *databricks.WorkspaceClient) (*Adapter, error) { @@ -123,20 +121,19 @@ func NewAdapter(typedNil any, client *databricks.WorkspaceClient) (*Adapter, err } impl := outs[0] adapter := &Adapter{ - prepareState: nil, - remapState: nil, - doRefresh: nil, - doDelete: nil, - doCreate: nil, - doUpdate: nil, - doUpdateWithID: nil, - doResize: nil, - waitAfterCreate: nil, - waitAfterUpdate: nil, - classifyChange: nil, - fieldTriggersLocal: map[string]deployplan.ActionType{}, - fieldTriggersRemote: map[string]deployplan.ActionType{}, - keyedSlices: nil, + prepareState: nil, + remapState: nil, + doRefresh: nil, + doDelete: nil, + doCreate: nil, + doUpdate: nil, + doUpdateWithID: nil, + doResize: nil, + waitAfterCreate: nil, + waitAfterUpdate: nil, + overrideChangeDesc: nil, + fieldTriggers: map[string]deployplan.ActionType{}, + keyedSlices: nil, } err = adapter.initMethods(impl) @@ -151,25 +148,7 @@ func NewAdapter(typedNil any, client *databricks.WorkspaceClient) (*Adapter, err } if triggerCall != nil { // Validate FieldTriggers signature: func(bool) map[string]deployplan.ActionType - if len(triggerCall.InTypes) != 1 || triggerCall.InTypes[0] != reflect.TypeOf(false) { - return nil, errors.New("FieldTriggers must take a single bool parameter (isLocal)") - } - if len(triggerCall.OutTypes) != 1 { - return nil, errors.New("FieldTriggers must return a single value") - } - expectedReturnType := reflect.TypeOf(map[string]deployplan.ActionType{}) - if triggerCall.OutTypes[0] != expectedReturnType { - return nil, fmt.Errorf("FieldTriggers must return map[string]deployplan.ActionType, got %v", triggerCall.OutTypes[0]) - } - - // Call with isLocal=true for local triggers - adapter.fieldTriggersLocal, err = loadFieldTriggers(triggerCall, true) - if err != nil { - return nil, err - } - - // Call with isLocal=false for remote triggers - adapter.fieldTriggersRemote, err = loadFieldTriggers(triggerCall, false) + adapter.fieldTriggers, err = loadFieldTriggers(triggerCall) if err != nil { return nil, err } @@ -184,10 +163,10 @@ func NewAdapter(typedNil any, client *databricks.WorkspaceClient) (*Adapter, err } // loadFieldTriggers calls FieldTriggers with isLocal parameter and returns the resulting map. -func loadFieldTriggers(triggerCall *calladapt.BoundCaller, isLocal bool) (map[string]deployplan.ActionType, error) { - outs, err := triggerCall.Call(isLocal) +func loadFieldTriggers(triggerCall *calladapt.BoundCaller) (map[string]deployplan.ActionType, error) { + outs, err := triggerCall.Call() if err != nil || len(outs) != 1 { - return nil, fmt.Errorf("failed to call FieldTriggers(%v): %w", isLocal, err) + return nil, fmt.Errorf("failed to call FieldTriggers(): %w", err) } fields := outs[0].(map[string]deployplan.ActionType) result := make(map[string]deployplan.ActionType, len(fields)) @@ -258,7 +237,7 @@ func (a *Adapter) initMethods(resource any) error { return err } - a.classifyChange, err = calladapt.PrepareCall(resource, calladapt.TypeOf[IResource](), "ClassifyChange") + a.overrideChangeDesc, err = calladapt.PrepareCall(resource, calladapt.TypeOf[IResource](), "OverrideChangeDesc") if err != nil { return err } @@ -375,36 +354,31 @@ func (a *Adapter) validate() error { validations = append(validations, "WaitAfterUpdate remoteState return", a.waitAfterUpdate.OutTypes[0], remoteType) } - if a.classifyChange != nil { - validations = append(validations, - "ClassifyChange remoteState", a.classifyChange.InTypes[1], remoteType, - "ClassifyChange isLocal", a.classifyChange.InTypes[2], reflect.TypeOf(false), - ) - } - err = validateTypes(validations...) if err != nil { return err } // FieldTriggers validation - hasUpdateWithIDTrigger := false - for _, action := range a.fieldTriggersLocal { - if action == deployplan.ActionTypeUpdateWithID { - hasUpdateWithIDTrigger = true + /* + hasUpdateWithIDTrigger := false + for _, action := range a.fieldTriggersLocal { + if action == deployplan.ActionTypeUpdateWithID { + hasUpdateWithIDTrigger = true + } } - } - for _, action := range a.fieldTriggersRemote { - if action == deployplan.ActionTypeUpdateWithID { - hasUpdateWithIDTrigger = true + for _, action := range a.fieldTriggersRemote { + if action == deployplan.ActionTypeUpdateWithID { + hasUpdateWithIDTrigger = true + } } - } - if hasUpdateWithIDTrigger && a.doUpdateWithID == nil { - return errors.New("FieldTriggers includes update_with_id but DoUpdateWithID is not implemented") - } - if a.doUpdateWithID != nil && !hasUpdateWithIDTrigger { - return errors.New("DoUpdateWithID is implemented but FieldTriggers lacks update_with_id trigger") - } + if hasUpdateWithIDTrigger && a.doUpdateWithID == nil { + return errors.New("FieldTriggers includes update_with_id but DoUpdateWithID is not implemented") + } + if a.doUpdateWithID != nil && !hasUpdateWithIDTrigger { + return errors.New("DoUpdateWithID is implemented but FieldTriggers lacks update_with_id trigger") + } + */ return nil } @@ -421,6 +395,10 @@ func (a *Adapter) RemoteType() reflect.Type { return a.doRefresh.OutTypes[0] } +func (a *Adapter) FieldTriggers() map[string]deployplan.ActionType { + return a.fieldTriggers +} + func (a *Adapter) PrepareState(input any) (any, error) { outs, err := a.prepareState.Call(input) if err != nil { @@ -488,7 +466,7 @@ func (a *Adapter) HasDoUpdate() bool { // DoUpdate updates the resource with information about changes computed during plan. // Returns remote state if available, otherwise nil. -func (a *Adapter) DoUpdate(ctx context.Context, id string, newState any, changes *deployplan.Changes) (any, error) { +func (a *Adapter) DoUpdate(ctx context.Context, id string, newState any, changes Changes) (any, error) { if a.doUpdate == nil { return nil, errors.New("internal error: DoUpdate not found") } @@ -532,6 +510,7 @@ func (a *Adapter) DoResize(ctx context.Context, id string, newState any) error { return err } +/* // classifyByTriggers classifies a change using FieldTriggers. // Defaults to ActionTypeUpdate. // The isLocal parameter determines which trigger map to use: @@ -551,6 +530,7 @@ func (a *Adapter) classifyByTriggers(change structdiff.Change, isLocal bool) dep } return deployplan.ActionTypeUpdate } +*/ // WaitAfterCreate waits for the resource to become ready after creation. // If the resource doesn't implement this method, this is a no-op. @@ -587,26 +567,12 @@ func (a *Adapter) WaitAfterUpdate(ctx context.Context, newState any) (any, error } // ClassifyChange classifies a change using custom logic or FieldTriggers. -// The isLocal parameter determines whether this is a local or remote change: -// - isLocal=true: classifying local changes (user modifications) -// - isLocal=false: classifying remote changes (drift detection) -func (a *Adapter) ClassifyChange(change structdiff.Change, remoteState any, isLocal bool) (deployplan.ActionType, error) { - actionType := deployplan.ActionTypeUndefined - - // If ClassifyChange is implemented, use it. - if a.classifyChange != nil { - outs, err := a.classifyChange.Call(change, remoteState, isLocal) - if err != nil { - return deployplan.ActionTypeUndefined, err - } - actionType = outs[0].(deployplan.ActionType) - } - - // If ClassifyChange is not implemented or is implemented but returns undefined, use FieldTriggers. - if actionType == deployplan.ActionTypeUndefined { - return a.classifyByTriggers(change, isLocal), nil +func (a *Adapter) OverrideChangeDesc(ctx context.Context, path *structpath.PathNode, change *ChangeDesc, remoteState any) error { + if a.overrideChangeDesc == nil { + return nil } - return actionType, nil + _, err := a.overrideChangeDesc.Call(ctx, path, change, remoteState) + return err } // KeyedSlices returns a map from path patterns to KeyFunc for comparing slices by key. diff --git a/bundle/direct/dresources/alert.go b/bundle/direct/dresources/alert.go index 5613649b9b..a8e31c136c 100644 --- a/bundle/direct/dresources/alert.go +++ b/bundle/direct/dresources/alert.go @@ -4,13 +4,10 @@ import ( "context" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/sql" ) -type Changes = deployplan.Changes - type ResourceAlert struct { client *databricks.WorkspaceClient } @@ -52,7 +49,7 @@ func (r *ResourceAlert) DoCreate(ctx context.Context, config *sql.AlertV2) (stri } // DoUpdate updates the alert in place. -func (r *ResourceAlert) DoUpdate(ctx context.Context, id string, config *sql.AlertV2, _ *Changes) (*sql.AlertV2, error) { +func (r *ResourceAlert) DoUpdate(ctx context.Context, id string, config *sql.AlertV2, _ Changes) (*sql.AlertV2, error) { request := sql.UpdateAlertV2Request{ Id: id, Alert: *config, diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 69c9fa4241..e1ba1a7fbf 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -612,21 +612,10 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W err = adapter.DoDelete(ctx, createdID) require.NoError(t, err) - path, err := structpath.Parse("name") + p, err := structpath.Parse("name") require.NoError(t, err) - _, err = adapter.ClassifyChange(structdiff.Change{ - Path: path, - Old: nil, - New: "mynewname", - }, remote, true) - require.NoError(t, err) - - _, err = adapter.ClassifyChange(structdiff.Change{ - Path: path, - Old: nil, - New: "mynewname", - }, remote, false) + err = adapter.OverrideChangeDesc(ctx, p, &deployplan.ChangeDesc{}, nil) require.NoError(t, err) deleteIsNoop := strings.HasSuffix(group, "permissions") || strings.HasSuffix(group, "grants") @@ -665,10 +654,7 @@ func TestFieldTriggers(t *testing.T) { require.NoError(t, err) t.Run(resourceName+"_local", func(t *testing.T) { - validateFields(t, adapter.StateType(), adapter.fieldTriggersLocal) - }) - t.Run(resourceName+"_remote", func(t *testing.T) { - validateFields(t, adapter.StateType(), adapter.fieldTriggersRemote) + validateFields(t, adapter.StateType(), adapter.FieldTriggers()) }) } } @@ -685,14 +671,7 @@ func TestFieldTriggersNoUpdateWhenNotImplemented(t *testing.T) { } t.Run(resourceName+"_local", func(t *testing.T) { - for field, action := range adapter.fieldTriggersLocal { - assert.NotEqual(t, deployplan.ActionTypeUpdate, action, - "resource %s does not implement DoUpdate but field %s triggers update action", resourceName, field) - } - }) - - t.Run(resourceName+"_remote", func(t *testing.T) { - for field, action := range adapter.fieldTriggersRemote { + for field, action := range adapter.FieldTriggers() { assert.NotEqual(t, deployplan.ActionTypeUpdate, action, "resource %s does not implement DoUpdate but field %s triggers update action", resourceName, field) } diff --git a/bundle/direct/dresources/app.go b/bundle/direct/dresources/app.go index 2de5af879c..8bc2580870 100644 --- a/bundle/direct/dresources/app.go +++ b/bundle/direct/dresources/app.go @@ -69,7 +69,7 @@ func (r *ResourceApp) DoCreate(ctx context.Context, config *apps.App) (string, * return app.Name, nil, nil } -func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *apps.App, _ *Changes) (*apps.App, error) { +func (r *ResourceApp) DoUpdate(ctx context.Context, id string, config *apps.App, _ Changes) (*apps.App, error) { request := apps.UpdateAppRequest{ App: *config, Name: id, @@ -91,7 +91,7 @@ func (r *ResourceApp) DoDelete(ctx context.Context, id string) error { return err } -func (*ResourceApp) FieldTriggers(_ bool) map[string]deployplan.ActionType { +func (*ResourceApp) FieldTriggers() map[string]deployplan.ActionType { return map[string]deployplan.ActionType{ "name": deployplan.ActionTypeRecreate, } diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 01e2b3a15f..2c33476a69 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -4,12 +4,11 @@ import ( "context" "errors" "fmt" - "strings" "time" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/structs/structdiff" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -85,7 +84,7 @@ func (r *ResourceCluster) DoCreate(ctx context.Context, config *compute.ClusterS return wait.ClusterId, nil, nil } -func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *compute.ClusterSpec, _ *Changes) (*compute.ClusterDetails, error) { +func (r *ResourceCluster) DoUpdate(ctx context.Context, id string, config *compute.ClusterSpec, _ Changes) (*compute.ClusterDetails, error) { // Same retry as in TF provider logic // https://github.com/databricks/terraform-provider-databricks/blob/3eecd0f90cf99d7777e79a3d03c41f9b2aafb004/clusters/resource_cluster.go#L624 timeout := 15 * time.Minute @@ -120,32 +119,31 @@ func (r *ResourceCluster) DoDelete(ctx context.Context, id string) error { return r.client.Clusters.PermanentDeleteByClusterId(ctx, id) } -func (r *ResourceCluster) ClassifyChange(change structdiff.Change, remoteState *compute.ClusterDetails, isLocal bool) (deployplan.ActionType, error) { - changedPath := change.Path.String() - if changedPath == "data_security_mode" && !isLocal { +func (r *ResourceCluster) OverrideChangeDesc(ctx context.Context, p *structpath.PathNode, change *ChangeDesc, remoteState *compute.ClusterDetails) error { + path := p.Prefix(1).String() + switch path { + case "data_security_mode": // We do change skip here in the same way TF provider does suppress diff if the alias is used. // https://github.com/databricks/terraform-provider-databricks/blob/main/clusters/resource_cluster.go#L109-L117 - if change.Old == compute.DataSecurityModeDataSecurityModeStandard && change.New == compute.DataSecurityModeUserIsolation { - return deployplan.ActionTypeSkip, nil + if change.Remote == compute.DataSecurityModeDataSecurityModeStandard && change.Old == compute.DataSecurityModeUserIsolation && change.New == change.Old { + change.Action = deployplan.ActionTypeSkipString + change.Reason = deployplan.ReasonAlias } - if change.Old == compute.DataSecurityModeDataSecurityModeDedicated && change.New == compute.DataSecurityModeSingleUser { - return deployplan.ActionTypeSkip, nil + if change.Remote == compute.DataSecurityModeDataSecurityModeDedicated && change.Old == compute.DataSecurityModeSingleUser && change.New == change.Old { + change.Action = deployplan.ActionTypeSkipString + change.Reason = deployplan.ReasonAlias } - if change.Old == compute.DataSecurityModeDataSecurityModeAuto && (change.New == compute.DataSecurityModeSingleUser || change.New == compute.DataSecurityModeUserIsolation) { - return deployplan.ActionTypeSkip, nil + if change.Remote == compute.DataSecurityModeDataSecurityModeAuto && (change.Old == compute.DataSecurityModeSingleUser || change.Old == compute.DataSecurityModeUserIsolation) && change.New == change.Old { + change.Action = deployplan.ActionTypeSkipString + change.Reason = deployplan.ReasonAlias } - } - - // Always update if the cluster is not running. - if remoteState.State != compute.StateRunning { - return deployplan.ActionTypeUpdate, nil - } - if changedPath == "num_workers" || strings.HasPrefix(changedPath, "autoscale") { - return deployplan.ActionTypeResize, nil + case "num_workers", "autoscale": + if remoteState.State == compute.StateRunning { + change.Action = deployplan.ActionTypeResizeString + } } - - return deployplan.ActionTypeUpdate, nil + return nil } func makeCreateCluster(config *compute.ClusterSpec) compute.CreateCluster { diff --git a/bundle/direct/dresources/dashboard.go b/bundle/direct/dresources/dashboard.go index 626c1285f2..b74c643445 100644 --- a/bundle/direct/dresources/dashboard.go +++ b/bundle/direct/dresources/dashboard.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" @@ -266,7 +267,7 @@ func (r *ResourceDashboard) DoCreate(ctx context.Context, config *resources.Dash return createResp.DashboardId, responseToState(createResp, publishResp, dashboard.SerializedDashboard), nil } -func (r *ResourceDashboard) DoUpdate(ctx context.Context, id string, config *resources.DashboardConfig, _ *Changes) (*resources.DashboardConfig, error) { +func (r *ResourceDashboard) DoUpdate(ctx context.Context, id string, config *resources.DashboardConfig, _ Changes) (*resources.DashboardConfig, error) { dashboard, err := prepareDashboardRequest(config) if err != nil { return nil, err @@ -301,33 +302,31 @@ func (r *ResourceDashboard) DoDelete(ctx context.Context, id string) error { }) } -func (*ResourceDashboard) FieldTriggers(isLocal bool) map[string]deployplan.ActionType { - // Common triggers for both local and remote. - triggers := map[string]deployplan.ActionType{ - // change in parent_path should trigger a recreate - "parent_path": deployplan.ActionTypeRecreate, - } - - if isLocal { - // Etags should only be compared for remote diffs, not local diffs. - triggers["etag"] = deployplan.ActionTypeSkip - } else { - // Note: If the etag changes remotely, it means the dashboard has been modified remotely - // and needs to be updated to match with the config. Since update is the default action type, - // we don't need to explicitly specify it here. - // - // Note: etags cannot be specified in the bundle config since we error if they are set. - +func (r *ResourceDashboard) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *resources.DashboardConfig) error { + switch path.String() { + case "etag": + // change.New is always nil for etag because it's not present in the config + // We compare stored etag with remote one. + if change.Old == change.Remote { + change.Action = deployplan.ActionTypeSkipString + } else { + change.Action = deployplan.ActionTypeUpdateString + } + case "parent_path": + if change.Action == deployplan.ActionTypeUpdateString { + change.Action = deployplan.ActionTypeRecreateString + } + case "serialized_dashboard", "dataset_catalog", "dataset_schema": // "serialized_dashboard" locally and remotely will have different diffs. // We only need to rely on etag here, and can skip this field for diff computation. - triggers["serialized_dashboard"] = deployplan.ActionTypeSkip // "dataset_catalog" and "dataset_schema" are write-only fields that are not returned by the server. // They will always differ between local config (which has values) and remote state (which has empty strings), // so we skip them for remote diff computation to avoid false positives. - triggers["dataset_catalog"] = deployplan.ActionTypeSkip - triggers["dataset_schema"] = deployplan.ActionTypeSkip + if change.Old == change.New { + change.Action = deployplan.ActionTypeSkipString + } } - return triggers + return nil } diff --git a/bundle/direct/dresources/database_catalog.go b/bundle/direct/dresources/database_catalog.go index 475a80f763..fdf09fbac9 100644 --- a/bundle/direct/dresources/database_catalog.go +++ b/bundle/direct/dresources/database_catalog.go @@ -34,7 +34,7 @@ func (r *ResourceDatabaseCatalog) DoCreate(ctx context.Context, config *database return result.Name, nil, nil } -func (r *ResourceDatabaseCatalog) DoUpdate(ctx context.Context, id string, config *database.DatabaseCatalog, _ *Changes) (*database.DatabaseCatalog, error) { +func (r *ResourceDatabaseCatalog) DoUpdate(ctx context.Context, id string, config *database.DatabaseCatalog, _ Changes) (*database.DatabaseCatalog, error) { request := database.UpdateDatabaseCatalogRequest{ DatabaseCatalog: *config, Name: id, diff --git a/bundle/direct/dresources/database_instance.go b/bundle/direct/dresources/database_instance.go index 7ed2c08306..39f74caa04 100644 --- a/bundle/direct/dresources/database_instance.go +++ b/bundle/direct/dresources/database_instance.go @@ -35,7 +35,7 @@ func (d *ResourceDatabaseInstance) DoCreate(ctx context.Context, config *databas return waiter.Response.Name, nil, nil } -func (d *ResourceDatabaseInstance) DoUpdate(ctx context.Context, id string, config *database.DatabaseInstance, _ *Changes) (*database.DatabaseInstance, error) { +func (d *ResourceDatabaseInstance) DoUpdate(ctx context.Context, id string, config *database.DatabaseInstance, _ Changes) (*database.DatabaseInstance, error) { request := database.UpdateDatabaseInstanceRequest{ DatabaseInstance: *config, Name: config.Name, diff --git a/bundle/direct/dresources/experiment.go b/bundle/direct/dresources/experiment.go index b2f5effbf5..1cce7d31a8 100644 --- a/bundle/direct/dresources/experiment.go +++ b/bundle/direct/dresources/experiment.go @@ -56,7 +56,7 @@ func (r *ResourceExperiment) DoCreate(ctx context.Context, config *ml.CreateExpe return result.ExperimentId, nil, nil } -func (r *ResourceExperiment) DoUpdate(ctx context.Context, id string, config *ml.CreateExperiment, _ *Changes) (*ml.Experiment, error) { +func (r *ResourceExperiment) DoUpdate(ctx context.Context, id string, config *ml.CreateExperiment, _ Changes) (*ml.Experiment, error) { updateReq := ml.UpdateExperiment{ ExperimentId: id, NewName: config.Name, @@ -72,7 +72,7 @@ func (r *ResourceExperiment) DoDelete(ctx context.Context, id string) error { }) } -func (*ResourceExperiment) FieldTriggers(_ bool) map[string]deployplan.ActionType { +func (*ResourceExperiment) FieldTriggers() map[string]deployplan.ActionType { // TF implementation: https://github.com/databricks/terraform-provider-databricks/blob/6c106e8e7052bb2726148d66309fd460ed444236/mlflow/resource_mlflow_experiment.go#L22 return map[string]deployplan.ActionType{ "name": deployplan.ActionTypeUpdate, diff --git a/bundle/direct/dresources/grants.go b/bundle/direct/dresources/grants.go index 121aaf6084..1f6b8678e5 100644 --- a/bundle/direct/dresources/grants.go +++ b/bundle/direct/dresources/grants.go @@ -145,7 +145,7 @@ func (r *ResourceGrants) DoCreate(ctx context.Context, state *GrantsState) (stri return makeGrantsID(state.SecurableType, state.FullName), nil, nil } -func (r *ResourceGrants) DoUpdate(ctx context.Context, _ string, state *GrantsState, _ *Changes) (*GrantsState, error) { +func (r *ResourceGrants) DoUpdate(ctx context.Context, _ string, state *GrantsState, _ Changes) (*GrantsState, error) { return nil, r.applyGrants(ctx, state) } diff --git a/bundle/direct/dresources/job.go b/bundle/direct/dresources/job.go index c3a668409e..e132ad0234 100644 --- a/bundle/direct/dresources/job.go +++ b/bundle/direct/dresources/job.go @@ -59,7 +59,7 @@ func (r *ResourceJob) DoCreate(ctx context.Context, config *jobs.JobSettings) (s return strconv.FormatInt(response.JobId, 10), nil, nil } -func (r *ResourceJob) DoUpdate(ctx context.Context, id string, config *jobs.JobSettings, _ *Changes) (*jobs.Job, error) { +func (r *ResourceJob) DoUpdate(ctx context.Context, id string, config *jobs.JobSettings, _ Changes) (*jobs.Job, error) { request, err := makeResetJob(*config, id) if err != nil { return nil, err diff --git a/bundle/direct/dresources/model.go b/bundle/direct/dresources/model.go index b1e2d62851..2b290d79c5 100644 --- a/bundle/direct/dresources/model.go +++ b/bundle/direct/dresources/model.go @@ -68,7 +68,7 @@ func (r *ResourceMlflowModel) DoCreate(ctx context.Context, config *ml.CreateMod return response.RegisteredModel.Name, modelDatabricks, nil } -func (r *ResourceMlflowModel) DoUpdate(ctx context.Context, id string, config *ml.CreateModelRequest, _ *Changes) (*ml.ModelDatabricks, error) { +func (r *ResourceMlflowModel) DoUpdate(ctx context.Context, id string, config *ml.CreateModelRequest, _ Changes) (*ml.ModelDatabricks, error) { updateRequest := ml.UpdateModelRequest{ Name: id, Description: config.Description, @@ -108,7 +108,7 @@ func (r *ResourceMlflowModel) DoDelete(ctx context.Context, id string) error { }) } -func (*ResourceMlflowModel) FieldTriggers(_ bool) map[string]deployplan.ActionType { +func (*ResourceMlflowModel) FieldTriggers() map[string]deployplan.ActionType { return map[string]deployplan.ActionType{ // Recreate matches current behavior of Terraform. It is possible to rename without recreate // but that would require dynamic select of the method during update since diff --git a/bundle/direct/dresources/model_serving_endpoint.go b/bundle/direct/dresources/model_serving_endpoint.go index 6795df6407..dbf7af4e00 100644 --- a/bundle/direct/dresources/model_serving_endpoint.go +++ b/bundle/direct/dresources/model_serving_endpoint.go @@ -276,7 +276,7 @@ func (r *ResourceModelServingEndpoint) updateTags(ctx context.Context, id string return nil } -func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, config *serving.CreateServingEndpoint, changes *Changes) (*RefreshOutput, error) { +func (r *ResourceModelServingEndpoint) DoUpdate(ctx context.Context, id string, config *serving.CreateServingEndpoint, changes Changes) (*RefreshOutput, error) { var err error // Terraform makes these API calls sequentially. We do the same here. @@ -317,7 +317,7 @@ func (r *ResourceModelServingEndpoint) DoDelete(ctx context.Context, id string) return r.client.ServingEndpoints.DeleteByName(ctx, id) } -func (*ResourceModelServingEndpoint) FieldTriggers(_ bool) map[string]deployplan.ActionType { +func (*ResourceModelServingEndpoint) FieldTriggers() map[string]deployplan.ActionType { // TF implementation: https://github.com/databricks/terraform-provider-databricks/blob/6c106e8e7052bb2726148d66309fd460ed444236/mlflow/resource_mlflow_experiment.go#L22 return map[string]deployplan.ActionType{ "name": deployplan.ActionTypeRecreate, diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 86e054bc5a..553f6b4e0b 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -175,7 +175,7 @@ func (r *ResourcePermissions) DoCreate(ctx context.Context, newState *Permission } // DoUpdate calls https://docs.databricks.com/api/workspace/jobs/setjobpermissions. -func (r *ResourcePermissions) DoUpdate(ctx context.Context, _ string, newState *PermissionsState, _ *Changes) (*PermissionsState, error) { +func (r *ResourcePermissions) DoUpdate(ctx context.Context, _ string, newState *PermissionsState, _ Changes) (*PermissionsState, error) { extractedType, extractedID, err := parsePermissionsID(newState.ObjectID) if err != nil { return nil, err diff --git a/bundle/direct/dresources/pipeline.go b/bundle/direct/dresources/pipeline.go index cffcdc7d8c..5355a607c3 100644 --- a/bundle/direct/dresources/pipeline.go +++ b/bundle/direct/dresources/pipeline.go @@ -5,6 +5,8 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structdiff" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/pipelines" @@ -76,7 +78,7 @@ func (r *ResourcePipeline) DoCreate(ctx context.Context, config *pipelines.Creat return response.PipelineId, nil, nil } -func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *pipelines.CreatePipeline, _ *Changes) (*pipelines.GetPipelineResponse, error) { +func (r *ResourcePipeline) DoUpdate(ctx context.Context, id string, config *pipelines.CreatePipeline, _ Changes) (*pipelines.GetPipelineResponse, error) { request := pipelines.EditPipeline{ AllowDuplicateNames: config.AllowDuplicateNames, BudgetPolicyId: config.BudgetPolicyId, @@ -120,22 +122,25 @@ func (r *ResourcePipeline) DoDelete(ctx context.Context, id string) error { return r.client.Pipelines.DeleteByPipelineId(ctx, id) } -func (*ResourcePipeline) FieldTriggers(isLocal bool) map[string]deployplan.ActionType { +func (*ResourcePipeline) FieldTriggers() map[string]deployplan.ActionType { result := map[string]deployplan.ActionType{ "storage": deployplan.ActionTypeRecreate, "ingestion_definition.connection_name": deployplan.ActionTypeRecreate, "ingestion_definition.ingestion_gateway_id": deployplan.ActionTypeRecreate, } - if !isLocal { - // We've seen that run_as is not consistently set by the backend - // TF also ignores it (by not copying it here: - // https://github.com/databricks/terraform-provider-databricks/blob/15e951f976e3857d9f01651c4b2513657f137796/pipelines/resource_pipeline.go#L304-L317 - // TODO: Rather than skipping, consider ignoring the change if run_as is not present but still triggering an update if run_as is different. - result["run_as"] = deployplan.ActionTypeSkip + return result +} + +func (*ResourcePipeline) OverrideChangeDesc(ctx context.Context, path *structpath.PathNode, ch *ChangeDesc, _ *pipelines.GetPipelineResponse) error { + if path.String() == "run_as" { + if structdiff.IsEqual(ch.Old, ch.New) { + ch.Action = deployplan.ActionTypeSkipString + ch.Reason = "override" + } } - return result + return nil } // Note, terraform provider either diff --git a/bundle/direct/dresources/registered_model.go b/bundle/direct/dresources/registered_model.go index f1e719fa7b..cd2feaf696 100644 --- a/bundle/direct/dresources/registered_model.go +++ b/bundle/direct/dresources/registered_model.go @@ -63,7 +63,7 @@ func (r *ResourceRegisteredModel) DoCreate(ctx context.Context, config *catalog. return response.FullName, response, nil } -func (r *ResourceRegisteredModel) DoUpdate(ctx context.Context, id string, config *catalog.CreateRegisteredModelRequest, _ *Changes) (*catalog.RegisteredModelInfo, error) { +func (r *ResourceRegisteredModel) DoUpdate(ctx context.Context, id string, config *catalog.CreateRegisteredModelRequest, _ Changes) (*catalog.RegisteredModelInfo, error) { updateRequest := catalog.UpdateRegisteredModelRequest{ FullName: id, Comment: config.Comment, @@ -103,7 +103,7 @@ func (r *ResourceRegisteredModel) DoDelete(ctx context.Context, id string) error }) } -func (*ResourceRegisteredModel) FieldTriggers(_ bool) map[string]deployplan.ActionType { +func (*ResourceRegisteredModel) FieldTriggers() map[string]deployplan.ActionType { return map[string]deployplan.ActionType{ // The name can technically be updated without recreated. We recreate for now though // to match TF implementation. diff --git a/bundle/direct/dresources/schema.go b/bundle/direct/dresources/schema.go index 4a3ea6d363..1c09817617 100644 --- a/bundle/direct/dresources/schema.go +++ b/bundle/direct/dresources/schema.go @@ -47,7 +47,7 @@ func (r *ResourceSchema) DoCreate(ctx context.Context, config *catalog.CreateSch } // DoUpdate updates the schema in place and returns remote state. -func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalog.CreateSchema, _ *Changes) (*catalog.SchemaInfo, error) { +func (r *ResourceSchema) DoUpdate(ctx context.Context, id string, config *catalog.CreateSchema, _ Changes) (*catalog.SchemaInfo, error) { updateRequest := catalog.UpdateSchema{ Comment: config.Comment, EnablePredictiveOptimization: "", // Not supported by DABs @@ -78,7 +78,7 @@ func (r *ResourceSchema) DoDelete(ctx context.Context, id string) error { }) } -func (*ResourceSchema) FieldTriggers(_ bool) map[string]deployplan.ActionType { +func (*ResourceSchema) FieldTriggers() map[string]deployplan.ActionType { return map[string]deployplan.ActionType{ "name": deployplan.ActionTypeRecreate, "catalog_name": deployplan.ActionTypeRecreate, diff --git a/bundle/direct/dresources/secret_scope.go b/bundle/direct/dresources/secret_scope.go index 271aa38040..6aa960cdc9 100644 --- a/bundle/direct/dresources/secret_scope.go +++ b/bundle/direct/dresources/secret_scope.go @@ -83,7 +83,7 @@ func (r *ResourceSecretScope) DoDelete(ctx context.Context, id string) error { return r.client.Secrets.DeleteScopeByScope(ctx, id) } -func (r *ResourceSecretScope) FieldTriggers(_ bool) map[string]deployplan.ActionType { +func (r *ResourceSecretScope) FieldTriggers() map[string]deployplan.ActionType { return map[string]deployplan.ActionType{ "scope": deployplan.ActionTypeRecreate, "scope_backend_type": deployplan.ActionTypeRecreate, diff --git a/bundle/direct/dresources/secret_scope_acls.go b/bundle/direct/dresources/secret_scope_acls.go index f6702a84c2..4c110a60f3 100644 --- a/bundle/direct/dresources/secret_scope_acls.go +++ b/bundle/direct/dresources/secret_scope_acls.go @@ -110,12 +110,12 @@ func (r *ResourceSecretScopeAcls) DoUpdateWithID(ctx context.Context, id string, return state.ScopeName, nil, nil } -func (r *ResourceSecretScopeAcls) DoUpdate(ctx context.Context, id string, state *SecretScopeAclsState, changes *deployplan.Changes) (*SecretScopeAclsState, error) { +func (r *ResourceSecretScopeAcls) DoUpdate(ctx context.Context, id string, state *SecretScopeAclsState, changes Changes) (*SecretScopeAclsState, error) { _, _, err := r.DoUpdateWithID(ctx, id, state) return nil, err } -func (r *ResourceSecretScopeAcls) FieldTriggers(isLocal bool) map[string]deployplan.ActionType { +func (r *ResourceSecretScopeAcls) FieldTriggers() map[string]deployplan.ActionType { // When scope name changes, we need a DoUpdateWithID trigger. This is necessary so that subsequent // DoRead operations use the correct ID and we do not end up with a persistent drift. return map[string]deployplan.ActionType{ diff --git a/bundle/direct/dresources/sql_warehouse.go b/bundle/direct/dresources/sql_warehouse.go index 43bf2bc4d1..5d9d7793b7 100644 --- a/bundle/direct/dresources/sql_warehouse.go +++ b/bundle/direct/dresources/sql_warehouse.go @@ -58,7 +58,7 @@ func (r *ResourceSqlWarehouse) DoCreate(ctx context.Context, config *sql.CreateW } // DoUpdate updates the warehouse in place. -func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *sql.CreateWarehouseRequest, _ *Changes) (*sql.GetWarehouseResponse, error) { +func (r *ResourceSqlWarehouse) DoUpdate(ctx context.Context, id string, config *sql.CreateWarehouseRequest, _ Changes) (*sql.GetWarehouseResponse, error) { request := sql.EditWarehouseRequest{ AutoStopMins: config.AutoStopMins, Channel: config.Channel, diff --git a/bundle/direct/dresources/synced_database_table.go b/bundle/direct/dresources/synced_database_table.go index 3a88bab3af..d0cb4987b9 100644 --- a/bundle/direct/dresources/synced_database_table.go +++ b/bundle/direct/dresources/synced_database_table.go @@ -34,7 +34,7 @@ func (r *ResourceSyncedDatabaseTable) DoCreate(ctx context.Context, config *data return result.Name, nil, nil } -func (r *ResourceSyncedDatabaseTable) DoUpdate(ctx context.Context, id string, config *database.SyncedDatabaseTable, _ *Changes) (*database.SyncedDatabaseTable, error) { +func (r *ResourceSyncedDatabaseTable) DoUpdate(ctx context.Context, id string, config *database.SyncedDatabaseTable, _ Changes) (*database.SyncedDatabaseTable, error) { request := database.UpdateSyncedDatabaseTableRequest{ SyncedTable: *config, Name: id, diff --git a/bundle/direct/dresources/volume.go b/bundle/direct/dresources/volume.go index 66ee22da17..6c490e5d15 100644 --- a/bundle/direct/dresources/volume.go +++ b/bundle/direct/dresources/volume.go @@ -49,7 +49,7 @@ func (r *ResourceVolume) DoCreate(ctx context.Context, config *catalog.CreateVol return response.FullName, response, nil } -func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, _ *Changes) (*catalog.VolumeInfo, error) { +func (r *ResourceVolume) DoUpdate(ctx context.Context, id string, config *catalog.CreateVolumeRequestContent, _ Changes) (*catalog.VolumeInfo, error) { updateRequest := catalog.UpdateVolumeRequestContent{ Comment: config.Comment, Name: id, @@ -113,7 +113,7 @@ func (r *ResourceVolume) DoDelete(ctx context.Context, id string) error { return r.client.Volumes.DeleteByName(ctx, id) } -func (*ResourceVolume) FieldTriggers(_ bool) map[string]deployplan.ActionType { +func (*ResourceVolume) FieldTriggers() map[string]deployplan.ActionType { return map[string]deployplan.ActionType{ "catalog_name": deployplan.ActionTypeRecreate, "schema_name": deployplan.ActionTypeRecreate, diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 065e2f46dd..16bd0c6e1e 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -214,7 +214,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *deployplan.Plan { if engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, localPath) + plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, localPath, direct.MigrateMode(false)) if err != nil { logdiag.LogError(ctx, err) return nil diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index e66d2f5e6a..7b656d7961 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -158,7 +158,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { var plan *deployplan.Plan if engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), nil, localPath) + plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), nil, localPath, direct.MigrateMode(false)) if err != nil { logdiag.LogError(ctx, err) return diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index aedc851ecf..b5efd4e9b5 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -224,11 +224,20 @@ To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env va } }() - plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, tempStatePath) + plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, tempStatePath, direct.MigrateMode(true)) if err != nil { return err } + for key, value := range plan.Plan { + if value.Action == "skip" { + logdiag.LogError(ctx, fmt.Errorf("unexpected planned action: %q for %q during migration", value.Action, key)) + } + } + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + // We need to copy ETag into new state. // For most resources state consists of fully resolved local config snapshot + id. // Dashboards are special in that they also store "etag" in state which is not provided by user but diff --git a/libs/calladapt/calladapt.go b/libs/calladapt/calladapt.go index bdb549f709..e99dfd743a 100644 --- a/libs/calladapt/calladapt.go +++ b/libs/calladapt/calladapt.go @@ -52,6 +52,11 @@ func (c *BoundCaller) call(args ...any) ([]reflect.Value, error) { for i, a := range args { it := c.InTypes[i] if a == nil { + // Allow untyped nil for pointer types, converting to typed nil + if it.Kind() == reflect.Ptr { + in[i+1] = reflect.Zero(it) + continue + } return nil, &CallAdaptError{Msg: fmt.Sprintf("%s: arg %d type mismatch: want %v, got nil", c.Name, i, it)} } v := reflect.ValueOf(a) diff --git a/libs/calladapt/calladapt_test.go b/libs/calladapt/calladapt_test.go index e88cc90e2b..9f5a1484c5 100644 --- a/libs/calladapt/calladapt_test.go +++ b/libs/calladapt/calladapt_test.go @@ -307,7 +307,7 @@ func TestCall(t *testing.T) { ifaceType: TypeOf[interface{ PMethodTransformDataPtr(data *Data) (any, error) }](), method: "PMethodTransformDataPtr", args: []any{nil}, - errMsg: "PMethodTransformDataPtr: arg 0 type mismatch: want *calladapt.Data, got nil", + errMsg: "data is nil", }, { name: "void method call returns no outs", From abab8534a87ba50b8f810e50dc8e01f78b75ce6c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 6 Jan 2026 19:19:43 +0100 Subject: [PATCH 2/9] clean up --- bundle/direct/dresources/adapter.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index aeb5505280..bdcab1231a 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -510,28 +510,6 @@ func (a *Adapter) DoResize(ctx context.Context, id string, newState any) error { return err } -/* -// classifyByTriggers classifies a change using FieldTriggers. -// Defaults to ActionTypeUpdate. -// The isLocal parameter determines which trigger map to use: -// - isLocal=true uses triggers from FieldTriggers(true) -// - isLocal=false uses triggers from FieldTriggers(false) -func (a *Adapter) classifyByTriggers(change structdiff.Change, isLocal bool) deployplan.ActionType { - var triggers map[string]deployplan.ActionType - if isLocal { - triggers = a.fieldTriggersLocal - } else { - triggers = a.fieldTriggersRemote - } - - action, ok := triggers[change.Path.String()] - if ok { - return action - } - return deployplan.ActionTypeUpdate -} -*/ - // WaitAfterCreate waits for the resource to become ready after creation. // If the resource doesn't implement this method, this is a no-op. // Returns the updated remoteState if available, otherwise returns nil From c86cb785491380512be86063aa14942fc4fe2d5f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 6 Jan 2026 19:29:50 +0100 Subject: [PATCH 3/9] fix --- .../dataset-catalog-schema/out.plan.direct.json | 8 ++------ .../dashboards/dataset-catalog-schema/out.test.toml | 1 - .../resources/dashboards/dataset-catalog-schema/script | 2 +- .../resources/dashboards/dataset-catalog-schema/test.toml | 5 ----- 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json index d87fcba0bc..777de21433 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.plan.direct.json @@ -37,12 +37,8 @@ }, "serialized_dashboard": { "action": "skip", - "old": "{\n \"pages\": [\n {\n \"name\": \"test_page\",\n \"displayName\": \"Test Page\",\n \"pageType\": \"PAGE_TYPE_CANVAS\"\n }\n ],\n \"datasets\": [\n {\n \"name\": \"bf8f76f4\",\n \"displayName\": \"Test Dataset\",\n \"queryLines\": [\n \"SELECT 1\\n\"\n ],\n \"catalog\": \"foobar\",\n \"schema\": \"foobar\"\n }\n ]\n}\n", - "new": "{\n \"pages\": [\n {\n \"name\": \"test_page\",\n \"displayName\": \"Test Page\",\n \"pageType\": \"PAGE_TYPE_CANVAS\"\n }\n ],\n \"datasets\": [\n {\n \"name\": \"bf8f76f4\",\n \"displayName\": \"Test Dataset\",\n \"queryLines\": [\n \"SELECT 1\\n\"\n ],\n \"catalog\": \"foobar\",\n \"schema\": \"foobar\"\n }\n ]\n}\n", - "remote": "{\"datasets\":[{\"catalog\":\"main\",\"displayName\":\"Test Dataset\",\"name\":\"bf8f76f4\",\"queryLines\":[\"SELECT 1\\n\"],\"schema\":\"default\"}],\"pages\":[{\"displayName\":\"Test Page\",\"name\":\"test_page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n" - }, - "remote": { - "serialized_dashboard": null + "old": "[SERIALIZED_FIXTURE_OLD]", + "new": "[SERIALIZED_FIXTURE_NEW]" } } } diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml index dbea8ee3ac..f53dec026c 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/out.test.toml @@ -4,4 +4,3 @@ RequiresWarehouse = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] - Ignore = ["databricks.yml"] diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script index f17a675801..e4c6e6dfec 100755 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/script @@ -35,7 +35,7 @@ if [ "$DATABRICKS_BUNDLE_ENGINE" = "direct" ]; then jq '.plan["resources.dashboards.dashboard1"].remote_state.serialized_dashboard |= if . then "[SERIALIZED_FIXTURE]" else . end' \ out.plan.direct.json > out.plan.direct.json.tmp && mv out.plan.direct.json.tmp out.plan.direct.json - jq '.plan["resources.dashboards.dashboard1"].changes.remote.serialized_dashboard |= + jq '.plan["resources.dashboards.dashboard1"].changes.serialized_dashboard |= if . then {"action": .action, "old": "[SERIALIZED_FIXTURE_OLD]", "new": "[SERIALIZED_FIXTURE_NEW]"} else . end' \ out.plan.direct.json > out.plan.direct.json.tmp && mv out.plan.direct.json.tmp out.plan.direct.json fi diff --git a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/test.toml b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/test.toml index 269db2d0f6..effbc20992 100644 --- a/acceptance/bundle/resources/dashboards/dataset-catalog-schema/test.toml +++ b/acceptance/bundle/resources/dashboards/dataset-catalog-schema/test.toml @@ -2,11 +2,6 @@ Local = true Cloud = true RequiresWarehouse = true RecordRequests = true -EnvVaryOutput = "DATABRICKS_BUNDLE_ENGINE" [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] - -Ignore = [ - "databricks.yml", -] From 28eb3e54400daed4d5084887de2f48e72dfb6de4 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 6 Jan 2026 19:40:11 +0100 Subject: [PATCH 4/9] cluster fix --- bundle/direct/dresources/cluster.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/bundle/direct/dresources/cluster.go b/bundle/direct/dresources/cluster.go index 2c33476a69..f099d5d0df 100644 --- a/bundle/direct/dresources/cluster.go +++ b/bundle/direct/dresources/cluster.go @@ -125,15 +125,13 @@ func (r *ResourceCluster) OverrideChangeDesc(ctx context.Context, p *structpath. case "data_security_mode": // We do change skip here in the same way TF provider does suppress diff if the alias is used. // https://github.com/databricks/terraform-provider-databricks/blob/main/clusters/resource_cluster.go#L109-L117 - if change.Remote == compute.DataSecurityModeDataSecurityModeStandard && change.Old == compute.DataSecurityModeUserIsolation && change.New == change.Old { + if change.New == compute.DataSecurityModeDataSecurityModeStandard && change.Remote == compute.DataSecurityModeUserIsolation && change.New == change.Old { change.Action = deployplan.ActionTypeSkipString change.Reason = deployplan.ReasonAlias - } - if change.Remote == compute.DataSecurityModeDataSecurityModeDedicated && change.Old == compute.DataSecurityModeSingleUser && change.New == change.Old { + } else if change.New == compute.DataSecurityModeDataSecurityModeDedicated && change.Remote == compute.DataSecurityModeSingleUser && change.New == change.Old { change.Action = deployplan.ActionTypeSkipString change.Reason = deployplan.ReasonAlias - } - if change.Remote == compute.DataSecurityModeDataSecurityModeAuto && (change.Old == compute.DataSecurityModeSingleUser || change.Old == compute.DataSecurityModeUserIsolation) && change.New == change.Old { + } else if change.New == compute.DataSecurityModeDataSecurityModeAuto && (change.Remote == compute.DataSecurityModeSingleUser || change.Remote == compute.DataSecurityModeUserIsolation) && change.New == change.Old { change.Action = deployplan.ActionTypeSkipString change.Reason = deployplan.ReasonAlias } From 3c19eeaf605dd250d3760cc3da4d1cc68e226a92 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Jan 2026 12:11:52 +0100 Subject: [PATCH 5/9] dont error when constructing diff --- bundle/direct/bundle_plan.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 0ef95df2ec..2e459c9ad3 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -349,10 +349,8 @@ func prepareChanges(ctx context.Context, adapter *dresources.Adapter, localDiff, // from remoteDiff we can find out remote value (ch.Old) and new config value (ch.New) but we don't know oldState value oldStateVal, err := structaccess.Get(oldState, ch.Path) var notFound *structaccess.NotFoundError - if errors.As(err, ¬Found) { - oldStateVal = nil - } else if err != nil { - return nil, fmt.Errorf("accessing %q on %T: %w", ch.Path, oldState, err) + if err != nil && !errors.As(err, ¬Found) { + log.Debugf(ctx, "Constructing diff: accessing %q on %T: %w", ch.Path, oldState, err) } m[ch.Path.String()] = &deployplan.ChangeDesc{ Old: oldStateVal, From 2c47c61fc2a3f9f14faa6c8ff51d7da2b50e9b25 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Jan 2026 12:21:45 +0100 Subject: [PATCH 6/9] fix Debugf %w --- bundle/direct/bundle_plan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 2e459c9ad3..849591e1bb 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -350,7 +350,7 @@ func prepareChanges(ctx context.Context, adapter *dresources.Adapter, localDiff, oldStateVal, err := structaccess.Get(oldState, ch.Path) var notFound *structaccess.NotFoundError if err != nil && !errors.As(err, ¬Found) { - log.Debugf(ctx, "Constructing diff: accessing %q on %T: %w", ch.Path, oldState, err) + log.Debugf(ctx, "Constructing diff: accessing %q on %T: %s", ch.Path, oldState, err) } m[ch.Path.String()] = &deployplan.ChangeDesc{ Old: oldStateVal, From 0eb4398fd1ccb619779ba702fca2132388c7c0d3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Jan 2026 12:45:30 +0100 Subject: [PATCH 7/9] clean up; force 'update' action in migrate --- bundle/direct/bundle_plan.go | 12 +----------- bundle/phases/deploy.go | 2 +- bundle/phases/destroy.go | 2 +- cmd/bundle/deployment/migrate.go | 13 +++++-------- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 849591e1bb..d634fa460f 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -109,7 +109,7 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. return nil } -func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath string, migrateMode MigrateMode) (*deployplan.Plan, error) { +func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root, statePath string) (*deployplan.Plan, error) { err := b.StateDB.Open(statePath) if err != nil { return nil, fmt.Errorf("reading state from %s: %w", statePath, err) @@ -167,11 +167,6 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks } if entry.Action == deployplan.ActionTypeDelete.String() { - if migrateMode { - logdiag.LogError(ctx, fmt.Errorf("%s: is planned for deletion, cannot migrate. Must perform deployment first", errorPrefix)) - return false - } - dbentry, hasEntry := b.StateDB.GetResourceEntry(resourceKey) if !hasEntry { logdiag.LogError(ctx, fmt.Errorf("%s: internal error, missing in state", errorPrefix)) @@ -200,11 +195,6 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } - if migrateMode { - entry.Action = deployplan.ActionTypeUpdateString - return true - } - dbentry, hasEntry := b.StateDB.GetResourceEntry(resourceKey) if !hasEntry { entry.Action = deployplan.ActionTypeCreate.String() diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 16bd0c6e1e..065e2f46dd 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -214,7 +214,7 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *deployplan.Plan { if engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, localPath, direct.MigrateMode(false)) + plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, localPath) if err != nil { logdiag.LogError(ctx, err) return nil diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index 7b656d7961..e66d2f5e6a 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -158,7 +158,7 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { var plan *deployplan.Plan if engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) - plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), nil, localPath, direct.MigrateMode(false)) + plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), nil, localPath) if err != nil { logdiag.LogError(ctx, err) return diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index b5efd4e9b5..13d7053430 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/cmd/bundle/utils" @@ -224,18 +225,14 @@ To start using direct engine, deploy with DATABRICKS_BUNDLE_ENGINE=direct env va } }() - plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, tempStatePath, direct.MigrateMode(true)) + plan, err := deploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(), &b.Config, tempStatePath) if err != nil { return err } - for key, value := range plan.Plan { - if value.Action == "skip" { - logdiag.LogError(ctx, fmt.Errorf("unexpected planned action: %q for %q during migration", value.Action, key)) - } - } - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + for _, entry := range plan.Plan { + // Force all actions to be "update" so that deploym below goes through every resource + entry.Action = deployplan.ActionTypeUpdateString } // We need to copy ETag into new state. From 447be8332f4401f7843dec75a4b996aa84396d9f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Jan 2026 13:14:35 +0100 Subject: [PATCH 8/9] clean up --- bundle/direct/dresources/adapter.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/bundle/direct/dresources/adapter.go b/bundle/direct/dresources/adapter.go index bdcab1231a..6e3c213c56 100644 --- a/bundle/direct/dresources/adapter.go +++ b/bundle/direct/dresources/adapter.go @@ -360,14 +360,9 @@ func (a *Adapter) validate() error { } // FieldTriggers validation - /* + if a.overrideChangeDesc == nil { hasUpdateWithIDTrigger := false - for _, action := range a.fieldTriggersLocal { - if action == deployplan.ActionTypeUpdateWithID { - hasUpdateWithIDTrigger = true - } - } - for _, action := range a.fieldTriggersRemote { + for _, action := range a.fieldTriggers { if action == deployplan.ActionTypeUpdateWithID { hasUpdateWithIDTrigger = true } @@ -378,7 +373,7 @@ func (a *Adapter) validate() error { if a.doUpdateWithID != nil && !hasUpdateWithIDTrigger { return errors.New("DoUpdateWithID is implemented but FieldTriggers lacks update_with_id trigger") } - */ + } return nil } From 86e256120444bdc52bf01e4a3931f21a0c798e05 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 7 Jan 2026 14:33:04 +0100 Subject: [PATCH 9/9] update NEXT CHangelog --- NEXT_CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index ed03a4adc7..47195e3c70 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -3,6 +3,7 @@ ## Release v0.282.0 ### Notable Changes +* engine/direct: Plan format changed to 2, see ([#4201](https://github.com/databricks/cli/pull/4201)) ### CLI * Skip non-exportable objects (e.g., `MLFLOW_EXPERIMENT`) during `workspace export-dir` instead of failing ([#4081](https://github.com/databricks/cli/issues/4081)) @@ -11,10 +12,11 @@ ### Bundles * Pass SYSTEM_ACCESSTOKEN from env to the Terraform provider ([#4135](https://github.com/databricks/cli/pull/4135)) * Added missing schema grants privileges ([#4139](https://github.com/databricks/cli/pull/4139)) -* Fix app deployment failure when app is in `DELETING` state ([#4176](https://github.com/databricks/cli/pull/4176)) * Add `ipykernel` to the `default` template to enable Databricks Connect notebooks in Cursor/VS Code ([#4164](https://github.com/databricks/cli/pull/4164)) * Add interactive SQL warehouse picker to `default-sql` and `dbt-sql` bundle templates ([#4170](https://github.com/databricks/cli/pull/4170)) * Add `name`, `target` and `mode` fields to the deployment metadata file ([#4180](https://github.com/databricks/cli/pull/4180)) +* engine/direct: Fix app deployment failure when app is in `DELETING` state ([#4176](https://github.com/databricks/cli/pull/4176)) +* engine/direct: Changes in config that match remote changes no longer trigger an update ([#4201](https://github.com/databricks/cli/pull/4201)) ### Dependency updates