direct: New plan format and logic re remote changes

NEXT_CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Release v0.282.0
 
 ### Notable Changes
+* engine/direct: Plan format changed to 2, see ([#4201](https://github.com/databricks/cli/pull/4201))
 
 ### CLI
 * Skip non-exportable objects (e.g., `MLFLOW_EXPERIMENT`) during `workspace export-dir` instead of failing ([#4081](https://github.com/databricks/cli/issues/4081))
@@ -11,10 +12,11 @@
 ### Bundles
 * Pass SYSTEM_ACCESSTOKEN from env to the Terraform provider ([#4135](https://github.com/databricks/cli/pull/4135))
 * Added missing schema grants privileges ([#4139](https://github.com/databricks/cli/pull/4139))
-* Fix app deployment failure when app is in `DELETING` state ([#4176](https://github.com/databricks/cli/pull/4176))
 * Add `ipykernel` to the `default` template to enable Databricks Connect notebooks in Cursor/VS Code ([#4164](https://github.com/databricks/cli/pull/4164))
 * Add interactive SQL warehouse picker to `default-sql` and `dbt-sql` bundle templates ([#4170](https://github.com/databricks/cli/pull/4170))
 * Add `name`, `target` and `mode` fields to the deployment metadata file ([#4180](https://github.com/databricks/cli/pull/4180))
+* engine/direct: Fix app deployment failure when app is in `DELETING` state ([#4176](https://github.com/databricks/cli/pull/4176))
+* engine/direct: Changes in config that match remote changes no longer trigger an update ([#4201](https://github.com/databricks/cli/pull/4201))
 
 ### Dependency updates
 

acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json

@@ -150,36 +150,38 @@
         }
       },
       "changes": {
-        "local": {
-          "environments[0].spec.dependencies[0]": {
-            "action": "update",
-            "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl",
-            "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
-          },
-          "tasks[task_key='TestTask'].for_each_task.task.libraries[0].whl": {
-            "action": "update",
-            "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl",
-            "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
-          },
-          "tasks[task_key='TestTask'].libraries[0].whl": {
-            "action": "update",
-            "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl",
-            "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
-          }
+        "email_notifications": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": {}
         },
-        "remote": {
-          "email_notifications": {
-            "action": "skip",
-            "reason": "server_side_default"
-          },
-          "timeout_seconds": {
-            "action": "skip",
-            "reason": "server_side_default"
-          },
-          "webhook_notifications": {
-            "action": "skip",
-            "reason": "server_side_default"
-          }
+        "environments[0].spec.dependencies[0]": {
+          "action": "update",
+          "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl",
+          "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl",
+          "remote": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
+        },
+        "tasks[task_key='TestTask'].for_each_task.task.libraries[0].whl": {
+          "action": "update",
+          "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl",
+          "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl",
+          "remote": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
+        },
+        "tasks[task_key='TestTask'].libraries[0].whl": {
+          "action": "update",
+          "old": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl",
+          "new": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl",
+          "remote": "/Workspace/Users/[USERNAME]/.bundle/test-bundle/default/artifacts/.internal/my_test_code-0.0.1+[UNIX_TIME_NANOS][2]-py3-none-any.whl"
+        },
+        "timeout_seconds": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": 0
+        },
+        "webhook_notifications": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": {}
         }
       }
     }

acceptance/bundle/deploy/readplan/basic/out.plan_skip.json

@@ -29,19 +29,20 @@
         }
       },
       "changes": {
-        "remote": {
-          "email_notifications": {
-            "action": "skip",
-            "reason": "server_side_default"
-          },
-          "timeout_seconds": {
-            "action": "skip",
-            "reason": "server_side_default"
-          },
-          "webhook_notifications": {
-            "action": "skip",
-            "reason": "server_side_default"
-          }
+        "email_notifications": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": {}
+        },
+        "timeout_seconds": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": 0
+        },
+        "webhook_notifications": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": {}
         }
       }
     }

acceptance/bundle/migrate/basic/out.plan_update.json

@@ -61,30 +61,31 @@
         }
       },
       "changes": {
-        "local": {
-          "name": {
-            "action": "update",
-            "old": "Test Migration Job",
-            "new": "Test Migrated Job"
-          }
+        "email_notifications": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": {}
         },
-        "remote": {
-          "email_notifications": {
-            "action": "skip",
-            "reason": "server_side_default"
-          },
-          "tasks[task_key='main'].notebook_task.source": {
-            "action": "skip",
-            "reason": "server_side_default"
-          },
-          "timeout_seconds": {
-            "action": "skip",
-            "reason": "server_side_default"
-          },
-          "webhook_notifications": {
-            "action": "skip",
-            "reason": "server_side_default"
-          }
+        "name": {
+          "action": "update",
+          "old": "Test Migration Job",
+          "new": "Test Migrated Job",
+          "remote": "Test Migration Job"
+        },
+        "tasks[task_key='main'].notebook_task.source": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": "WORKSPACE"
+        },
+        "timeout_seconds": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": 0
+        },
+        "webhook_notifications": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": {}
         }
       }
     },
@@ -177,23 +178,22 @@
         "state": "IDLE"
       },
       "changes": {
-        "local": {
-          "name": {
-            "action": "update",
-            "old": "Test Migration Pipeline",
-            "new": "Test Migrated Pipeline"
-          },
-          "tags['myjob_name']": {
-            "action": "update",
-            "old": "Test Migration Job",
-            "new": "Test Migrated Job"
-          }
+        "name": {
+          "action": "update",
+          "old": "Test Migration Pipeline",
+          "new": "Test Migrated Pipeline",
+          "remote": "Test Migration Pipeline"
         },
-        "remote": {
-          "storage": {
-            "action": "skip",
-            "reason": "server_side_default"
-          }
+        "storage": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": "dbfs:/pipelines/[UUID]"
+        },
+        "tags['myjob_name']": {
+          "action": "update",
+          "old": "Test Migration Job",
+          "new": "Test Migrated Job",
+          "remote": "Test Migration Job"
         }
       }
     },
@@ -212,11 +212,10 @@
         "volume_type": "MANAGED"
       },
       "changes": {
-        "remote": {
-          "storage_location": {
-            "action": "skip",
-            "reason": "server_side_default"
-          }
+        "storage_location": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": "s3://deco-uc-prod-isolated-aws-us-east-1/metastore/[UUID]/volumes/[UUID]"
         }
       }
     }

acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json

@@ -20,18 +20,16 @@
         "warehouse_id": "123456"
       },
       "changes": {
-        "local": {
-          "etag": {
-            "action": "skip",
-            "old": "[NUMID]"
-          }
+        "etag": {
+          "action": "skip",
+          "old": "[NUMID]",
+          "remote": "[NUMID]"
         },
-        "remote": {
-          "serialized_dashboard": {
-            "action": "skip",
-            "old": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
-            "new": "{\"pages\":[{\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"name\":\"02724bf2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"
-          }
+        "serialized_dashboard": {
+          "action": "skip",
+          "old": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
+          "new": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n",
+          "remote": "{\"pages\":[{\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"name\":\"02724bf2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n"
         }
       }
     }

acceptance/bundle/migrate/default-python/out.plan_after_deploy.json

@@ -217,35 +217,37 @@
         }
       },
       "changes": {
-        "local": {
-          "tasks[task_key='notebook_task'].libraries[0].whl": {
-            "action": "update",
-            "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl",
-            "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
-          },
-          "tasks[task_key='python_wheel_task'].libraries[0].whl": {
-            "action": "update",
-            "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl",
-            "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl"
-          }
+        "email_notifications": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": {}
         },
-        "remote": {
-          "email_notifications": {
-            "action": "skip",
-            "reason": "server_side_default"
-          },
-          "tasks[task_key='notebook_task'].notebook_task.source": {
-            "action": "skip",
-            "reason": "server_side_default"
-          },
-          "timeout_seconds": {
-            "action": "skip",
-            "reason": "server_side_default"
-          },
-          "webhook_notifications": {
-            "action": "skip",
-            "reason": "server_side_default"
-          }
+        "tasks[task_key='notebook_task'].libraries[0].whl": {
+          "action": "update",
+          "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl",
+          "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl",
+          "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
+        },
+        "tasks[task_key='notebook_task'].notebook_task.source": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": "WORKSPACE"
+        },
+        "tasks[task_key='python_wheel_task'].libraries[0].whl": {
+          "action": "update",
+          "old": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl",
+          "new": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][0]-py3-none-any.whl",
+          "remote": "/Workspace/Users/[USERNAME]/.bundle/my_default_python/dev/artifacts/.internal/my_default_python-0.0.1+[UNIX_TIME_NANOS][1]-py3-none-any.whl"
+        },
+        "timeout_seconds": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": 0
+        },
+        "webhook_notifications": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": {}
         }
       }
     },
@@ -289,11 +291,10 @@
         "state": "IDLE"
       },
       "changes": {
-        "remote": {
-          "storage": {
-            "action": "skip",
-            "reason": "server_side_default"
-          }
+        "storage": {
+          "action": "skip",
+          "reason": "server_side_default",
+          "remote": "dbfs:/pipelines/[UUID]"
         }
       }
     }