From 2d373b58da60592b15af3e1439210e1e23a422f7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 25 Feb 2025 17:31:44 +0100 Subject: [PATCH 01/50] WIP --- .../bundle}/python_wheel/python_wheel/.gitignore | 0 .../bundle/python_wheel/python_wheel/databricks.yml | 0 .../bundle}/python_wheel/python_wheel/my_test_code/setup.py | 0 .../python_wheel/python_wheel/my_test_code/src/__init__.py | 0 .../python_wheel/python_wheel/my_test_code/src/__main__.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) rename {bundle/tests => acceptance/bundle}/python_wheel/python_wheel/.gitignore (100%) rename bundle/tests/python_wheel/python_wheel/bundle.yml => acceptance/bundle/python_wheel/python_wheel/databricks.yml (100%) rename {bundle/tests => acceptance/bundle}/python_wheel/python_wheel/my_test_code/setup.py (100%) rename {bundle/tests => acceptance/bundle}/python_wheel/python_wheel/my_test_code/src/__init__.py (100%) rename {bundle/tests => acceptance/bundle}/python_wheel/python_wheel/my_test_code/src/__main__.py (100%) diff --git a/bundle/tests/python_wheel/python_wheel/.gitignore b/acceptance/bundle/python_wheel/python_wheel/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel/.gitignore rename to acceptance/bundle/python_wheel/python_wheel/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel/bundle.yml b/acceptance/bundle/python_wheel/python_wheel/databricks.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel/bundle.yml rename to acceptance/bundle/python_wheel/python_wheel/databricks.yml diff --git a/bundle/tests/python_wheel/python_wheel/my_test_code/setup.py b/acceptance/bundle/python_wheel/python_wheel/my_test_code/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel/my_test_code/setup.py rename to acceptance/bundle/python_wheel/python_wheel/my_test_code/setup.py diff --git a/bundle/tests/python_wheel/python_wheel/my_test_code/src/__init__.py b/acceptance/bundle/python_wheel/python_wheel/my_test_code/src/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel/my_test_code/src/__init__.py rename to acceptance/bundle/python_wheel/python_wheel/my_test_code/src/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel/my_test_code/src/__main__.py b/acceptance/bundle/python_wheel/python_wheel/my_test_code/src/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel/my_test_code/src/__main__.py rename to acceptance/bundle/python_wheel/python_wheel/my_test_code/src/__main__.py From 6727be9783017f41ca14399f6e677ec3a83c4165 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 27 Feb 2025 12:46:38 +0100 Subject: [PATCH 02/50] add acc test --- .../python_wheel/out.requests.paths.txt | 16 +++++ .../python_wheel/python_wheel/output.txt | 11 ++++ .../bundle/python_wheel/python_wheel/script | 4 ++ .../python_wheel/python_wheel/test.toml | 61 +++++++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 acceptance/bundle/python_wheel/python_wheel/out.requests.paths.txt create mode 100644 acceptance/bundle/python_wheel/python_wheel/output.txt create mode 100644 acceptance/bundle/python_wheel/python_wheel/script create mode 100644 acceptance/bundle/python_wheel/python_wheel/test.toml diff --git a/acceptance/bundle/python_wheel/python_wheel/out.requests.paths.txt b/acceptance/bundle/python_wheel/python_wheel/out.requests.paths.txt new file mode 100644 index 0000000000..842f6b8a28 --- /dev/null +++ b/acceptance/bundle/python_wheel/python_wheel/out.requests.paths.txt @@ -0,0 +1,16 @@ +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/.gitignore" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/databricks.yml" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/setup.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/src/__init__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/src/__main__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/out.requests.txt" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/output.txt" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/repls.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/script" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/test.toml" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/deployment.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/metadata.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/terraform.tfstate" diff --git a/acceptance/bundle/python_wheel/python_wheel/output.txt b/acceptance/bundle/python_wheel/python_wheel/output.txt new file mode 100644 index 0000000000..56dced574e --- /dev/null +++ b/acceptance/bundle/python_wheel/python_wheel/output.txt @@ -0,0 +1,11 @@ + +>>> [CLI] bundle deploy +Building my_test_code... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find . +./my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/python_wheel/python_wheel/script b/acceptance/bundle/python_wheel/python_wheel/script new file mode 100644 index 0000000000..32952da69b --- /dev/null +++ b/acceptance/bundle/python_wheel/python_wheel/script @@ -0,0 +1,4 @@ +trace $CLI bundle deploy +trace find . | grep whl +jq .path < out.requests.txt | grep import | sort > out.requests.paths.txt +rm out.requests.txt diff --git a/acceptance/bundle/python_wheel/python_wheel/test.toml b/acceptance/bundle/python_wheel/python_wheel/test.toml new file mode 100644 index 0000000000..75ce045f13 --- /dev/null +++ b/acceptance/bundle/python_wheel/python_wheel/test.toml @@ -0,0 +1,61 @@ +# Testing that the wheel is built and uploaded +Badness = "Wheel is uploaded twice, once to dist/ and once more to .internal/" +Cloud = false +RecordRequests = true + +[[Server]] +Pattern = "GET /api/2.1/clusters/get" +Response.Body = ''' +{ + "cluster_id": "0717-132531-5opeqon1", + "spark_context_id": 4020997813441462000, + "cluster_name": "my-cluster", + "spark_version": "13.3.x-scala2.12", + "aws_attributes": { + "zone_id": "us-west-2c", + "first_on_demand": 1, + "availability": "SPOT_WITH_FALLBACK", + "spot_bid_price_percent": 100, + "ebs_volume_count": 0 + }, + "node_type_id": "i3.xlarge", + "driver_node_type_id": "i3.xlarge", + "autotermination_minutes": 120, + "enable_elastic_disk": false, + "disk_spec": { + "disk_count": 0 + }, + "cluster_source": "UI", + "enable_local_disk_encryption": false, + "instance_source": { + "node_type_id": "i3.xlarge" + }, + "driver_instance_source": { + "node_type_id": "i3.xlarge" + }, + "state": "TERMINATED", + "state_message": "Inactive cluster terminated (inactive for 120 minutes).", + "start_time": 1618263108824, + "terminated_time": 1619746525713, + "last_state_loss_time": 1619739324740, + "num_workers": 30, + "default_tags": { + "Vendor": "Databricks", + "Creator": "someone@example.com", + "ClusterName": "my-cluster", + "ClusterId": "1234-567890-reef123" + }, + "creator_user_name": "someone@example.com", + "termination_reason": { + "code": "INACTIVITY", + "parameters": { + "inactivity_duration_min": "120" + }, + "type": "SUCCESS" + }, + "init_scripts_safe_mode": false, + "spec": { + "spark_version": "13.3.x-scala2.12" + } +} +''' From 41026913c648fc5e178f30e3b7ed1f02148541b8 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 27 Feb 2025 12:59:19 +0100 Subject: [PATCH 03/50] Convert python_wheel test to acceptance Instead of calling mutator, log request paths and check that the wheel is uploaded. --- bundle/tests/python_wheel_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index 22702ec44c..600070a9f4 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -14,22 +14,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestPythonWheelBuild(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel/my_test_code/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - func TestPythonWheelBuildAutoDetect(t *testing.T) { b := loadTarget(t, "./python_wheel/python_wheel_no_artifact", "default") From 6a55de68aeb669192b9cf2c29c0067c2067140e2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 27 Feb 2025 14:11:53 +0100 Subject: [PATCH 04/50] set PYTHONDONTWRITEBYTECODE=1, add note about __pycache__ --- acceptance/bundle/python_wheel/python_wheel/script | 1 + acceptance/bundle/python_wheel/python_wheel/test.toml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/python_wheel/python_wheel/script b/acceptance/bundle/python_wheel/python_wheel/script index 32952da69b..1b2f16fbd3 100644 --- a/acceptance/bundle/python_wheel/python_wheel/script +++ b/acceptance/bundle/python_wheel/python_wheel/script @@ -1,3 +1,4 @@ +export PYTHONDONTWRITEBYTECODE=1 trace $CLI bundle deploy trace find . | grep whl jq .path < out.requests.txt | grep import | sort > out.requests.paths.txt diff --git a/acceptance/bundle/python_wheel/python_wheel/test.toml b/acceptance/bundle/python_wheel/python_wheel/test.toml index 75ce045f13..0ee50ab20b 100644 --- a/acceptance/bundle/python_wheel/python_wheel/test.toml +++ b/acceptance/bundle/python_wheel/python_wheel/test.toml @@ -1,5 +1,5 @@ # Testing that the wheel is built and uploaded -Badness = "Wheel is uploaded twice, once to dist/ and once more to .internal/" +Badness = "Wheel is uploaded twice, once to dist/ and once more to .internal/; A bunch of unnecessary stuff is uploaded, e.g. __pycache__" Cloud = false RecordRequests = true From 332130f868ae147831784804389609fd84061205 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 27 Feb 2025 14:18:19 +0100 Subject: [PATCH 05/50] simplify stubbed response --- .../python_wheel/python_wheel/test.toml | 50 +------------------ 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/acceptance/bundle/python_wheel/python_wheel/test.toml b/acceptance/bundle/python_wheel/python_wheel/test.toml index 0ee50ab20b..dcaaffe117 100644 --- a/acceptance/bundle/python_wheel/python_wheel/test.toml +++ b/acceptance/bundle/python_wheel/python_wheel/test.toml @@ -8,54 +8,6 @@ Pattern = "GET /api/2.1/clusters/get" Response.Body = ''' { "cluster_id": "0717-132531-5opeqon1", - "spark_context_id": 4020997813441462000, - "cluster_name": "my-cluster", - "spark_version": "13.3.x-scala2.12", - "aws_attributes": { - "zone_id": "us-west-2c", - "first_on_demand": 1, - "availability": "SPOT_WITH_FALLBACK", - "spot_bid_price_percent": 100, - "ebs_volume_count": 0 - }, - "node_type_id": "i3.xlarge", - "driver_node_type_id": "i3.xlarge", - "autotermination_minutes": 120, - "enable_elastic_disk": false, - "disk_spec": { - "disk_count": 0 - }, - "cluster_source": "UI", - "enable_local_disk_encryption": false, - "instance_source": { - "node_type_id": "i3.xlarge" - }, - "driver_instance_source": { - "node_type_id": "i3.xlarge" - }, - "state": "TERMINATED", - "state_message": "Inactive cluster terminated (inactive for 120 minutes).", - "start_time": 1618263108824, - "terminated_time": 1619746525713, - "last_state_loss_time": 1619739324740, - "num_workers": 30, - "default_tags": { - "Vendor": "Databricks", - "Creator": "someone@example.com", - "ClusterName": "my-cluster", - "ClusterId": "1234-567890-reef123" - }, - "creator_user_name": "someone@example.com", - "termination_reason": { - "code": "INACTIVITY", - "parameters": { - "inactivity_duration_min": "120" - }, - "type": "SUCCESS" - }, - "init_scripts_safe_mode": false, - "spec": { - "spark_version": "13.3.x-scala2.12" - } + "spark_version": "13.3.x-scala2.12" } ''' From 4c1d9fb7737d234721bc1ef35f631a88d80942af Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 27 Feb 2025 14:30:29 +0100 Subject: [PATCH 06/50] move --- .../bundle/{python_wheel => artifacts}/python_wheel/.gitignore | 0 .../{python_wheel => artifacts}/python_wheel/databricks.yml | 0 .../python_wheel/my_test_code/setup.py | 0 .../python_wheel/my_test_code/src/__init__.py | 0 .../python_wheel/my_test_code/src/__main__.py | 0 .../python_wheel/out.requests.paths.txt | 0 .../bundle/{python_wheel => artifacts}/python_wheel/output.txt | 0 acceptance/bundle/{python_wheel => artifacts}/python_wheel/script | 0 .../bundle/{python_wheel => artifacts}/python_wheel/test.toml | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename acceptance/bundle/{python_wheel => artifacts}/python_wheel/.gitignore (100%) rename acceptance/bundle/{python_wheel => artifacts}/python_wheel/databricks.yml (100%) rename acceptance/bundle/{python_wheel => artifacts}/python_wheel/my_test_code/setup.py (100%) rename acceptance/bundle/{python_wheel => artifacts}/python_wheel/my_test_code/src/__init__.py (100%) rename acceptance/bundle/{python_wheel => artifacts}/python_wheel/my_test_code/src/__main__.py (100%) rename acceptance/bundle/{python_wheel => artifacts}/python_wheel/out.requests.paths.txt (100%) rename acceptance/bundle/{python_wheel => artifacts}/python_wheel/output.txt (100%) rename acceptance/bundle/{python_wheel => artifacts}/python_wheel/script (100%) rename acceptance/bundle/{python_wheel => artifacts}/python_wheel/test.toml (100%) diff --git a/acceptance/bundle/python_wheel/python_wheel/.gitignore b/acceptance/bundle/artifacts/python_wheel/.gitignore similarity index 100% rename from acceptance/bundle/python_wheel/python_wheel/.gitignore rename to acceptance/bundle/artifacts/python_wheel/.gitignore diff --git a/acceptance/bundle/python_wheel/python_wheel/databricks.yml b/acceptance/bundle/artifacts/python_wheel/databricks.yml similarity index 100% rename from acceptance/bundle/python_wheel/python_wheel/databricks.yml rename to acceptance/bundle/artifacts/python_wheel/databricks.yml diff --git a/acceptance/bundle/python_wheel/python_wheel/my_test_code/setup.py b/acceptance/bundle/artifacts/python_wheel/my_test_code/setup.py similarity index 100% rename from acceptance/bundle/python_wheel/python_wheel/my_test_code/setup.py rename to acceptance/bundle/artifacts/python_wheel/my_test_code/setup.py diff --git a/acceptance/bundle/python_wheel/python_wheel/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/python_wheel/my_test_code/src/__init__.py similarity index 100% rename from acceptance/bundle/python_wheel/python_wheel/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/python_wheel/my_test_code/src/__init__.py diff --git a/acceptance/bundle/python_wheel/python_wheel/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/python_wheel/my_test_code/src/__main__.py similarity index 100% rename from acceptance/bundle/python_wheel/python_wheel/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/python_wheel/my_test_code/src/__main__.py diff --git a/acceptance/bundle/python_wheel/python_wheel/out.requests.paths.txt b/acceptance/bundle/artifacts/python_wheel/out.requests.paths.txt similarity index 100% rename from acceptance/bundle/python_wheel/python_wheel/out.requests.paths.txt rename to acceptance/bundle/artifacts/python_wheel/out.requests.paths.txt diff --git a/acceptance/bundle/python_wheel/python_wheel/output.txt b/acceptance/bundle/artifacts/python_wheel/output.txt similarity index 100% rename from acceptance/bundle/python_wheel/python_wheel/output.txt rename to acceptance/bundle/artifacts/python_wheel/output.txt diff --git a/acceptance/bundle/python_wheel/python_wheel/script b/acceptance/bundle/artifacts/python_wheel/script similarity index 100% rename from acceptance/bundle/python_wheel/python_wheel/script rename to acceptance/bundle/artifacts/python_wheel/script diff --git a/acceptance/bundle/python_wheel/python_wheel/test.toml b/acceptance/bundle/artifacts/python_wheel/test.toml similarity index 100% rename from acceptance/bundle/python_wheel/python_wheel/test.toml rename to acceptance/bundle/artifacts/python_wheel/test.toml From 0d387d193aa7fe9d1e7422cfb15d8aaa8df4ebf6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 27 Feb 2025 15:23:10 +0100 Subject: [PATCH 07/50] update --- .../bundle/artifacts}/python_wheel_no_artifact/.gitignore | 0 .../bundle/artifacts}/python_wheel_no_artifact/bundle.yml | 0 .../artifacts}/python_wheel_no_artifact/my_test_code/__init__.py | 0 .../artifacts}/python_wheel_no_artifact/my_test_code/__main__.py | 0 .../bundle/artifacts}/python_wheel_no_artifact/setup.py | 0 acceptance/bundle/artifacts/{python_wheel => }/test.toml | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact/.gitignore (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact/bundle.yml (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact/my_test_code/__init__.py (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact/my_test_code/__main__.py (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact/setup.py (100%) rename acceptance/bundle/artifacts/{python_wheel => }/test.toml (100%) diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/.gitignore b/acceptance/bundle/artifacts/python_wheel_no_artifact/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/.gitignore rename to acceptance/bundle/artifacts/python_wheel_no_artifact/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/bundle.yml b/acceptance/bundle/artifacts/python_wheel_no_artifact/bundle.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_no_artifact/bundle.yml diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__init__.py b/acceptance/bundle/artifacts/python_wheel_no_artifact/my_test_code/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__init__.py rename to acceptance/bundle/artifacts/python_wheel_no_artifact/my_test_code/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__main__.py b/acceptance/bundle/artifacts/python_wheel_no_artifact/my_test_code/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/my_test_code/__main__.py rename to acceptance/bundle/artifacts/python_wheel_no_artifact/my_test_code/__main__.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact/setup.py b/acceptance/bundle/artifacts/python_wheel_no_artifact/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact/setup.py rename to acceptance/bundle/artifacts/python_wheel_no_artifact/setup.py diff --git a/acceptance/bundle/artifacts/python_wheel/test.toml b/acceptance/bundle/artifacts/test.toml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel/test.toml rename to acceptance/bundle/artifacts/test.toml From ea88f27d61b756049446d4959c83895c7e4360c3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 27 Feb 2025 15:24:08 +0100 Subject: [PATCH 08/50] update --- acceptance/bundle/artifacts/python_wheel/out.requests.paths.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/acceptance/bundle/artifacts/python_wheel/out.requests.paths.txt b/acceptance/bundle/artifacts/python_wheel/out.requests.paths.txt index 842f6b8a28..dbf926e4e1 100644 --- a/acceptance/bundle/artifacts/python_wheel/out.requests.paths.txt +++ b/acceptance/bundle/artifacts/python_wheel/out.requests.paths.txt @@ -9,7 +9,6 @@ "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/output.txt" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/repls.json" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/script" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/test.toml" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/deploy.lock" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/deployment.json" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/metadata.json" From a7b0c8a3fabeeef89a1a0405f66aaaa4efdda59f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 27 Feb 2025 15:28:43 +0100 Subject: [PATCH 09/50] add missing --- .../out.requests.paths.txt | 15 +++++++++++++++ .../artifacts/python_wheel_no_artifact/output.txt | 11 +++++++++++ .../artifacts/python_wheel_no_artifact/script | 5 +++++ 3 files changed, 31 insertions(+) create mode 100644 acceptance/bundle/artifacts/python_wheel_no_artifact/out.requests.paths.txt create mode 100644 acceptance/bundle/artifacts/python_wheel_no_artifact/output.txt create mode 100644 acceptance/bundle/artifacts/python_wheel_no_artifact/script diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/out.requests.paths.txt b/acceptance/bundle/artifacts/python_wheel_no_artifact/out.requests.paths.txt new file mode 100644 index 0000000000..156bc03f14 --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact/out.requests.paths.txt @@ -0,0 +1,15 @@ +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/.gitignore" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/bundle.yml" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/__init__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/__main__.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/out.requests.txt" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/output.txt" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/repls.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/script" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/setup.py" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/deploy.lock" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/deployment.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/metadata.json" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/terraform.tfstate" diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/output.txt b/acceptance/bundle/artifacts/python_wheel_no_artifact/output.txt new file mode 100644 index 0000000000..0e6a96bc9a --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact/output.txt @@ -0,0 +1,11 @@ + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find . +./dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/script b/acceptance/bundle/artifacts/python_wheel_no_artifact/script new file mode 100644 index 0000000000..2830b1831f --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact/script @@ -0,0 +1,5 @@ +export PYTHONDONTWRITEBYTECODE=1 +trace $CLI bundle deploy +trace find . | grep whl +jq .path < out.requests.txt | grep import | sort > out.requests.paths.txt +rm -fr out.requests.txt dist build my_test_code.egg-info From a96dfc414bc1f6cde903f64350890c7b52ee6ba6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 27 Feb 2025 15:44:36 +0100 Subject: [PATCH 10/50] add python_wheel_no_build --- .../artifacts}/python_wheel_no_build/.gitignore | 0 .../artifacts}/python_wheel_no_build/bundle.yml | 0 .../dist/lib/my_test_code-0.0.1-py3-none-any.whl | Bin .../dist/my_test_code-0.0.1-py3-none-any.whl | Bin .../python_wheel_no_build/out.requests.paths.txt | 4 ++++ .../artifacts/python_wheel_no_build/output.txt | 12 ++++++++++++ .../bundle/artifacts/python_wheel_no_build/script | 5 +++++ 7 files changed, 21 insertions(+) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_build/.gitignore (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_build/bundle.yml (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl (100%) create mode 100644 acceptance/bundle/artifacts/python_wheel_no_build/out.requests.paths.txt create mode 100644 acceptance/bundle/artifacts/python_wheel_no_build/output.txt create mode 100644 acceptance/bundle/artifacts/python_wheel_no_build/script diff --git a/bundle/tests/python_wheel/python_wheel_no_build/.gitignore b/acceptance/bundle/artifacts/python_wheel_no_build/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_build/.gitignore rename to acceptance/bundle/artifacts/python_wheel_no_build/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_build/bundle.yml b/acceptance/bundle/artifacts/python_wheel_no_build/bundle.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_build/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_no_build/bundle.yml diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl diff --git a/bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/out.requests.paths.txt b/acceptance/bundle/artifacts/python_wheel_no_build/out.requests.paths.txt new file mode 100644 index 0000000000..4a8baf73db --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_no_build/out.requests.paths.txt @@ -0,0 +1,4 @@ +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/lib/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/output.txt b/acceptance/bundle/artifacts/python_wheel_no_build/output.txt new file mode 100644 index 0000000000..9825e859d7 --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_no_build/output.txt @@ -0,0 +1,12 @@ + +>>> [CLI] bundle deploy +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find . +./dist/lib/my_test_code-0.0.1-py3-none-any.whl +./dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/script b/acceptance/bundle/artifacts/python_wheel_no_build/script new file mode 100644 index 0000000000..fca3d7a482 --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_no_build/script @@ -0,0 +1,5 @@ +export PYTHONDONTWRITEBYTECODE=1 +trace $CLI bundle deploy +trace find . | grep whl +jq .path < out.requests.txt | grep import | grep whl | sort > out.requests.paths.txt +rm out.requests.txt From 74733991b9b3500fac3ad537334a52bfc9743088 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 27 Feb 2025 15:45:13 +0100 Subject: [PATCH 11/50] clean up --- bundle/tests/python_wheel_test.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index 600070a9f4..e484be780d 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -14,22 +14,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestPythonWheelBuildAutoDetect(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_artifact", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_notebook", "default") @@ -114,15 +98,3 @@ func TestPythonWheelBuildMultiple(t *testing.T) { diags = bundle.Apply(ctx, b, match) require.NoError(t, diags.Error()) } - -func TestPythonWheelNoBuild(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_build", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} From c83c7bb23f21caddd39de5a6bca1c2cbe896e580 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 27 Feb 2025 15:48:55 +0100 Subject: [PATCH 12/50] update --- .../python_wheel_no_artifact_notebook/.gitignore | 0 .../python_wheel_no_artifact_notebook/bundle.yml | 0 .../my_test_code/__init__.py | 0 .../my_test_code/__main__.py | 0 .../python_wheel_no_artifact_notebook/notebook.py | 0 .../out.requests.paths.txt | 2 ++ .../python_wheel_no_artifact_notebook/output.txt | 11 +++++++++++ .../python_wheel_no_artifact_notebook/script | 5 +++++ .../python_wheel_no_artifact_notebook/setup.py | 0 9 files changed, 18 insertions(+) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact_notebook/.gitignore (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact_notebook/bundle.yml (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact_notebook/my_test_code/__init__.py (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact_notebook/my_test_code/__main__.py (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact_notebook/notebook.py (100%) create mode 100644 acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/out.requests.paths.txt create mode 100644 acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/output.txt create mode 100644 acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact_notebook/setup.py (100%) diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/.gitignore rename to acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/bundle.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/bundle.yml diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/my_test_code/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__init__.py rename to acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/my_test_code/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/my_test_code/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/my_test_code/__main__.py rename to acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/my_test_code/__main__.py diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/notebook.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/notebook.py rename to acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/notebook.py diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/out.requests.paths.txt b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/out.requests.paths.txt new file mode 100644 index 0000000000..d26e76bc76 --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/out.requests.paths.txt @@ -0,0 +1,2 @@ +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/output.txt b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/output.txt new file mode 100644 index 0000000000..b1ec4a0230 --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/output.txt @@ -0,0 +1,11 @@ + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> find . +./dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script new file mode 100644 index 0000000000..ac13faf10a --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script @@ -0,0 +1,5 @@ +export PYTHONDONTWRITEBYTECODE=1 +trace $CLI bundle deploy +trace find . | grep whl +jq .path < out.requests.txt | grep import | grep whl | sort > out.requests.paths.txt +rm -fr out.requests.txt build dist my_test_code.egg-info diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_notebook/setup.py rename to acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/setup.py From 89503889e887cb60fc23d90fd6552023d5482551 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 10:28:27 +0100 Subject: [PATCH 13/50] convert python_wheel_dbfs_lib --- .../artifacts}/python_wheel_dbfs_lib/bundle.yml | 0 .../python_wheel_dbfs_lib/out.jobs.create.txt | 15 +++++++++++++++ .../artifacts/python_wheel_dbfs_lib/output.txt | 14 ++++++++++++++ .../bundle/artifacts/python_wheel_dbfs_lib/script | 5 +++++ 4 files changed, 34 insertions(+) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_dbfs_lib/bundle.yml (100%) create mode 100644 acceptance/bundle/artifacts/python_wheel_dbfs_lib/out.jobs.create.txt create mode 100644 acceptance/bundle/artifacts/python_wheel_dbfs_lib/output.txt create mode 100644 acceptance/bundle/artifacts/python_wheel_dbfs_lib/script diff --git a/bundle/tests/python_wheel/python_wheel_dbfs_lib/bundle.yml b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/bundle.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_dbfs_lib/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_dbfs_lib/bundle.yml diff --git a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/out.jobs.create.txt b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/out.jobs.create.txt new file mode 100644 index 0000000000..0184d3c4a0 --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/out.jobs.create.txt @@ -0,0 +1,15 @@ +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "dbfs:/path/to/dist/mywheel.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] diff --git a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/output.txt b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/output.txt new file mode 100644 index 0000000000..18ca40aa20 --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/output.txt @@ -0,0 +1,14 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> errcode sh -c jq .path < out.requests.txt | grep whl + +Exit code: 1 + +>>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt + +>>> rm out.requests.txt diff --git a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script new file mode 100644 index 0000000000..c445b65ac1 --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script @@ -0,0 +1,5 @@ +export PYTHONDONTWRITEBYTECODE=1 +trace $CLI bundle deploy +trace errcode sh -c 'jq .path < out.requests.txt | grep whl' # should error as there are no wheels uploaded +trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt > out.jobs.create.txt +trace rm out.requests.txt From acdf1de99c0ac98e857604f69e25392bc460c6dd Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 10:42:29 +0100 Subject: [PATCH 14/50] migrate multiple wheels --- .../artifacts}/environment_key/.gitignore | 0 .../artifacts}/environment_key/databricks.yml | 0 .../environment_key/my_test_code/setup.py | 0 .../my_test_code/src/__init__.py | 0 .../my_test_code/src/__main__.py | 0 .../python_wheel_multiple/.gitignore | 0 .../python_wheel_multiple/bundle.yml | 0 .../my_test_code/setup.py | 0 .../my_test_code/setup2.py | 0 .../my_test_code/src/__init__.py | 0 .../my_test_code/src/__main__.py | 0 .../python_wheel_multiple/output.txt | 43 ++++++++++++++++++ .../artifacts/python_wheel_multiple/script | 13 ++++++ .../.gitignore | 0 .../bundle.yml | 0 .../my_test_code-0.0.1-py3-none-any.whl | Bin 16 files changed, 56 insertions(+) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/environment_key/.gitignore (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/environment_key/databricks.yml (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/environment_key/my_test_code/setup.py (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/environment_key/my_test_code/src/__init__.py (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/environment_key/my_test_code/src/__main__.py (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_multiple/.gitignore (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_multiple/bundle.yml (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_multiple/my_test_code/setup.py (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_multiple/my_test_code/setup2.py (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_multiple/my_test_code/src/__init__.py (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_multiple/my_test_code/src/__main__.py (100%) create mode 100644 acceptance/bundle/artifacts/python_wheel_multiple/output.txt create mode 100644 acceptance/bundle/artifacts/python_wheel_multiple/script rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact_no_setup/.gitignore (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact_no_setup/bundle.yml (100%) rename {bundle/tests/python_wheel => acceptance/bundle/artifacts}/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl (100%) diff --git a/bundle/tests/python_wheel/environment_key/.gitignore b/acceptance/bundle/artifacts/environment_key/.gitignore similarity index 100% rename from bundle/tests/python_wheel/environment_key/.gitignore rename to acceptance/bundle/artifacts/environment_key/.gitignore diff --git a/bundle/tests/python_wheel/environment_key/databricks.yml b/acceptance/bundle/artifacts/environment_key/databricks.yml similarity index 100% rename from bundle/tests/python_wheel/environment_key/databricks.yml rename to acceptance/bundle/artifacts/environment_key/databricks.yml diff --git a/bundle/tests/python_wheel/environment_key/my_test_code/setup.py b/acceptance/bundle/artifacts/environment_key/my_test_code/setup.py similarity index 100% rename from bundle/tests/python_wheel/environment_key/my_test_code/setup.py rename to acceptance/bundle/artifacts/environment_key/my_test_code/setup.py diff --git a/bundle/tests/python_wheel/environment_key/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/environment_key/my_test_code/src/__init__.py similarity index 100% rename from bundle/tests/python_wheel/environment_key/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/environment_key/my_test_code/src/__init__.py diff --git a/bundle/tests/python_wheel/environment_key/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/environment_key/my_test_code/src/__main__.py similarity index 100% rename from bundle/tests/python_wheel/environment_key/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/environment_key/my_test_code/src/__main__.py diff --git a/bundle/tests/python_wheel/python_wheel_multiple/.gitignore b/acceptance/bundle/artifacts/python_wheel_multiple/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/.gitignore rename to acceptance/bundle/artifacts/python_wheel_multiple/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_multiple/bundle.yml b/acceptance/bundle/artifacts/python_wheel_multiple/bundle.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_multiple/bundle.yml diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py b/acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/setup.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup.py rename to acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/setup.py diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py b/acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/setup2.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/setup2.py rename to acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/setup2.py diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/src/__init__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/src/__init__.py diff --git a/bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/src/__main__.py similarity index 100% rename from bundle/tests/python_wheel/python_wheel_multiple/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/output.txt b/acceptance/bundle/artifacts/python_wheel_multiple/output.txt new file mode 100644 index 0000000000..d6f99ba6e6 --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_multiple/output.txt @@ -0,0 +1,43 @@ + +>>> [CLI] bundle deploy +Building my_test_code... +Building my_test_code_2... +Deploying resources... +Deployment complete! +Updating deployment state... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading my_test_code_2-0.0.1-py3-none-any.whl... + +=== Expecting to find 2 wheels +>>> sh -c find . | grep whl +./my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl +./my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 2 wheels in libraries section in /jobs/create +>>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + }, + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 2 wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code_2-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/script b/acceptance/bundle/artifacts/python_wheel_multiple/script new file mode 100644 index 0000000000..5593ae26f3 --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_multiple/script @@ -0,0 +1,13 @@ +export PYTHONDONTWRITEBYTECODE=1 +trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order + +title "Expecting to find 2 wheels" +trace sh -c 'find . | grep whl' # expecting 2 wheels + +title "Expecting 2 wheels in libraries section in /jobs/create" +trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 2 wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/.gitignore b/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/.gitignore similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/.gitignore rename to acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/.gitignore diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml b/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/bundle.yml similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/bundle.yml diff --git a/bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from bundle/tests/python_wheel/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl From 0af4a2dfa76c7e6df02cd5ebb1e88bc30b62e9ac Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 10:47:31 +0100 Subject: [PATCH 15/50] add environment_key --- .../artifacts/environment_key/output.txt | 55 +++++++++++++++++++ .../bundle/artifacts/environment_key/script | 13 +++++ 2 files changed, 68 insertions(+) create mode 100644 acceptance/bundle/artifacts/environment_key/output.txt create mode 100644 acceptance/bundle/artifacts/environment_key/script diff --git a/acceptance/bundle/artifacts/environment_key/output.txt b/acceptance/bundle/artifacts/environment_key/output.txt new file mode 100644 index 0000000000..1f53dd5dd7 --- /dev/null +++ b/acceptance/bundle/artifacts/environment_key/output.txt @@ -0,0 +1,55 @@ + +>>> [CLI] bundle deploy +Building my_test_code... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/environment_key/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Expecting to find 1 wheel +>>> sh -c find . | grep whl +./my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in environments section in /jobs/create +>>> jq select(.path=="/api/2.1/jobs/create") | .body out.requests.txt +{ + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/environment_key/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "environments": [ + { + "environment_key": "test_env", + "spec": { + "client": "1", + "dependencies": [ + "/Workspace/Users/[USERNAME]/.bundle/environment_key/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + ] + } + } + ], + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "My Wheel Job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "environment_key": "test_env", + "existing_cluster_id": "0717-132531-5opeqon1", + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } + ] +} + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/environment_key/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/environment_key/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/environment_key/script b/acceptance/bundle/artifacts/environment_key/script new file mode 100644 index 0000000000..ef7dd08d1c --- /dev/null +++ b/acceptance/bundle/artifacts/environment_key/script @@ -0,0 +1,13 @@ +export PYTHONDONTWRITEBYTECODE=1 +trace $CLI bundle deploy + +title "Expecting to find 1 wheel" +trace sh -c 'find . | grep whl' + +title "Expecting 1 wheel in environments section in /jobs/create" +trace jq 'select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt From 3c06d8d9462e289b8a62bfb348b759fe55701e82 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 10:49:44 +0100 Subject: [PATCH 16/50] add no_artifact_no_setup --- .../output.txt | 47 +++++++++++++++++++ .../python_wheel_no_artifact_no_setup/script | 13 +++++ 2 files changed, 60 insertions(+) create mode 100644 acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/output.txt create mode 100644 acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/script diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/output.txt b/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/output.txt new file mode 100644 index 0000000000..037c88dbee --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/output.txt @@ -0,0 +1,47 @@ + +>>> [CLI] bundle deploy +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Expecting to find 1 wheel +>>> sh -c find . | grep whl +./package/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq select(.path=="/api/2.1/jobs/create") | .body out.requests.txt +{ + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/state/metadata.json" + }, + "edit_mode": "UI_LOCKED", + "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "[default] My Wheel Job", + "queue": { + "enabled": true + }, + "tasks": [ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } + ] +} + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/files/package/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/script b/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/script new file mode 100644 index 0000000000..6b6f3cb38d --- /dev/null +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/script @@ -0,0 +1,13 @@ +export PYTHONDONTWRITEBYTECODE=1 +trace $CLI bundle deploy + +title "Expecting to find 1 wheel" +trace sh -c 'find . | grep whl' + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq 'select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt From cb47517b26ca28b03b44bae4aab001b59eb27b4f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 10:51:49 +0100 Subject: [PATCH 17/50] update --- .../python_wheel/out.requests.paths.txt | 15 ----------- .../bundle/artifacts/python_wheel/output.txt | 26 ++++++++++++++++++- .../bundle/artifacts/python_wheel/script | 12 +++++++-- 3 files changed, 35 insertions(+), 18 deletions(-) delete mode 100644 acceptance/bundle/artifacts/python_wheel/out.requests.paths.txt diff --git a/acceptance/bundle/artifacts/python_wheel/out.requests.paths.txt b/acceptance/bundle/artifacts/python_wheel/out.requests.paths.txt deleted file mode 100644 index dbf926e4e1..0000000000 --- a/acceptance/bundle/artifacts/python_wheel/out.requests.paths.txt +++ /dev/null @@ -1,15 +0,0 @@ -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/.gitignore" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/databricks.yml" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/setup.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/src/__init__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/src/__main__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/out.requests.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/output.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/repls.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/script" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/deployment.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/metadata.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/terraform.tfstate" diff --git a/acceptance/bundle/artifacts/python_wheel/output.txt b/acceptance/bundle/artifacts/python_wheel/output.txt index 56dced574e..34abcca1df 100644 --- a/acceptance/bundle/artifacts/python_wheel/output.txt +++ b/acceptance/bundle/artifacts/python_wheel/output.txt @@ -7,5 +7,29 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> find . +=== Expecting to find 1 wheel +>>> sh -c find . | grep whl ./my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/python_wheel/script b/acceptance/bundle/artifacts/python_wheel/script index 1b2f16fbd3..f075ea2d03 100644 --- a/acceptance/bundle/artifacts/python_wheel/script +++ b/acceptance/bundle/artifacts/python_wheel/script @@ -1,5 +1,13 @@ export PYTHONDONTWRITEBYTECODE=1 trace $CLI bundle deploy -trace find . | grep whl -jq .path < out.requests.txt | grep import | sort > out.requests.paths.txt + +title "Expecting to find 1 wheel" +trace sh -c 'find . | grep whl' + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + rm out.requests.txt From 3bca7e322c000d90dc06c32a0e3f301d1f079396 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 10:55:04 +0100 Subject: [PATCH 18/50] update --- .../python_wheel_dbfs_lib/out.jobs.create.txt | 15 ------------ .../python_wheel_dbfs_lib/output.txt | 24 +++++++++++++++++-- .../artifacts/python_wheel_dbfs_lib/script | 14 ++++++++--- 3 files changed, 33 insertions(+), 20 deletions(-) delete mode 100644 acceptance/bundle/artifacts/python_wheel_dbfs_lib/out.jobs.create.txt diff --git a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/out.jobs.create.txt b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/out.jobs.create.txt deleted file mode 100644 index 0184d3c4a0..0000000000 --- a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/out.jobs.create.txt +++ /dev/null @@ -1,15 +0,0 @@ -[ - { - "existing_cluster_id": "0717-132531-5opeqon1", - "libraries": [ - { - "whl": "dbfs:/path/to/dist/mywheel.whl" - } - ], - "python_wheel_task": { - "entry_point": "run", - "package_name": "my_test_code" - }, - "task_key": "TestTask" - } -] diff --git a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/output.txt b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/output.txt index 18ca40aa20..9758d77495 100644 --- a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/output.txt +++ b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/output.txt @@ -5,10 +5,30 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> errcode sh -c jq .path < out.requests.txt | grep whl +=== Expecting to find no wheels, must error +>>> errcode sh -c find . | grep whl Exit code: 1 +=== Expecting 1 wheel in libraries section in /jobs/create >>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "dbfs:/path/to/dist/mywheel.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] ->>> rm out.requests.txt +=== Expecting no wheels to be uploaded +>>> errcode sh -c jq .path < out.requests.txt | grep import | grep whl + +Exit code: 1 diff --git a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script index c445b65ac1..111a0f64cd 100644 --- a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script +++ b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script @@ -1,5 +1,13 @@ export PYTHONDONTWRITEBYTECODE=1 trace $CLI bundle deploy -trace errcode sh -c 'jq .path < out.requests.txt | grep whl' # should error as there are no wheels uploaded -trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt > out.jobs.create.txt -trace rm out.requests.txt + +title "Expecting to find no wheels, must error" +trace errcode sh -c 'find . | grep whl' + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting no wheels to be uploaded" +trace errcode sh -c 'jq .path < out.requests.txt | grep import | grep whl' + +rm out.requests.txt From 958163f99824fefbd52b8774095aa7313e11e20f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 10:55:29 +0100 Subject: [PATCH 19/50] rename to databricks.yml --- .../python_wheel_dbfs_lib/{bundle.yml => databricks.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename acceptance/bundle/artifacts/python_wheel_dbfs_lib/{bundle.yml => databricks.yml} (100%) diff --git a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/bundle.yml b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_dbfs_lib/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_dbfs_lib/databricks.yml From 5591cfc7d2e77214d249c2b2807f3b581d054936 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 10:58:15 +0100 Subject: [PATCH 20/50] update --- .../python_wheel_multiple/{bundle.yml => databricks.yml} | 0 .../python_wheel_no_artifact/{bundle.yml => databricks.yml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename acceptance/bundle/artifacts/python_wheel_multiple/{bundle.yml => databricks.yml} (100%) rename acceptance/bundle/artifacts/python_wheel_no_artifact/{bundle.yml => databricks.yml} (100%) diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/bundle.yml b/acceptance/bundle/artifacts/python_wheel_multiple/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_multiple/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_multiple/databricks.yml diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/bundle.yml b/acceptance/bundle/artifacts/python_wheel_no_artifact/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_no_artifact/databricks.yml From 5143133d381fd56af516202c27725067d19807d7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 10:58:41 +0100 Subject: [PATCH 21/50] rename --- .../{bundle.yml => databricks.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/{bundle.yml => databricks.yml} (100%) diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/bundle.yml b/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/databricks.yml From 4f707389de4699f34a46c9d383b084027143da3b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 10:59:06 +0100 Subject: [PATCH 22/50] update --- .../out.requests.paths.txt | 15 ----------- .../python_wheel_no_artifact/output.txt | 26 ++++++++++++++++++- .../artifacts/python_wheel_no_artifact/script | 14 +++++++--- 3 files changed, 36 insertions(+), 19 deletions(-) delete mode 100644 acceptance/bundle/artifacts/python_wheel_no_artifact/out.requests.paths.txt diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/out.requests.paths.txt b/acceptance/bundle/artifacts/python_wheel_no_artifact/out.requests.paths.txt deleted file mode 100644 index 156bc03f14..0000000000 --- a/acceptance/bundle/artifacts/python_wheel_no_artifact/out.requests.paths.txt +++ /dev/null @@ -1,15 +0,0 @@ -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/.gitignore" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/bundle.yml" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/__init__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/__main__.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/out.requests.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/output.txt" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/repls.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/script" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/setup.py" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/deploy.lock" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/deployment.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/metadata.json" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/state/terraform.tfstate" diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/output.txt b/acceptance/bundle/artifacts/python_wheel_no_artifact/output.txt index 0e6a96bc9a..1a829648c7 100644 --- a/acceptance/bundle/artifacts/python_wheel_no_artifact/output.txt +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact/output.txt @@ -7,5 +7,29 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> find . +=== Expecting to find 1 wheel +>>> sh -c find . | grep whl ./dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheels in libraries section in /jobs/create +>>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/script b/acceptance/bundle/artifacts/python_wheel_no_artifact/script index 2830b1831f..ba1b098829 100644 --- a/acceptance/bundle/artifacts/python_wheel_no_artifact/script +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact/script @@ -1,5 +1,13 @@ export PYTHONDONTWRITEBYTECODE=1 trace $CLI bundle deploy -trace find . | grep whl -jq .path < out.requests.txt | grep import | sort > out.requests.paths.txt -rm -fr out.requests.txt dist build my_test_code.egg-info + +title "Expecting to find 1 wheel" +trace sh -c 'find . | grep whl' + +title "Expecting 1 wheels in libraries section in /jobs/create" +trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 1 wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm -fr out.requests.txt build dist my_test_code.egg-info From cafb19e5b29ebdce59374173cd97f6e4928693f2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 11:00:57 +0100 Subject: [PATCH 23/50] update --- .../out.requests.paths.txt | 4 --- .../python_wheel_no_build/output.txt | 36 ++++++++++++++++++- .../artifacts/python_wheel_no_build/script | 16 +++++++-- 3 files changed, 49 insertions(+), 7 deletions(-) delete mode 100644 acceptance/bundle/artifacts/python_wheel_no_build/out.requests.paths.txt diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/out.requests.paths.txt b/acceptance/bundle/artifacts/python_wheel_no_build/out.requests.paths.txt deleted file mode 100644 index 4a8baf73db..0000000000 --- a/acceptance/bundle/artifacts/python_wheel_no_build/out.requests.paths.txt +++ /dev/null @@ -1,4 +0,0 @@ -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/lib/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/output.txt b/acceptance/bundle/artifacts/python_wheel_no_build/output.txt index 9825e859d7..c69571a4d6 100644 --- a/acceptance/bundle/artifacts/python_wheel_no_build/output.txt +++ b/acceptance/bundle/artifacts/python_wheel_no_build/output.txt @@ -1,4 +1,9 @@ +=== Expecting to find 2 wheels, initially provided +>>> sh -c find . | grep whl +./dist/lib/my_test_code-0.0.1-py3-none-any.whl +./dist/my_test_code-0.0.1-py3-none-any.whl + >>> [CLI] bundle deploy Uploading my_test_code-0.0.1-py3-none-any.whl... Uploading my_test_code-0.0.1-py3-none-any.whl... @@ -7,6 +12,35 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> find . +=== Expecting to find 2 wheels, same as initially provided +>>> sh -c find . | grep whl ./dist/lib/my_test_code-0.0.1-py3-none-any.whl ./dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 2 wheels in libraries section in /jobs/create +>>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + }, + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 2 wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/lib/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/script b/acceptance/bundle/artifacts/python_wheel_no_build/script index fca3d7a482..c349315f7f 100644 --- a/acceptance/bundle/artifacts/python_wheel_no_build/script +++ b/acceptance/bundle/artifacts/python_wheel_no_build/script @@ -1,5 +1,17 @@ export PYTHONDONTWRITEBYTECODE=1 + +title "Expecting to find 2 wheels, initially provided" +trace sh -c 'find . | grep whl' + trace $CLI bundle deploy -trace find . | grep whl -jq .path < out.requests.txt | grep import | grep whl | sort > out.requests.paths.txt + +title "Expecting to find 2 wheels, same as initially provided" +trace sh -c 'find . | grep whl' + +title "Expecting 2 wheels in libraries section in /jobs/create" +trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 2 wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + rm out.requests.txt From 0866d3cfbeba0903d4ede71d1a643e95054b449b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 11:01:09 +0100 Subject: [PATCH 24/50] rename --- .../python_wheel_no_build/{bundle.yml => databricks.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename acceptance/bundle/artifacts/python_wheel_no_build/{bundle.yml => databricks.yml} (100%) diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/bundle.yml b/acceptance/bundle/artifacts/python_wheel_no_build/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_build/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_no_build/databricks.yml From 58032d1389f6c5096278ba8f3533be90f7511aaf Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 11:02:23 +0100 Subject: [PATCH 25/50] update --- .../bundle/artifacts/environment_key/script | 1 - .../bundle/artifacts/python_wheel/script | 1 - .../artifacts/python_wheel_dbfs_lib/script | 1 - .../artifacts/python_wheel_multiple/script | 1 - .../artifacts/python_wheel_no_artifact/script | 1 - .../python_wheel_no_artifact_no_setup/script | 1 - .../python_wheel_no_artifact_notebook/script | 1 - .../artifacts/python_wheel_no_build/script | 2 - acceptance/bundle/artifacts/script.prepare | 1 + bundle/tests/python_wheel_test.go | 60 ------------------- 10 files changed, 1 insertion(+), 69 deletions(-) create mode 100644 acceptance/bundle/artifacts/script.prepare diff --git a/acceptance/bundle/artifacts/environment_key/script b/acceptance/bundle/artifacts/environment_key/script index ef7dd08d1c..9551f39eda 100644 --- a/acceptance/bundle/artifacts/environment_key/script +++ b/acceptance/bundle/artifacts/environment_key/script @@ -1,4 +1,3 @@ -export PYTHONDONTWRITEBYTECODE=1 trace $CLI bundle deploy title "Expecting to find 1 wheel" diff --git a/acceptance/bundle/artifacts/python_wheel/script b/acceptance/bundle/artifacts/python_wheel/script index f075ea2d03..3fc1e4c862 100644 --- a/acceptance/bundle/artifacts/python_wheel/script +++ b/acceptance/bundle/artifacts/python_wheel/script @@ -1,4 +1,3 @@ -export PYTHONDONTWRITEBYTECODE=1 trace $CLI bundle deploy title "Expecting to find 1 wheel" diff --git a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script index 111a0f64cd..aa075451ea 100644 --- a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script +++ b/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script @@ -1,4 +1,3 @@ -export PYTHONDONTWRITEBYTECODE=1 trace $CLI bundle deploy title "Expecting to find no wheels, must error" diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/script b/acceptance/bundle/artifacts/python_wheel_multiple/script index 5593ae26f3..99c7a5d72b 100644 --- a/acceptance/bundle/artifacts/python_wheel_multiple/script +++ b/acceptance/bundle/artifacts/python_wheel_multiple/script @@ -1,4 +1,3 @@ -export PYTHONDONTWRITEBYTECODE=1 trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order title "Expecting to find 2 wheels" diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/script b/acceptance/bundle/artifacts/python_wheel_no_artifact/script index ba1b098829..e41d052d77 100644 --- a/acceptance/bundle/artifacts/python_wheel_no_artifact/script +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact/script @@ -1,4 +1,3 @@ -export PYTHONDONTWRITEBYTECODE=1 trace $CLI bundle deploy title "Expecting to find 1 wheel" diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/script b/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/script index 6b6f3cb38d..f84ca5a195 100644 --- a/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/script +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/script @@ -1,4 +1,3 @@ -export PYTHONDONTWRITEBYTECODE=1 trace $CLI bundle deploy title "Expecting to find 1 wheel" diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script index ac13faf10a..e59e8d76a5 100644 --- a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script @@ -1,4 +1,3 @@ -export PYTHONDONTWRITEBYTECODE=1 trace $CLI bundle deploy trace find . | grep whl jq .path < out.requests.txt | grep import | grep whl | sort > out.requests.paths.txt diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/script b/acceptance/bundle/artifacts/python_wheel_no_build/script index c349315f7f..9396b1958b 100644 --- a/acceptance/bundle/artifacts/python_wheel_no_build/script +++ b/acceptance/bundle/artifacts/python_wheel_no_build/script @@ -1,5 +1,3 @@ -export PYTHONDONTWRITEBYTECODE=1 - title "Expecting to find 2 wheels, initially provided" trace sh -c 'find . | grep whl' diff --git a/acceptance/bundle/artifacts/script.prepare b/acceptance/bundle/artifacts/script.prepare new file mode 100644 index 0000000000..eca243a532 --- /dev/null +++ b/acceptance/bundle/artifacts/script.prepare @@ -0,0 +1 @@ +export PYTHONDONTWRITEBYTECODE=1 diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go index e484be780d..6e185af699 100644 --- a/bundle/tests/python_wheel_test.go +++ b/bundle/tests/python_wheel_test.go @@ -14,34 +14,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestPythonWheelBuildAutoDetectWithNotebookTask(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_notebook", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel_no_artifact_notebook/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelWithDBFSLib(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_dbfs_lib", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_no_setup", "default") @@ -66,35 +38,3 @@ func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { require.Empty(t, diags) require.Equal(t, "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].Libraries[0].Whl) } - -func TestPythonWheelBuildWithEnvironmentKey(t *testing.T) { - b := loadTarget(t, "./python_wheel/environment_key", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/environment_key/my_test_code/dist/my_test_code-*.whl") - require.NoError(t, err) - require.Len(t, matches, 1) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} - -func TestPythonWheelBuildMultiple(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_multiple", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - matches, err := filepath.Glob("./python_wheel/python_wheel_multiple/my_test_code/dist/my_test_code*.whl") - require.NoError(t, err) - require.Len(t, matches, 2) - - match := libraries.ExpandGlobReferences() - diags = bundle.Apply(ctx, b, match) - require.NoError(t, diags.Error()) -} From 231a1dff20d4a808a207bfd7656e770c177ba935 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 11:02:32 +0100 Subject: [PATCH 26/50] cleanup --- bundle/tests/python_wheel_test.go | 40 ------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 bundle/tests/python_wheel_test.go diff --git a/bundle/tests/python_wheel_test.go b/bundle/tests/python_wheel_test.go deleted file mode 100644 index 6e185af699..0000000000 --- a/bundle/tests/python_wheel_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package config_tests - -import ( - "context" - "path/filepath" - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/bundle/phases" - mockfiler "github.com/databricks/cli/internal/mocks/libs/filer" - "github.com/databricks/cli/libs/filer" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" -) - -func TestPythonWheelBuildNoBuildJustUpload(t *testing.T) { - b := loadTarget(t, "./python_wheel/python_wheel_no_artifact_no_setup", "default") - - ctx := context.Background() - diags := phases.Build(ctx, b) - require.NoError(t, diags.Error()) - - mockFiler := mockfiler.NewMockFiler(t) - mockFiler.EXPECT().Write( - mock.Anything, - filepath.Join("my_test_code-0.0.1-py3-none-any.whl"), - mock.AnythingOfType("*os.File"), - filer.OverwriteIfExists, - filer.CreateParentDirectories, - ).Return(nil) - - diags = bundle.ApplySeq(ctx, b, - libraries.ExpandGlobReferences(), - libraries.UploadWithClient(mockFiler), - ) - require.NoError(t, diags.Error()) - require.Empty(t, diags) - require.Equal(t, "/Workspace/foo/bar/.internal/my_test_code-0.0.1-py3-none-any.whl", b.Config.Resources.Jobs["test_job"].JobSettings.Tasks[0].Libraries[0].Whl) -} From 59d7b59ec2f0b58cb9149ec02ba5e3d15ac77859 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 11:04:01 +0100 Subject: [PATCH 27/50] update --- .../out.requests.paths.txt | 2 -- .../output.txt | 25 ++++++++++++++++++- .../python_wheel_no_artifact_notebook/script | 12 +++++++-- 3 files changed, 34 insertions(+), 5 deletions(-) delete mode 100644 acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/out.requests.paths.txt diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/out.requests.paths.txt b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/out.requests.paths.txt deleted file mode 100644 index d26e76bc76..0000000000 --- a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/out.requests.paths.txt +++ /dev/null @@ -1,2 +0,0 @@ -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/output.txt b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/output.txt index b1ec4a0230..d5296837f2 100644 --- a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/output.txt +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/output.txt @@ -7,5 +7,28 @@ Deploying resources... Updating deployment state... Deployment complete! ->>> find . +=== Expecting to find 1 wheel +>>> sh -c find . | grep whl ./dist/my_test_code-0.0.1-py3-none-any.whl + +=== Expecting 1 wheel in libraries section in /jobs/create +>>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-aaaaa-bbbbbb", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + } + ], + "notebook_task": { + "notebook_path": "/notebook.py" + }, + "task_key": "TestTask" + } +] + +=== Expecting 1 wheel to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script index e59e8d76a5..6e738e0058 100644 --- a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script +++ b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script @@ -1,4 +1,12 @@ trace $CLI bundle deploy -trace find . | grep whl -jq .path < out.requests.txt | grep import | grep whl | sort > out.requests.paths.txt + +title "Expecting to find 1 wheel" +trace sh -c 'find . | grep whl' + +title "Expecting 1 wheel in libraries section in /jobs/create" +trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 1 wheel to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + rm -fr out.requests.txt build dist my_test_code.egg-info From 35dae48ba365b2be5b866f8a74b38f0188756d1b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 11:04:15 +0100 Subject: [PATCH 28/50] rename --- .../{bundle.yml => databricks.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/{bundle.yml => databricks.yml} (100%) diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/bundle.yml b/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/bundle.yml rename to acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/databricks.yml From bfb88bcfa78f764edcadf9797469afc27253d909 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 11:09:44 +0100 Subject: [PATCH 29/50] rename --- .../databricks.yml | 0 .../{python_wheel_dbfs_lib => whl_dbfs}/output.txt | 0 .../{python_wheel_dbfs_lib => whl_dbfs}/script | 0 .../{environment_key => whl_explicit}/.gitignore | 0 .../{python_wheel => whl_explicit}/databricks.yml | 0 .../my_test_code/setup.py | 0 .../my_test_code/src/__init__.py | 0 .../my_test_code/src/__main__.py | 0 .../{python_wheel => whl_explicit}/output.txt | 0 .../artifacts/{python_wheel => whl_explicit}/script | 0 .../{python_wheel => whl_implicit}/.gitignore | 0 .../databricks.yml | 0 .../src => whl_implicit/my_test_code}/__init__.py | 0 .../src => whl_implicit/my_test_code}/__main__.py | 0 .../output.txt | 0 .../script | 0 .../setup.py | 0 .../.gitignore | 0 .../databricks.yml | 0 .../output.txt | 0 .../package/my_test_code-0.0.1-py3-none-any.whl | Bin .../script | 0 .../.gitignore | 0 .../databricks.yml | 0 .../my_test_code}/__init__.py | 0 .../my_test_code}/__main__.py | 0 .../notebook.py | 0 .../output.txt | 0 .../script | 0 .../setup.py | 0 .../.gitignore | 0 .../databricks.yml | 0 .../my_test_code/setup.py | 0 .../my_test_code/setup2.py | 0 .../my_test_code/src}/__init__.py | 0 .../my_test_code/src}/__main__.py | 0 .../output.txt | 0 .../{python_wheel_multiple => whl_multiple}/script | 0 .../.gitignore | 0 .../databricks.yml | 0 .../dist/lib/my_test_code-0.0.1-py3-none-any.whl | Bin .../dist/my_test_code-0.0.1-py3-none-any.whl | Bin .../output.txt | 0 .../script | 0 .../.gitignore | 0 .../databricks.yml | 0 .../my_test_code/setup.py | 0 .../my_test_code/src}/__init__.py | 0 .../my_test_code/src}/__main__.py | 0 .../output.txt | 0 .../script | 0 51 files changed, 0 insertions(+), 0 deletions(-) rename acceptance/bundle/artifacts/{python_wheel_dbfs_lib => whl_dbfs}/databricks.yml (100%) rename acceptance/bundle/artifacts/{python_wheel_dbfs_lib => whl_dbfs}/output.txt (100%) rename acceptance/bundle/artifacts/{python_wheel_dbfs_lib => whl_dbfs}/script (100%) rename acceptance/bundle/artifacts/{environment_key => whl_explicit}/.gitignore (100%) rename acceptance/bundle/artifacts/{python_wheel => whl_explicit}/databricks.yml (100%) rename acceptance/bundle/artifacts/{environment_key => whl_explicit}/my_test_code/setup.py (100%) rename acceptance/bundle/artifacts/{environment_key => whl_explicit}/my_test_code/src/__init__.py (100%) rename acceptance/bundle/artifacts/{environment_key => whl_explicit}/my_test_code/src/__main__.py (100%) rename acceptance/bundle/artifacts/{python_wheel => whl_explicit}/output.txt (100%) rename acceptance/bundle/artifacts/{python_wheel => whl_explicit}/script (100%) rename acceptance/bundle/artifacts/{python_wheel => whl_implicit}/.gitignore (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact => whl_implicit}/databricks.yml (100%) rename acceptance/bundle/artifacts/{python_wheel/my_test_code/src => whl_implicit/my_test_code}/__init__.py (100%) rename acceptance/bundle/artifacts/{python_wheel/my_test_code/src => whl_implicit/my_test_code}/__main__.py (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact => whl_implicit}/output.txt (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact => whl_implicit}/script (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact => whl_implicit}/setup.py (100%) rename acceptance/bundle/artifacts/{python_wheel_multiple => whl_implicit_custom_path}/.gitignore (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_no_setup => whl_implicit_custom_path}/databricks.yml (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_no_setup => whl_implicit_custom_path}/output.txt (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_no_setup => whl_implicit_custom_path}/package/my_test_code-0.0.1-py3-none-any.whl (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_no_setup => whl_implicit_custom_path}/script (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact => whl_implicit_notebook}/.gitignore (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_notebook => whl_implicit_notebook}/databricks.yml (100%) rename acceptance/bundle/artifacts/{python_wheel_multiple/my_test_code/src => whl_implicit_notebook/my_test_code}/__init__.py (100%) rename acceptance/bundle/artifacts/{python_wheel_multiple/my_test_code/src => whl_implicit_notebook/my_test_code}/__main__.py (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_notebook => whl_implicit_notebook}/notebook.py (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_notebook => whl_implicit_notebook}/output.txt (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_notebook => whl_implicit_notebook}/script (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_notebook => whl_implicit_notebook}/setup.py (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_no_setup => whl_multiple}/.gitignore (100%) rename acceptance/bundle/artifacts/{python_wheel_multiple => whl_multiple}/databricks.yml (100%) rename acceptance/bundle/artifacts/{python_wheel => whl_multiple}/my_test_code/setup.py (100%) rename acceptance/bundle/artifacts/{python_wheel_multiple => whl_multiple}/my_test_code/setup2.py (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact/my_test_code => whl_multiple/my_test_code/src}/__init__.py (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact/my_test_code => whl_multiple/my_test_code/src}/__main__.py (100%) rename acceptance/bundle/artifacts/{python_wheel_multiple => whl_multiple}/output.txt (100%) rename acceptance/bundle/artifacts/{python_wheel_multiple => whl_multiple}/script (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_notebook => whl_prebuilt_multiple}/.gitignore (100%) rename acceptance/bundle/artifacts/{python_wheel_no_build => whl_prebuilt_multiple}/databricks.yml (100%) rename acceptance/bundle/artifacts/{python_wheel_no_build => whl_prebuilt_multiple}/dist/lib/my_test_code-0.0.1-py3-none-any.whl (100%) rename acceptance/bundle/artifacts/{python_wheel_no_build => whl_prebuilt_multiple}/dist/my_test_code-0.0.1-py3-none-any.whl (100%) rename acceptance/bundle/artifacts/{python_wheel_no_build => whl_prebuilt_multiple}/output.txt (100%) rename acceptance/bundle/artifacts/{python_wheel_no_build => whl_prebuilt_multiple}/script (100%) rename acceptance/bundle/artifacts/{python_wheel_no_build => whl_via_environment_key}/.gitignore (100%) rename acceptance/bundle/artifacts/{environment_key => whl_via_environment_key}/databricks.yml (100%) rename acceptance/bundle/artifacts/{python_wheel_multiple => whl_via_environment_key}/my_test_code/setup.py (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_notebook/my_test_code => whl_via_environment_key/my_test_code/src}/__init__.py (100%) rename acceptance/bundle/artifacts/{python_wheel_no_artifact_notebook/my_test_code => whl_via_environment_key/my_test_code/src}/__main__.py (100%) rename acceptance/bundle/artifacts/{environment_key => whl_via_environment_key}/output.txt (100%) rename acceptance/bundle/artifacts/{environment_key => whl_via_environment_key}/script (100%) diff --git a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/databricks.yml b/acceptance/bundle/artifacts/whl_dbfs/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_dbfs_lib/databricks.yml rename to acceptance/bundle/artifacts/whl_dbfs/databricks.yml diff --git a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_dbfs_lib/output.txt rename to acceptance/bundle/artifacts/whl_dbfs/output.txt diff --git a/acceptance/bundle/artifacts/python_wheel_dbfs_lib/script b/acceptance/bundle/artifacts/whl_dbfs/script similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_dbfs_lib/script rename to acceptance/bundle/artifacts/whl_dbfs/script diff --git a/acceptance/bundle/artifacts/environment_key/.gitignore b/acceptance/bundle/artifacts/whl_explicit/.gitignore similarity index 100% rename from acceptance/bundle/artifacts/environment_key/.gitignore rename to acceptance/bundle/artifacts/whl_explicit/.gitignore diff --git a/acceptance/bundle/artifacts/python_wheel/databricks.yml b/acceptance/bundle/artifacts/whl_explicit/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel/databricks.yml rename to acceptance/bundle/artifacts/whl_explicit/databricks.yml diff --git a/acceptance/bundle/artifacts/environment_key/my_test_code/setup.py b/acceptance/bundle/artifacts/whl_explicit/my_test_code/setup.py similarity index 100% rename from acceptance/bundle/artifacts/environment_key/my_test_code/setup.py rename to acceptance/bundle/artifacts/whl_explicit/my_test_code/setup.py diff --git a/acceptance/bundle/artifacts/environment_key/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__init__.py similarity index 100% rename from acceptance/bundle/artifacts/environment_key/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__init__.py diff --git a/acceptance/bundle/artifacts/environment_key/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__main__.py similarity index 100% rename from acceptance/bundle/artifacts/environment_key/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/whl_explicit/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/python_wheel/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt similarity index 100% rename from acceptance/bundle/artifacts/python_wheel/output.txt rename to acceptance/bundle/artifacts/whl_explicit/output.txt diff --git a/acceptance/bundle/artifacts/python_wheel/script b/acceptance/bundle/artifacts/whl_explicit/script similarity index 100% rename from acceptance/bundle/artifacts/python_wheel/script rename to acceptance/bundle/artifacts/whl_explicit/script diff --git a/acceptance/bundle/artifacts/python_wheel/.gitignore b/acceptance/bundle/artifacts/whl_implicit/.gitignore similarity index 100% rename from acceptance/bundle/artifacts/python_wheel/.gitignore rename to acceptance/bundle/artifacts/whl_implicit/.gitignore diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/databricks.yml b/acceptance/bundle/artifacts/whl_implicit/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact/databricks.yml rename to acceptance/bundle/artifacts/whl_implicit/databricks.yml diff --git a/acceptance/bundle/artifacts/python_wheel/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/whl_implicit/my_test_code/__init__.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/whl_implicit/my_test_code/__init__.py diff --git a/acceptance/bundle/artifacts/python_wheel/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/whl_implicit/my_test_code/__main__.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/whl_implicit/my_test_code/__main__.py diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact/output.txt rename to acceptance/bundle/artifacts/whl_implicit/output.txt diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/script b/acceptance/bundle/artifacts/whl_implicit/script similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact/script rename to acceptance/bundle/artifacts/whl_implicit/script diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/setup.py b/acceptance/bundle/artifacts/whl_implicit/setup.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact/setup.py rename to acceptance/bundle/artifacts/whl_implicit/setup.py diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/.gitignore b/acceptance/bundle/artifacts/whl_implicit_custom_path/.gitignore similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_multiple/.gitignore rename to acceptance/bundle/artifacts/whl_implicit_custom_path/.gitignore diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/databricks.yml b/acceptance/bundle/artifacts/whl_implicit_custom_path/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/databricks.yml rename to acceptance/bundle/artifacts/whl_implicit_custom_path/databricks.yml diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/output.txt rename to acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/whl_implicit_custom_path/package/my_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/whl_implicit_custom_path/package/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/script rename to acceptance/bundle/artifacts/whl_implicit_custom_path/script diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/.gitignore b/acceptance/bundle/artifacts/whl_implicit_notebook/.gitignore similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact/.gitignore rename to acceptance/bundle/artifacts/whl_implicit_notebook/.gitignore diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/databricks.yml b/acceptance/bundle/artifacts/whl_implicit_notebook/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/databricks.yml rename to acceptance/bundle/artifacts/whl_implicit_notebook/databricks.yml diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/src/__init__.py b/acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__init__.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/src/__init__.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__init__.py diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/src/__main__.py b/acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__main__.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/src/__main__.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/my_test_code/__main__.py diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/notebook.py b/acceptance/bundle/artifacts/whl_implicit_notebook/notebook.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/notebook.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/notebook.py diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/output.txt rename to acceptance/bundle/artifacts/whl_implicit_notebook/output.txt diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/script rename to acceptance/bundle/artifacts/whl_implicit_notebook/script diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/setup.py b/acceptance/bundle/artifacts/whl_implicit_notebook/setup.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/setup.py rename to acceptance/bundle/artifacts/whl_implicit_notebook/setup.py diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/.gitignore b/acceptance/bundle/artifacts/whl_multiple/.gitignore similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_no_setup/.gitignore rename to acceptance/bundle/artifacts/whl_multiple/.gitignore diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/databricks.yml b/acceptance/bundle/artifacts/whl_multiple/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_multiple/databricks.yml rename to acceptance/bundle/artifacts/whl_multiple/databricks.yml diff --git a/acceptance/bundle/artifacts/python_wheel/my_test_code/setup.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/setup.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel/my_test_code/setup.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/setup.py diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/setup2.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/setup2.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/setup2.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/setup2.py diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/my_test_code/__init__.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__init__.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact/my_test_code/__init__.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__init__.py diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact/my_test_code/__main__.py b/acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__main__.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact/my_test_code/__main__.py rename to acceptance/bundle/artifacts/whl_multiple/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_multiple/output.txt rename to acceptance/bundle/artifacts/whl_multiple/output.txt diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_multiple/script rename to acceptance/bundle/artifacts/whl_multiple/script diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/.gitignore b/acceptance/bundle/artifacts/whl_prebuilt_multiple/.gitignore similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/.gitignore rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/.gitignore diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/databricks.yml b/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_build/databricks.yml rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/my_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_build/dist/lib/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/my_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_build/dist/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_build/output.txt rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_build/script rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/script diff --git a/acceptance/bundle/artifacts/python_wheel_no_build/.gitignore b/acceptance/bundle/artifacts/whl_via_environment_key/.gitignore similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_build/.gitignore rename to acceptance/bundle/artifacts/whl_via_environment_key/.gitignore diff --git a/acceptance/bundle/artifacts/environment_key/databricks.yml b/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml similarity index 100% rename from acceptance/bundle/artifacts/environment_key/databricks.yml rename to acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml diff --git a/acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/setup.py b/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/setup.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_multiple/my_test_code/setup.py rename to acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/setup.py diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/my_test_code/__init__.py b/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__init__.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/my_test_code/__init__.py rename to acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__init__.py diff --git a/acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/my_test_code/__main__.py b/acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__main__.py similarity index 100% rename from acceptance/bundle/artifacts/python_wheel_no_artifact_notebook/my_test_code/__main__.py rename to acceptance/bundle/artifacts/whl_via_environment_key/my_test_code/src/__main__.py diff --git a/acceptance/bundle/artifacts/environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt similarity index 100% rename from acceptance/bundle/artifacts/environment_key/output.txt rename to acceptance/bundle/artifacts/whl_via_environment_key/output.txt diff --git a/acceptance/bundle/artifacts/environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script similarity index 100% rename from acceptance/bundle/artifacts/environment_key/script rename to acceptance/bundle/artifacts/whl_via_environment_key/script From 25a6e62bf80eccabaada94104a35ddea437ce9b2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 11:18:20 +0100 Subject: [PATCH 30/50] cleanup --- bundle/tests/enviroment_key_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bundle/tests/enviroment_key_test.go b/bundle/tests/enviroment_key_test.go index 135ef19177..c220593135 100644 --- a/bundle/tests/enviroment_key_test.go +++ b/bundle/tests/enviroment_key_test.go @@ -9,11 +9,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestEnvironmentKeySupported(t *testing.T) { - _, diags := loadTargetWithDiags("./python_wheel/environment_key", "default") - require.Empty(t, diags) -} - func TestEnvironmentKeyProvidedAndNoPanic(t *testing.T) { b, diags := loadTargetWithDiags("./environment_key_only", "default") require.Empty(t, diags) From 62bad8085c81e112e9e28b6a7bbd11a7dd9254f9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 11:56:21 +0100 Subject: [PATCH 31/50] fix duplicate --- .../artifacts/whl_prebuilt_multiple/databricks.yml | 2 +- ...y.whl => other_test_code-0.0.1-py3-none-any.whl} | Bin .../artifacts/whl_prebuilt_multiple/output.txt | 12 ++++++------ 3 files changed, 7 insertions(+), 7 deletions(-) rename acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/{my_test_code-0.0.1-py3-none-any.whl => other_test_code-0.0.1-py3-none-any.whl} (100%) diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml b/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml index e10e3993d6..4ad0c6afa5 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/databricks.yml @@ -13,4 +13,4 @@ resources: entry_point: "run" libraries: - whl: ./dist/*.whl - - whl: ./dist/lib/my_test_code-0.0.1-py3-none-any.whl + - whl: ./dist/lib/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/my_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl similarity index 100% rename from acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/my_test_code-0.0.1-py3-none-any.whl rename to acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index c69571a4d6..d4dd1c18bc 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -1,11 +1,11 @@ === Expecting to find 2 wheels, initially provided >>> sh -c find . | grep whl -./dist/lib/my_test_code-0.0.1-py3-none-any.whl +./dist/lib/other_test_code-0.0.1-py3-none-any.whl ./dist/my_test_code-0.0.1-py3-none-any.whl >>> [CLI] bundle deploy -Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading other_test_code-0.0.1-py3-none-any.whl... Uploading my_test_code-0.0.1-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... Deploying resources... @@ -14,7 +14,7 @@ Deployment complete! === Expecting to find 2 wheels, same as initially provided >>> sh -c find . | grep whl -./dist/lib/my_test_code-0.0.1-py3-none-any.whl +./dist/lib/other_test_code-0.0.1-py3-none-any.whl ./dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 2 wheels in libraries section in /jobs/create @@ -27,7 +27,7 @@ Deployment complete! "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" }, { - "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" } ], "python_wheel_task": { @@ -41,6 +41,6 @@ Deployment complete! === Expecting 2 wheels to be uploaded >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1-py3-none-any.whl" -"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/lib/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/lib/other_test_code-0.0.1-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/dist/my_test_code-0.0.1-py3-none-any.whl" From d04eb51bb88a585632541c94b1706a80486c3395 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 12:04:28 +0100 Subject: [PATCH 32/50] fixes --- acceptance/bundle/artifacts/same_name_libraries/test.toml | 2 ++ .../bundle/artifacts/unique_name_libraries/test.toml | 2 +- .../bundle/artifacts/whl_prebuilt_multiple/output.txt | 8 ++++---- acceptance/bundle/artifacts/whl_prebuilt_multiple/script | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/artifacts/same_name_libraries/test.toml b/acceptance/bundle/artifacts/same_name_libraries/test.toml index a298217f21..a17f2659f2 100644 --- a/acceptance/bundle/artifacts/same_name_libraries/test.toml +++ b/acceptance/bundle/artifacts/same_name_libraries/test.toml @@ -1,3 +1,5 @@ +RecordRequests = false + [[Repls]] Old = '\\' New = '/' diff --git a/acceptance/bundle/artifacts/unique_name_libraries/test.toml b/acceptance/bundle/artifacts/unique_name_libraries/test.toml index 280338bd6a..f548ef4fc6 100644 --- a/acceptance/bundle/artifacts/unique_name_libraries/test.toml +++ b/acceptance/bundle/artifacts/unique_name_libraries/test.toml @@ -1,4 +1,4 @@ -Cloud = false +RecordRequests = false # The order in which files are uploaded can be different, so we just replace the name [[Repls]] diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index d4dd1c18bc..81f6f6ddcf 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -5,12 +5,12 @@ ./dist/my_test_code-0.0.1-py3-none-any.whl >>> [CLI] bundle deploy -Uploading other_test_code-0.0.1-py3-none-any.whl... -Uploading my_test_code-0.0.1-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... Deploying resources... -Updating deployment state... Deployment complete! +Updating deployment state... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading other_test_code-0.0.1-py3-none-any.whl... === Expecting to find 2 wheels, same as initially provided >>> sh -c find . | grep whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script index 9396b1958b..0149deed8e 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -1,7 +1,7 @@ title "Expecting to find 2 wheels, initially provided" trace sh -c 'find . | grep whl' -trace $CLI bundle deploy +trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order title "Expecting to find 2 wheels, same as initially provided" trace sh -c 'find . | grep whl' From 0ae345108536728ea02898e4301cd49f4882fd22 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 12:07:24 +0100 Subject: [PATCH 33/50] clean up --- acceptance/bundle/artifacts/test.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/acceptance/bundle/artifacts/test.toml b/acceptance/bundle/artifacts/test.toml index dcaaffe117..b8fbfc38f8 100644 --- a/acceptance/bundle/artifacts/test.toml +++ b/acceptance/bundle/artifacts/test.toml @@ -1,5 +1,3 @@ -# Testing that the wheel is built and uploaded -Badness = "Wheel is uploaded twice, once to dist/ and once more to .internal/; A bunch of unnecessary stuff is uploaded, e.g. __pycache__" Cloud = false RecordRequests = true From a01e88efd469138d1d3c9c1d784d465ac4538399 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 12:11:47 +0100 Subject: [PATCH 34/50] clean up --- acceptance/bundle/artifacts/whl_explicit/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script index 3fc1e4c862..3ab3a795ca 100644 --- a/acceptance/bundle/artifacts/whl_explicit/script +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -9,4 +9,4 @@ trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort -rm out.requests.txt +rm -fr out.requests.txt my_test_code/build my_test_code/dist my_test_code/my_test_code.egg-info From 8c019a4bfc9598fad381069e76990e4787ca9952 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 12:39:25 +0100 Subject: [PATCH 35/50] clean up --- acceptance/bundle/artifacts/whl_multiple/script | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script index 99c7a5d72b..45efc40d96 100644 --- a/acceptance/bundle/artifacts/whl_multiple/script +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -9,4 +9,6 @@ trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting 2 wheels to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort -rm out.requests.txt +rm -fr out.requests.txt +cd my_test_code +rm -fr build dist my_test_code.egg-info my_test_code_2.egg-info From b7f2f79e2e8c1124e0c8f357bfe52bec640f2870 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 12:40:38 +0100 Subject: [PATCH 36/50] clean up --- acceptance/bundle/artifacts/whl_via_environment_key/script | 2 ++ 1 file changed, 2 insertions(+) diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index 9551f39eda..cb40753900 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -10,3 +10,5 @@ title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort rm out.requests.txt +cd my_test_code +rm -fr build dist my_test_code.egg-info From b931eae946a64d7c9621bcd2b030d5f0eb4038c2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 13:55:22 +0100 Subject: [PATCH 37/50] chore: Add virtual env setup to script.prepare --- acceptance/bundle/artifacts/script.prepare | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acceptance/bundle/artifacts/script.prepare b/acceptance/bundle/artifacts/script.prepare index eca243a532..78355bc90a 100644 --- a/acceptance/bundle/artifacts/script.prepare +++ b/acceptance/bundle/artifacts/script.prepare @@ -1 +1,5 @@ export PYTHONDONTWRITEBYTECODE=1 + +uv venv -q --python 3.12 .env +source .env/bin/activate +uv pip install -q setuptools From 67d8ebe5f0c93f18bc089623b2cf535d73142344 Mon Sep 17 00:00:00 2001 From: "Denis Bilenko (aider)" Date: Fri, 28 Feb 2025 13:55:31 +0100 Subject: [PATCH 38/50] fix: use correct venv activation path for Windows in script.prepare --- acceptance/bundle/artifacts/script.prepare | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/artifacts/script.prepare b/acceptance/bundle/artifacts/script.prepare index 78355bc90a..77c3c94411 100644 --- a/acceptance/bundle/artifacts/script.prepare +++ b/acceptance/bundle/artifacts/script.prepare @@ -1,5 +1,9 @@ export PYTHONDONTWRITEBYTECODE=1 uv venv -q --python 3.12 .env -source .env/bin/activate +if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then + source .venv/Scripts/activate +else + source .env/bin/activate +fi uv pip install -q setuptools From 7fbc7519dce7ef123e83fb11e0fdf19594823889 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 14:00:58 +0100 Subject: [PATCH 39/50] set up venv with uv --- acceptance/bundle/artifacts/script.cleanup | 1 + acceptance/bundle/artifacts/script.prepare | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 acceptance/bundle/artifacts/script.cleanup diff --git a/acceptance/bundle/artifacts/script.cleanup b/acceptance/bundle/artifacts/script.cleanup new file mode 100644 index 0000000000..ef95eb5a15 --- /dev/null +++ b/acceptance/bundle/artifacts/script.cleanup @@ -0,0 +1 @@ +rm -fr .venv diff --git a/acceptance/bundle/artifacts/script.prepare b/acceptance/bundle/artifacts/script.prepare index 77c3c94411..673b20af98 100644 --- a/acceptance/bundle/artifacts/script.prepare +++ b/acceptance/bundle/artifacts/script.prepare @@ -1,9 +1,9 @@ export PYTHONDONTWRITEBYTECODE=1 -uv venv -q --python 3.12 .env +uv venv -q --python 3.12 .venv if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then source .venv/Scripts/activate else - source .env/bin/activate + source .venv/bin/activate fi uv pip install -q setuptools From 98d392772e7a4bee72f5e13c2cffe5402c83c8f7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 14:48:26 +0100 Subject: [PATCH 40/50] fixes --- acceptance/bin/find.py | 25 +++++++++++++++++++ acceptance/bundle/artifacts/script.cleanup | 1 - acceptance/bundle/artifacts/test.toml | 7 ++++++ .../bundle/artifacts/whl_dbfs/output.txt | 6 ++--- acceptance/bundle/artifacts/whl_dbfs/script | 4 +-- .../bundle/artifacts/whl_explicit/output.txt | 5 ++-- .../bundle/artifacts/whl_explicit/script | 3 +-- .../bundle/artifacts/whl_implicit/output.txt | 5 ++-- .../bundle/artifacts/whl_implicit/script | 3 +-- .../whl_implicit_custom_path/output.txt | 5 ++-- .../artifacts/whl_implicit_custom_path/script | 3 +-- .../whl_implicit_notebook/output.txt | 5 ++-- .../artifacts/whl_implicit_notebook/script | 3 +-- .../bundle/artifacts/whl_multiple/output.txt | 7 +++--- .../bundle/artifacts/whl_multiple/script | 3 +-- .../whl_prebuilt_multiple/output.txt | 13 +++++----- .../artifacts/whl_prebuilt_multiple/script | 5 ++-- .../whl_via_environment_key/output.txt | 5 ++-- .../artifacts/whl_via_environment_key/script | 3 +-- 19 files changed, 63 insertions(+), 48 deletions(-) create mode 100755 acceptance/bin/find.py delete mode 100644 acceptance/bundle/artifacts/script.cleanup diff --git a/acceptance/bin/find.py b/acceptance/bin/find.py new file mode 100755 index 0000000000..243b574ce7 --- /dev/null +++ b/acceptance/bin/find.py @@ -0,0 +1,25 @@ +#!/usr/bin/python3 +import sys +import os +import re +import argparse + + +parser = argparse.ArgumentParser() +parser.add_argument('regex') +parser.add_argument('--expect', type=int) +args = parser.parse_args() + +regex = re.compile(args.regex) +count = 0 + +for root, dirs, files in os.walk("."): + for filename in files: + path = os.path.join(root, filename).lstrip('./') + if regex.search(path): + print(path) + count += 1 + +if args.expect is not None: + if args.expect != count: + sys.exit(f'Expected {args.expect}, got {count}') diff --git a/acceptance/bundle/artifacts/script.cleanup b/acceptance/bundle/artifacts/script.cleanup deleted file mode 100644 index ef95eb5a15..0000000000 --- a/acceptance/bundle/artifacts/script.cleanup +++ /dev/null @@ -1 +0,0 @@ -rm -fr .venv diff --git a/acceptance/bundle/artifacts/test.toml b/acceptance/bundle/artifacts/test.toml index b8fbfc38f8..42432706e1 100644 --- a/acceptance/bundle/artifacts/test.toml +++ b/acceptance/bundle/artifacts/test.toml @@ -1,5 +1,12 @@ Cloud = false RecordRequests = true +Ignore = [ + '.venv', + 'dist', + 'build', + '*egg-info', + '.databricks', +] [[Server]] Pattern = "GET /api/2.1/clusters/get" diff --git a/acceptance/bundle/artifacts/whl_dbfs/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt index 9758d77495..4cfca5d00e 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/output.txt +++ b/acceptance/bundle/artifacts/whl_dbfs/output.txt @@ -5,10 +5,8 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Expecting to find no wheels, must error ->>> errcode sh -c find . | grep whl - -Exit code: 1 +=== Expecting to find no wheels +>>> errcode find.py --expect 0 whl === Expecting 1 wheel in libraries section in /jobs/create >>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_dbfs/script b/acceptance/bundle/artifacts/whl_dbfs/script index aa075451ea..19ae132b75 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/script +++ b/acceptance/bundle/artifacts/whl_dbfs/script @@ -1,7 +1,7 @@ trace $CLI bundle deploy -title "Expecting to find no wheels, must error" -trace errcode sh -c 'find . | grep whl' +title "Expecting to find no wheels" +trace errcode find.py --expect 0 whl title "Expecting 1 wheel in libraries section in /jobs/create" trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt index 34abcca1df..f539bf6518 100644 --- a/acceptance/bundle/artifacts/whl_explicit/output.txt +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -7,9 +7,8 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Expecting to find 1 wheel ->>> sh -c find . | grep whl -./my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +>>> find.py --expect 1 whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/create >>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script index 3ab3a795ca..8bf338dc73 100644 --- a/acceptance/bundle/artifacts/whl_explicit/script +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -1,7 +1,6 @@ trace $CLI bundle deploy -title "Expecting to find 1 wheel" -trace sh -c 'find . | grep whl' +trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt index 1a829648c7..e689d93145 100644 --- a/acceptance/bundle/artifacts/whl_implicit/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -7,9 +7,8 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Expecting to find 1 wheel ->>> sh -c find . | grep whl -./dist/my_test_code-0.0.1-py3-none-any.whl +>>> find.py --expect 1 whl +dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheels in libraries section in /jobs/create >>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit/script b/acceptance/bundle/artifacts/whl_implicit/script index e41d052d77..afc29eb6d7 100644 --- a/acceptance/bundle/artifacts/whl_implicit/script +++ b/acceptance/bundle/artifacts/whl_implicit/script @@ -1,7 +1,6 @@ trace $CLI bundle deploy -title "Expecting to find 1 wheel" -trace sh -c 'find . | grep whl' +trace find.py --expect 1 whl title "Expecting 1 wheels in libraries section in /jobs/create" trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt index 037c88dbee..80726c0c77 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -6,9 +6,8 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Expecting to find 1 wheel ->>> sh -c find . | grep whl -./package/my_test_code-0.0.1-py3-none-any.whl +>>> find.py --expect 1 whl +package/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/create >>> jq select(.path=="/api/2.1/jobs/create") | .body out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index f84ca5a195..037274a7a6 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -1,7 +1,6 @@ trace $CLI bundle deploy -title "Expecting to find 1 wheel" -trace sh -c 'find . | grep whl' +trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" trace jq 'select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt index d5296837f2..e5745bf57e 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -7,9 +7,8 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Expecting to find 1 wheel ->>> sh -c find . | grep whl -./dist/my_test_code-0.0.1-py3-none-any.whl +>>> find.py --expect 1 whl +dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/create >>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script index 6e738e0058..7bfbffafc0 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/script +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/script @@ -1,7 +1,6 @@ trace $CLI bundle deploy -title "Expecting to find 1 wheel" -trace sh -c 'find . | grep whl' +trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt index d6f99ba6e6..316dc9458c 100644 --- a/acceptance/bundle/artifacts/whl_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -9,10 +9,9 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/defau Uploading my_test_code-0.0.1-py3-none-any.whl... Uploading my_test_code_2-0.0.1-py3-none-any.whl... -=== Expecting to find 2 wheels ->>> sh -c find . | grep whl -./my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl -./my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +>>> find.py --expect 2 whl +my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 2 wheels in libraries section in /jobs/create >>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script index 45efc40d96..81c77ee40e 100644 --- a/acceptance/bundle/artifacts/whl_multiple/script +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -1,7 +1,6 @@ trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order -title "Expecting to find 2 wheels" -trace sh -c 'find . | grep whl' # expecting 2 wheels +trace find.py --expect 2 whl title "Expecting 2 wheels in libraries section in /jobs/create" trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index 81f6f6ddcf..092db59c0a 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -1,8 +1,7 @@ -=== Expecting to find 2 wheels, initially provided ->>> sh -c find . | grep whl -./dist/lib/other_test_code-0.0.1-py3-none-any.whl -./dist/my_test_code-0.0.1-py3-none-any.whl +>>> find.py --expect 2 whl +dist/my_test_code-0.0.1-py3-none-any.whl +dist/lib/other_test_code-0.0.1-py3-none-any.whl >>> [CLI] bundle deploy Deploying resources... @@ -13,9 +12,9 @@ Uploading my_test_code-0.0.1-py3-none-any.whl... Uploading other_test_code-0.0.1-py3-none-any.whl... === Expecting to find 2 wheels, same as initially provided ->>> sh -c find . | grep whl -./dist/lib/other_test_code-0.0.1-py3-none-any.whl -./dist/my_test_code-0.0.1-py3-none-any.whl +>>> find.py --expect 2 whl +dist/my_test_code-0.0.1-py3-none-any.whl +dist/lib/other_test_code-0.0.1-py3-none-any.whl === Expecting 2 wheels in libraries section in /jobs/create >>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script index 0149deed8e..fbb44434dd 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -1,10 +1,9 @@ -title "Expecting to find 2 wheels, initially provided" -trace sh -c 'find . | grep whl' +trace find.py --expect 2 whl trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order title "Expecting to find 2 wheels, same as initially provided" -trace sh -c 'find . | grep whl' +trace find.py --expect 2 whl title "Expecting 2 wheels in libraries section in /jobs/create" trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt index 1f53dd5dd7..29dbb9b552 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -7,9 +7,8 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Expecting to find 1 wheel ->>> sh -c find . | grep whl -./my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +>>> find.py --expect 1 whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in environments section in /jobs/create >>> jq select(.path=="/api/2.1/jobs/create") | .body out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index cb40753900..7957daddf7 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -1,7 +1,6 @@ trace $CLI bundle deploy -title "Expecting to find 1 wheel" -trace sh -c 'find . | grep whl' +trace find.py --expect 1 whl title "Expecting 1 wheel in environments section in /jobs/create" trace jq 'select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt From 70d8d6914bbfc577ad9e19a39fe49558a5240960 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 14:59:47 +0100 Subject: [PATCH 41/50] ruff --- acceptance/bin/find.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bin/find.py b/acceptance/bin/find.py index 243b574ce7..3ae12a9cae 100755 --- a/acceptance/bin/find.py +++ b/acceptance/bin/find.py @@ -6,8 +6,8 @@ parser = argparse.ArgumentParser() -parser.add_argument('regex') -parser.add_argument('--expect', type=int) +parser.add_argument("regex") +parser.add_argument("--expect", type=int) args = parser.parse_args() regex = re.compile(args.regex) @@ -15,11 +15,11 @@ for root, dirs, files in os.walk("."): for filename in files: - path = os.path.join(root, filename).lstrip('./') + path = os.path.join(root, filename).lstrip("./") if regex.search(path): print(path) count += 1 if args.expect is not None: if args.expect != count: - sys.exit(f'Expected {args.expect}, got {count}') + sys.exit(f"Expected {args.expect}, got {count}") From bd13b501e90d9530489f6b584ba8e499a2c2101a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 15:02:56 +0100 Subject: [PATCH 42/50] fix shebang --- acceptance/bin/find.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bin/find.py b/acceptance/bin/find.py index 3ae12a9cae..bc69849d8f 100755 --- a/acceptance/bin/find.py +++ b/acceptance/bin/find.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!/usr/bin/env python3 import sys import os import re From dcd100ed322b412b89959ae026560126b2607899 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 15:12:11 +0100 Subject: [PATCH 43/50] fix slashes in find.py on windows --- acceptance/bin/find.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bin/find.py b/acceptance/bin/find.py index bc69849d8f..2772d1c596 100755 --- a/acceptance/bin/find.py +++ b/acceptance/bin/find.py @@ -15,7 +15,7 @@ for root, dirs, files in os.walk("."): for filename in files: - path = os.path.join(root, filename).lstrip("./") + path = os.path.join(root, filename).lstrip("./\\").replace('\\', '/') if regex.search(path): print(path) count += 1 From 1b97ac37128936bd58171311b0e0874d767164f6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 15:30:58 +0100 Subject: [PATCH 44/50] ruff --- acceptance/bin/find.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bin/find.py b/acceptance/bin/find.py index 2772d1c596..d95c7e5efd 100755 --- a/acceptance/bin/find.py +++ b/acceptance/bin/find.py @@ -15,7 +15,7 @@ for root, dirs, files in os.walk("."): for filename in files: - path = os.path.join(root, filename).lstrip("./\\").replace('\\', '/') + path = os.path.join(root, filename).lstrip("./\\").replace("\\", "/") if regex.search(path): print(path) count += 1 From d538e9e97c8ace06f7f6a476baf0a3cff388ff52 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 15:37:55 +0100 Subject: [PATCH 45/50] fix windows - do not use python3, it is not part of virtualenv --- acceptance/bundle/artifacts/whl_explicit/databricks.yml | 2 +- acceptance/bundle/artifacts/whl_multiple/databricks.yml | 4 ++-- .../bundle/artifacts/whl_via_environment_key/databricks.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/artifacts/whl_explicit/databricks.yml b/acceptance/bundle/artifacts/whl_explicit/databricks.yml index 017fe1c431..679c72b372 100644 --- a/acceptance/bundle/artifacts/whl_explicit/databricks.yml +++ b/acceptance/bundle/artifacts/whl_explicit/databricks.yml @@ -5,7 +5,7 @@ artifacts: my_test_code: type: whl path: "./my_test_code" - build: "python3 setup.py bdist_wheel" + build: python setup.py bdist_wheel resources: jobs: diff --git a/acceptance/bundle/artifacts/whl_multiple/databricks.yml b/acceptance/bundle/artifacts/whl_multiple/databricks.yml index 770110416d..2bfd85abd0 100644 --- a/acceptance/bundle/artifacts/whl_multiple/databricks.yml +++ b/acceptance/bundle/artifacts/whl_multiple/databricks.yml @@ -5,11 +5,11 @@ artifacts: my_test_code: type: whl path: "./my_test_code" - build: "python3 setup.py bdist_wheel" + build: "python setup.py bdist_wheel" my_test_code_2: type: whl path: "./my_test_code" - build: "python3 setup2.py bdist_wheel" + build: "python setup2.py bdist_wheel" resources: jobs: diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml b/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml index 198f8c0d2a..4ca3f91136 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml +++ b/acceptance/bundle/artifacts/whl_via_environment_key/databricks.yml @@ -5,7 +5,7 @@ artifacts: my_test_code: type: whl path: "./my_test_code" - build: "python3 setup.py bdist_wheel" + build: python setup.py bdist_wheel resources: jobs: From f608acad3ccf2fe301c0f17dd96236bcd3e04cbe Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 15:39:41 +0100 Subject: [PATCH 46/50] clean up unnecessary rm --- acceptance/bundle/artifacts/whl_explicit/script | 4 ++-- acceptance/bundle/artifacts/whl_implicit/script | 2 +- acceptance/bundle/artifacts/whl_implicit_notebook/script | 2 +- acceptance/bundle/artifacts/whl_multiple/script | 2 -- acceptance/bundle/artifacts/whl_via_environment_key/script | 2 -- 5 files changed, 4 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script index 8bf338dc73..bb7e26ae1f 100644 --- a/acceptance/bundle/artifacts/whl_explicit/script +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -3,9 +3,9 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" -trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort -rm -fr out.requests.txt my_test_code/build my_test_code/dist my_test_code/my_test_code.egg-info +rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit/script b/acceptance/bundle/artifacts/whl_implicit/script index afc29eb6d7..a6d09eeb0c 100644 --- a/acceptance/bundle/artifacts/whl_implicit/script +++ b/acceptance/bundle/artifacts/whl_implicit/script @@ -8,4 +8,4 @@ trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheels to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort -rm -fr out.requests.txt build dist my_test_code.egg-info +rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script index 7bfbffafc0..631dc9422d 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/script +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/script @@ -8,4 +8,4 @@ trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort -rm -fr out.requests.txt build dist my_test_code.egg-info +rm out.requests.txt diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script index 81c77ee40e..de218d0100 100644 --- a/acceptance/bundle/artifacts/whl_multiple/script +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -9,5 +9,3 @@ title "Expecting 2 wheels to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort rm -fr out.requests.txt -cd my_test_code -rm -fr build dist my_test_code.egg-info my_test_code_2.egg-info diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index 7957daddf7..a73dd1ffc7 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -9,5 +9,3 @@ title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort rm out.requests.txt -cd my_test_code -rm -fr build dist my_test_code.egg-info From fe50cb1e43132fc7cbadc2c4335cb544ee02129e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 15:59:09 +0100 Subject: [PATCH 47/50] clean up --- acceptance/bundle/artifacts/whl_explicit/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script index bb7e26ae1f..631dc9422d 100644 --- a/acceptance/bundle/artifacts/whl_explicit/script +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" -trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt +trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort From 176c247714e042392a17cb72d5f25fbe3cd3405b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 17:21:08 +0100 Subject: [PATCH 48/50] use jq -s '.[] | X' instead of 'jq -s X', this helps on windows --- acceptance/bundle/artifacts/whl_implicit/output.txt | 2 +- acceptance/bundle/artifacts/whl_implicit/script | 2 +- acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt | 2 +- acceptance/bundle/artifacts/whl_implicit_custom_path/script | 2 +- acceptance/bundle/artifacts/whl_implicit_notebook/output.txt | 2 +- acceptance/bundle/artifacts/whl_implicit_notebook/script | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt index e689d93145..69ff56c42c 100644 --- a/acceptance/bundle/artifacts/whl_implicit/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -11,7 +11,7 @@ Deployment complete! dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheels in libraries section in /jobs/create ->>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt [ { "existing_cluster_id": "0717-aaaaa-bbbbbb", diff --git a/acceptance/bundle/artifacts/whl_implicit/script b/acceptance/bundle/artifacts/whl_implicit/script index a6d09eeb0c..78c4d75e07 100644 --- a/acceptance/bundle/artifacts/whl_implicit/script +++ b/acceptance/bundle/artifacts/whl_implicit/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheels in libraries section in /jobs/create" -trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheels to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt index 80726c0c77..0658dce3ab 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -10,7 +10,7 @@ Deployment complete! package/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/create ->>> jq select(.path=="/api/2.1/jobs/create") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body out.requests.txt { "deployment": { "kind": "BUNDLE", diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/script b/acceptance/bundle/artifacts/whl_implicit_custom_path/script index 037274a7a6..d4c2438dbb 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/script +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" -trace jq 'select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt index e5745bf57e..9c7296b9a6 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -11,7 +11,7 @@ Deployment complete! dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/create ->>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt [ { "existing_cluster_id": "0717-aaaaa-bbbbbb", diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/script b/acceptance/bundle/artifacts/whl_implicit_notebook/script index 631dc9422d..bb7e26ae1f 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/script +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" -trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort From 8f65743221acd32d73d7fbed01e76139218def7c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 17:28:20 +0100 Subject: [PATCH 49/50] sort results in find.py; more use of 'jq -s' --- acceptance/bin/find.py | 14 +++++++++----- acceptance/bundle/artifacts/whl_dbfs/output.txt | 2 +- acceptance/bundle/artifacts/whl_dbfs/script | 2 +- .../bundle/artifacts/whl_explicit/output.txt | 2 +- acceptance/bundle/artifacts/whl_explicit/script | 2 +- .../bundle/artifacts/whl_multiple/output.txt | 4 ++-- acceptance/bundle/artifacts/whl_multiple/script | 2 +- .../artifacts/whl_prebuilt_multiple/output.txt | 6 +++--- .../bundle/artifacts/whl_prebuilt_multiple/script | 2 +- .../artifacts/whl_via_environment_key/output.txt | 2 +- .../artifacts/whl_via_environment_key/script | 2 +- 11 files changed, 22 insertions(+), 18 deletions(-) diff --git a/acceptance/bin/find.py b/acceptance/bin/find.py index d95c7e5efd..0acc8f1e5c 100755 --- a/acceptance/bin/find.py +++ b/acceptance/bin/find.py @@ -11,15 +11,19 @@ args = parser.parse_args() regex = re.compile(args.regex) -count = 0 +result = [] for root, dirs, files in os.walk("."): for filename in files: path = os.path.join(root, filename).lstrip("./\\").replace("\\", "/") if regex.search(path): - print(path) - count += 1 + result.append(path) + +result.sort() +for item in result: + print(item) +sys.stdout.flush() if args.expect is not None: - if args.expect != count: - sys.exit(f"Expected {args.expect}, got {count}") + if args.expect != len(result): + sys.exit(f"Expected {args.expect}, got {len(result)}") diff --git a/acceptance/bundle/artifacts/whl_dbfs/output.txt b/acceptance/bundle/artifacts/whl_dbfs/output.txt index 4cfca5d00e..f0615c558a 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/output.txt +++ b/acceptance/bundle/artifacts/whl_dbfs/output.txt @@ -9,7 +9,7 @@ Deployment complete! >>> errcode find.py --expect 0 whl === Expecting 1 wheel in libraries section in /jobs/create ->>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt [ { "existing_cluster_id": "0717-132531-5opeqon1", diff --git a/acceptance/bundle/artifacts/whl_dbfs/script b/acceptance/bundle/artifacts/whl_dbfs/script index 19ae132b75..d3b12cb8e9 100644 --- a/acceptance/bundle/artifacts/whl_dbfs/script +++ b/acceptance/bundle/artifacts/whl_dbfs/script @@ -4,7 +4,7 @@ title "Expecting to find no wheels" trace errcode find.py --expect 0 whl title "Expecting 1 wheel in libraries section in /jobs/create" -trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting no wheels to be uploaded" trace errcode sh -c 'jq .path < out.requests.txt | grep import | grep whl' diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt index f539bf6518..1018501db5 100644 --- a/acceptance/bundle/artifacts/whl_explicit/output.txt +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -11,7 +11,7 @@ Deployment complete! my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in libraries section in /jobs/create ->>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt [ { "existing_cluster_id": "0717-132531-5opeqon1", diff --git a/acceptance/bundle/artifacts/whl_explicit/script b/acceptance/bundle/artifacts/whl_explicit/script index 631dc9422d..bb7e26ae1f 100644 --- a/acceptance/bundle/artifacts/whl_explicit/script +++ b/acceptance/bundle/artifacts/whl_explicit/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in libraries section in /jobs/create" -trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt index 316dc9458c..9335b9cc57 100644 --- a/acceptance/bundle/artifacts/whl_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -10,11 +10,11 @@ Uploading my_test_code-0.0.1-py3-none-any.whl... Uploading my_test_code_2-0.0.1-py3-none-any.whl... >>> find.py --expect 2 whl -my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl === Expecting 2 wheels in libraries section in /jobs/create ->>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt [ { "existing_cluster_id": "0717-132531-5opeqon1", diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script index de218d0100..a475e9f730 100644 --- a/acceptance/bundle/artifacts/whl_multiple/script +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' me trace find.py --expect 2 whl title "Expecting 2 wheels in libraries section in /jobs/create" -trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting 2 wheels to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index 092db59c0a..c9cd82fb88 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -1,7 +1,7 @@ >>> find.py --expect 2 whl -dist/my_test_code-0.0.1-py3-none-any.whl dist/lib/other_test_code-0.0.1-py3-none-any.whl +dist/my_test_code-0.0.1-py3-none-any.whl >>> [CLI] bundle deploy Deploying resources... @@ -13,11 +13,11 @@ Uploading other_test_code-0.0.1-py3-none-any.whl... === Expecting to find 2 wheels, same as initially provided >>> find.py --expect 2 whl -dist/my_test_code-0.0.1-py3-none-any.whl dist/lib/other_test_code-0.0.1-py3-none-any.whl +dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 2 wheels in libraries section in /jobs/create ->>> jq select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt [ { "existing_cluster_id": "0717-132531-5opeqon1", diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script index fbb44434dd..5265151e26 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -6,7 +6,7 @@ title "Expecting to find 2 wheels, same as initially provided" trace find.py --expect 2 whl title "Expecting 2 wheels in libraries section in /jobs/create" -trace jq 'select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt title "Expecting 2 wheels to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt index 29dbb9b552..8b6b781aae 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -11,7 +11,7 @@ Deployment complete! my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl === Expecting 1 wheel in environments section in /jobs/create ->>> jq select(.path=="/api/2.1/jobs/create") | .body out.requests.txt +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body out.requests.txt { "deployment": { "kind": "BUNDLE", diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/script b/acceptance/bundle/artifacts/whl_via_environment_key/script index a73dd1ffc7..1914aeb8c5 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/script +++ b/acceptance/bundle/artifacts/whl_via_environment_key/script @@ -3,7 +3,7 @@ trace $CLI bundle deploy trace find.py --expect 1 whl title "Expecting 1 wheel in environments section in /jobs/create" -trace jq 'select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body' out.requests.txt title "Expecting 1 wheel to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort From 7dd0c0ef23a94a3bf649fff1f369a9deb3b74e7e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Feb 2025 17:32:29 +0100 Subject: [PATCH 50/50] add comments --- acceptance/bin/find.py | 7 +++++++ acceptance/bundle/artifacts/whl_explicit/databricks.yml | 1 + 2 files changed, 8 insertions(+) diff --git a/acceptance/bin/find.py b/acceptance/bin/find.py index 0acc8f1e5c..d122404b2f 100755 --- a/acceptance/bin/find.py +++ b/acceptance/bin/find.py @@ -1,4 +1,11 @@ #!/usr/bin/env python3 +""" +Usage: find.py +Finds all files within current directory matching regex. The output is sorted and slashes are always forward. + +If --expect N is provided, the number of matches must be N or error is printed. +""" + import sys import os import re diff --git a/acceptance/bundle/artifacts/whl_explicit/databricks.yml b/acceptance/bundle/artifacts/whl_explicit/databricks.yml index 679c72b372..45106c0b00 100644 --- a/acceptance/bundle/artifacts/whl_explicit/databricks.yml +++ b/acceptance/bundle/artifacts/whl_explicit/databricks.yml @@ -5,6 +5,7 @@ artifacts: my_test_code: type: whl path: "./my_test_code" + # using 'python' there because 'python3' does not exist in virtualenv on windows build: python setup.py bdist_wheel resources: