Skip to content

Commit 2c0746b

Browse files
denikdenik
authored
Convert default-python integration test to acceptance test (#2532)
## Changes - Convert init_default_python_test.go to acceptance test - Record dev targets in addition to prod targets. - Record validate json output in addition to summary json. - Add venv_activate test helper. Note, GCP is disabled in this test due to differences from the aws and azure (IsServicePrincipal is false there, causing different config). Depends on #2535 ## Why init_default_python_test.go is one of a kind test - it uses golden files, but it's a separate thing from the rest of acceptance tests. This change makes it easier to update it with the rest of acceptance tests.
1 parent f7e9afd commit 2c0746b

File tree

11 files changed

+548
-256
lines changed

11 files changed

+548
-256
lines changed

‎acceptance/bundle/artifacts/script.prepare‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
11
export PYTHONDONTWRITEBYTECODE=1
22

33
uv venv -q .venv
4-
if [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" || "$OSTYPE" == "win32" ]]; then
5-
source .venv/Scripts/activate
6-
else
7-
source .venv/bin/activate
8-
fi
4+
venv_activate
95
uv pip install -q setuptools

‎acceptance/bundle/templates/default-python/integration_classic/input.json.tmpl‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"project_name": "project_name_$UNIQUE_NAME",
3+
"include_notebook": "yes",
4+
"include_dlt": "yes",
5+
"include_python": "yes",
6+
"serverless": "no"
7+
}

integration/bundle/testdata/default_python/bundle_summary.txt renamed to acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json

Lines changed: 49 additions & 88 deletions
Original file line numberDiff line numberDiff line change
@@ -1,86 +1,49 @@
11
{
2+
"artifacts": {
3+
"python_artifact": {
4+
"build": "python3 setup.py bdist_wheel",
5+
"path": "[TEST_TMP_DIR]/project_name_[UNIQUE_NAME]",
6+
"type": "whl"
7+
}
8+
},
29
"bundle": {
3-
"name": "project_name_$UNIQUE_PRJ",
4-
"target": "dev",
5-
"environment": "dev",
6-
"terraform": {
7-
"exec_path": "/tmp/.../terraform"
8-
},
9-
"git": {
10-
"bundle_root_path": "."
11-
},
12-
"mode": "development",
1310
"deployment": {
1411
"lock": {
1512
"enabled": false
1613
}
1714
},
15+
"environment": "dev",
16+
"git": {
17+
"bundle_root_path": "."
18+
},
19+
"mode": "development",
20+
"name": "project_name_[UNIQUE_NAME]",
21+
"target": "dev",
22+
"terraform": {},
1823
"uuid": "[UUID]"
1924
},
2025
"include": [
21-
"resources/project_name_$UNIQUE_PRJ.job.yml",
22-
"resources/project_name_$UNIQUE_PRJ.pipeline.yml"
26+
"resources/project_name_[UNIQUE_NAME].job.yml",
27+
"resources/project_name_[UNIQUE_NAME].pipeline.yml"
2328
],
24-
"workspace": {
25-
"host": "[DATABRICKS_URL]",
26-
"current_user": {
27-
"active": true,
28-
"displayName": "[USERNAME]",
29-
"emails": [
30-
{
31-
"primary": true,
32-
"type": "work",
33-
"value": "[USERNAME]"
34-
}
35-
],
36-
"groups": [
37-
{
38-
"$ref": "Groups/[USERGROUP]",
39-
"display": "team.engineering",
40-
"type": "direct",
41-
"value": "[USERGROUP]"
42-
}
43-
],
44-
"id": "[USERID]",
45-
"name": {
46-
"familyName": "[USERNAME]",
47-
"givenName": "[USERNAME]"
48-
},
49-
"schemas": [
50-
"urn:ietf:params:scim:schemas:core:2.0:User",
51-
"urn:ietf:params:scim:schemas:extension:workspace:2.0:User"
52-
],
53-
"short_name": "[USERNAME]",
54-
"userName": "[USERNAME]"
29+
"presets": {
30+
"jobs_max_concurrent_runs": 4,
31+
"name_prefix": "[dev [USERNAME]] ",
32+
"pipelines_development": true,
33+
"tags": {
34+
"dev": "[USERNAME]"
5535
},
56-
"root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev",
57-
"file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files",
58-
"resource_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/resources",
59-
"artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/artifacts",
60-
"state_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/state"
61-
},
62-
"artifacts": {
63-
"python_artifact": {
64-
"type": "whl",
65-
"path": "/tmp/.../project_name_$UNIQUE_PRJ",
66-
"build": "python3 setup.py bdist_wheel"
67-
}
36+
"trigger_pause_status": "PAUSED"
6837
},
6938
"resources": {
7039
"jobs": {
71-
"project_name_$UNIQUE_PRJ_job": {
40+
"project_name_[UNIQUE_NAME]_job": {
7241
"deployment": {
7342
"kind": "BUNDLE",
74-
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json"
43+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json"
7544
},
7645
"edit_mode": "UI_LOCKED",
77-
"email_notifications": {
78-
"on_failure": [
79-
"[USERNAME]"
80-
]
81-
},
8246
"format": "MULTI_TASK",
83-
"id": "[NUMID]",
8447
"job_clusters": [
8548
{
8649
"job_cluster_key": "job_cluster",
@@ -90,13 +53,14 @@
9053
"min_workers": 1
9154
},
9255
"data_security_mode": "SINGLE_USER",
93-
"node_type_id": "i3.xlarge",
56+
"node_type_id": "[NODE_TYPE_ID]",
9457
"spark_version": "15.4.x-scala2.12"
9558
}
9659
}
9760
],
9861
"max_concurrent_runs": 4,
99-
"name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_job",
62+
"name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_job",
63+
"permissions": [],
10064
"queue": {
10165
"enabled": true
10266
},
@@ -107,7 +71,7 @@
10771
{
10872
"job_cluster_key": "job_cluster",
10973
"notebook_task": {
110-
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/notebook"
74+
"notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook"
11175
},
11276
"task_key": "notebook_task"
11377
},
@@ -118,7 +82,7 @@
11882
}
11983
],
12084
"pipeline_task": {
121-
"pipeline_id": "${resources.pipelines.project_name_$UNIQUE_PRJ_pipeline.id}"
85+
"pipeline_id": "${resources.pipelines.project_name_[UNIQUE_NAME]_pipeline.id}"
12286
},
12387
"task_key": "refresh_pipeline"
12488
},
@@ -136,7 +100,7 @@
136100
],
137101
"python_wheel_task": {
138102
"entry_point": "main",
139-
"package_name": "project_name_$UNIQUE_PRJ"
103+
"package_name": "project_name_[UNIQUE_NAME]"
140104
},
141105
"task_key": "main_task"
142106
}
@@ -147,32 +111,29 @@
147111
"interval": 1,
148112
"unit": "DAYS"
149113
}
150-
},
151-
"url": "[DATABRICKS_URL]/jobs/[NUMID]?o=[NUMID]"
114+
}
152115
}
153116
},
154117
"pipelines": {
155-
"project_name_$UNIQUE_PRJ_pipeline": {
156-
"catalog": "main",
118+
"project_name_[UNIQUE_NAME]_pipeline": {
157119
"configuration": {
158-
"bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files/src"
120+
"bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src"
159121
},
160122
"deployment": {
161123
"kind": "BUNDLE",
162-
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json"
124+
"metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json"
163125
},
164126
"development": true,
165-
"id": "[UUID]",
166127
"libraries": [
167128
{
168129
"notebook": {
169-
"path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files/src/dlt_pipeline"
130+
"path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/dlt_pipeline"
170131
}
171132
}
172133
],
173-
"name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_pipeline",
174-
"schema": "project_name_$UNIQUE_PRJ_dev",
175-
"url": "[DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]"
134+
"name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline",
135+
"permissions": [],
136+
"schema": "project_name_[UNIQUE_NAME]_dev"
176137
}
177138
}
178139
},
@@ -181,13 +142,13 @@
181142
"."
182143
]
183144
},
184-
"presets": {
185-
"name_prefix": "[dev [USERNAME]] ",
186-
"pipelines_development": true,
187-
"trigger_pause_status": "PAUSED",
188-
"jobs_max_concurrent_runs": 4,
189-
"tags": {
190-
"dev": "[USERNAME]"
191-
}
145+
"targets": null,
146+
"workspace": {
147+
"artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts",
148+
"file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files",
149+
"host": "[DATABRICKS_URL]",
150+
"resource_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/resources",
151+
"root_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev",
152+
"state_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state"
192153
}
193-
}
154+
}

0 commit comments

Comments
 (0)