Skip to content

Commit 2f0a9e3

Browse files
shreyas-goenkashreyas-goenka
authored
Narrow scope of telemetry acceptance tests (#3053)
## Changes This PR narrows the telemetry acceptance tests to on assert on the interesting part of the telemetry request payload instead of asserting on the entire payload. ## Why Having more targeted tests will result in a smaller difference when adding new metrics that we want to track in telemetry. Today, any new fields added to telemetry end up touching multiple tests. This PR will prevent that moving forward.
1 parent c9354c0 commit 2f0a9e3

File tree

21 files changed

+145
-663
lines changed

21 files changed

+145
-663
lines changed

‎acceptance/bundle/telemetry/deploy-artifact-path-type/output.txt‎

Lines changed: 2 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -11,98 +11,8 @@ Deployment complete!
1111

1212
>>> cat out.requests.txt
1313
{
14-
"bundle_uuid": "[UUID]",
15-
"deployment_id": "[UUID]",
16-
"resource_count": 0,
17-
"resource_job_count": 0,
18-
"resource_pipeline_count": 0,
19-
"resource_model_count": 0,
20-
"resource_experiment_count": 0,
21-
"resource_model_serving_endpoint_count": 0,
22-
"resource_registered_model_count": 0,
23-
"resource_quality_monitor_count": 0,
24-
"resource_schema_count": 0,
25-
"resource_volume_count": 0,
26-
"resource_cluster_count": 0,
27-
"resource_dashboard_count": 0,
28-
"resource_app_count": 0,
29-
"experimental": {
30-
"configuration_file_count": 1,
31-
"variable_count": 0,
32-
"complex_variable_count": 0,
33-
"lookup_variable_count": 0,
34-
"target_count": 2,
35-
"bool_values": [
36-
{
37-
"key": "skip_artifact_cleanup",
38-
"value": false
39-
},
40-
{
41-
"key": "python_wheel_wrapper_is_set",
42-
"value": false
43-
},
44-
{
45-
"key": "has_serverless_compute",
46-
"value": false
47-
},
48-
{
49-
"key": "has_classic_job_compute",
50-
"value": false
51-
},
52-
{
53-
"key": "has_classic_interactive_compute",
54-
"value": false
55-
}
56-
],
57-
"bundle_mode": "TYPE_UNSPECIFIED",
58-
"workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM"
59-
}
14+
"workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM"
6015
}
6116
{
62-
"bundle_uuid": "[UUID]",
63-
"deployment_id": "[UUID]",
64-
"resource_count": 0,
65-
"resource_job_count": 0,
66-
"resource_pipeline_count": 0,
67-
"resource_model_count": 0,
68-
"resource_experiment_count": 0,
69-
"resource_model_serving_endpoint_count": 0,
70-
"resource_registered_model_count": 0,
71-
"resource_quality_monitor_count": 0,
72-
"resource_schema_count": 0,
73-
"resource_volume_count": 0,
74-
"resource_cluster_count": 0,
75-
"resource_dashboard_count": 0,
76-
"resource_app_count": 0,
77-
"experimental": {
78-
"configuration_file_count": 1,
79-
"variable_count": 0,
80-
"complex_variable_count": 0,
81-
"lookup_variable_count": 0,
82-
"target_count": 2,
83-
"bool_values": [
84-
{
85-
"key": "skip_artifact_cleanup",
86-
"value": false
87-
},
88-
{
89-
"key": "python_wheel_wrapper_is_set",
90-
"value": false
91-
},
92-
{
93-
"key": "has_serverless_compute",
94-
"value": false
95-
},
96-
{
97-
"key": "has_classic_job_compute",
98-
"value": false
99-
},
100-
{
101-
"key": "has_classic_interactive_compute",
102-
"value": false
103-
}
104-
],
105-
"bundle_mode": "TYPE_UNSPECIFIED",
106-
"workspace_artifact_path_type": "UC_VOLUME"
107-
}
17+
"workspace_artifact_path_type": "UC_VOLUME"
10818
}

‎acceptance/bundle/telemetry/deploy-artifact-path-type/script‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@ trace $CLI bundle deploy -t one
22

33
trace $CLI bundle deploy -t two
44

5-
trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event'
5+
trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {workspace_artifact_path_type}'
66

77
rm out.requests.txt

‎acceptance/bundle/telemetry/deploy-config-file-count/out.telemetry.txt‎

Lines changed: 0 additions & 63 deletions
This file was deleted.

‎acceptance/bundle/telemetry/deploy-config-file-count/output.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,6 @@ Deploying resources...
55
Deployment complete!
66

77
>>> cat out.requests.txt
8+
{
9+
"configuration_file_count": 4
10+
}

‎acceptance/bundle/telemetry/deploy-config-file-count/script‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
11
trace $CLI bundle deploy
22

3-
trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson' > out.telemetry.txt
4-
5-
cmd_exec_id=$(extract_command_exec_id.py)
6-
7-
update_file.py out.telemetry.txt $cmd_exec_id '[CMD-EXEC-ID]'
3+
trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {configuration_file_count}'
84

95
rm out.requests.txt

‎acceptance/bundle/telemetry/deploy-error/out.telemetry.txt‎

Lines changed: 7 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,10 @@
11
{
2-
"frontend_log_event_id": "[UUID]",
3-
"entry": {
4-
"databricks_cli_log": {
5-
"execution_context": {
6-
"cmd_exec_id": "[CMD-EXEC-ID]",
7-
"version": "[DEV_VERSION]",
8-
"command": "bundle_deploy",
9-
"operating_system": "[OS]",
10-
"execution_time_ms": SMALL_INT,
11-
"exit_code": 1
12-
},
13-
"bundle_deploy_event": {
14-
"resource_count": 0,
15-
"resource_job_count": 0,
16-
"resource_pipeline_count": 0,
17-
"resource_model_count": 0,
18-
"resource_experiment_count": 0,
19-
"resource_model_serving_endpoint_count": 0,
20-
"resource_registered_model_count": 0,
21-
"resource_quality_monitor_count": 0,
22-
"resource_schema_count": 0,
23-
"resource_volume_count": 0,
24-
"resource_cluster_count": 0,
25-
"resource_dashboard_count": 0,
26-
"resource_app_count": 0
27-
}
28-
}
2+
"execution_context": {
3+
"cmd_exec_id": "[CMD-EXEC-ID]",
4+
"version": "[DEV_VERSION]",
5+
"command": "bundle_deploy",
6+
"operating_system": "[OS]",
7+
"execution_time_ms": SMALL_INT,
8+
"exit_code": 1
299
}
3010
}

‎acceptance/bundle/telemetry/deploy-error/script‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
errcode trace $CLI bundle deploy
22

3-
trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson' > out.telemetry.txt
3+
trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log | {execution_context}' > out.telemetry.txt
44

55
cmd_exec_id=$(extract_command_exec_id.py)
66

‎acceptance/bundle/telemetry/deploy-mode/output.txt‎

Lines changed: 2 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -17,98 +17,8 @@ A common practice is to use a username or principal name in this path, i.e. use
1717

1818
>>> cat out.requests.txt
1919
{
20-
"bundle_uuid": "[UUID]",
21-
"deployment_id": "[UUID]",
22-
"resource_count": 0,
23-
"resource_job_count": 0,
24-
"resource_pipeline_count": 0,
25-
"resource_model_count": 0,
26-
"resource_experiment_count": 0,
27-
"resource_model_serving_endpoint_count": 0,
28-
"resource_registered_model_count": 0,
29-
"resource_quality_monitor_count": 0,
30-
"resource_schema_count": 0,
31-
"resource_volume_count": 0,
32-
"resource_cluster_count": 0,
33-
"resource_dashboard_count": 0,
34-
"resource_app_count": 0,
35-
"experimental": {
36-
"configuration_file_count": 1,
37-
"variable_count": 0,
38-
"complex_variable_count": 0,
39-
"lookup_variable_count": 0,
40-
"target_count": 2,
41-
"bool_values": [
42-
{
43-
"key": "skip_artifact_cleanup",
44-
"value": false
45-
},
46-
{
47-
"key": "python_wheel_wrapper_is_set",
48-
"value": false
49-
},
50-
{
51-
"key": "has_serverless_compute",
52-
"value": false
53-
},
54-
{
55-
"key": "has_classic_job_compute",
56-
"value": false
57-
},
58-
{
59-
"key": "has_classic_interactive_compute",
60-
"value": false
61-
}
62-
],
63-
"bundle_mode": "DEVELOPMENT",
64-
"workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM"
65-
}
20+
"bundle_mode": "DEVELOPMENT"
6621
}
6722
{
68-
"bundle_uuid": "[UUID]",
69-
"deployment_id": "[UUID]",
70-
"resource_count": 0,
71-
"resource_job_count": 0,
72-
"resource_pipeline_count": 0,
73-
"resource_model_count": 0,
74-
"resource_experiment_count": 0,
75-
"resource_model_serving_endpoint_count": 0,
76-
"resource_registered_model_count": 0,
77-
"resource_quality_monitor_count": 0,
78-
"resource_schema_count": 0,
79-
"resource_volume_count": 0,
80-
"resource_cluster_count": 0,
81-
"resource_dashboard_count": 0,
82-
"resource_app_count": 0,
83-
"experimental": {
84-
"configuration_file_count": 1,
85-
"variable_count": 0,
86-
"complex_variable_count": 0,
87-
"lookup_variable_count": 0,
88-
"target_count": 2,
89-
"bool_values": [
90-
{
91-
"key": "skip_artifact_cleanup",
92-
"value": false
93-
},
94-
{
95-
"key": "python_wheel_wrapper_is_set",
96-
"value": false
97-
},
98-
{
99-
"key": "has_serverless_compute",
100-
"value": false
101-
},
102-
{
103-
"key": "has_classic_job_compute",
104-
"value": false
105-
},
106-
{
107-
"key": "has_classic_interactive_compute",
108-
"value": false
109-
}
110-
],
111-
"bundle_mode": "PRODUCTION",
112-
"workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM"
113-
}
23+
"bundle_mode": "PRODUCTION"
11424
}

‎acceptance/bundle/telemetry/deploy-mode/script‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,6 @@ trace $CLI bundle deploy -t dev
22

33
trace $CLI bundle deploy -t prod
44

5-
trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event'
5+
trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bundle_mode}'
66

77
rm out.requests.txt

0 commit comments

Comments
 (0)