Skip to content

Commit 50d26b8

Browse files
datarobot-harness[bot]datarobot-harness[bot]
authored
[YOLO] new test pipeline for tests per framework (#1452)
1 parent 96f0407 commit 50d26b8

File tree

3 files changed

+236
-8
lines changed

3 files changed

+236
-8
lines changed

‎.harness/pipelines/test_functional_by_framework_multisteps/input_sets/default_on_pr_input.yaml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ inputSet:
88
properties:
99
ci:
1010
codebase:
11-
repoName: datarobot-user-models
1211
build:
1312
type: PR
1413
spec:

‎.harness/reusable_build_images.yaml‎

Lines changed: 161 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
pipeline:
2+
name: WIP REUSABLE PIPELINE to build images
3+
identifier: WIP_build_images
4+
projectIdentifier: datarobotusermodels
5+
orgIdentifier: Custom_Models
6+
tags: {}
7+
properties:
8+
ci:
9+
codebase:
10+
connectorRef: account.svc_harness_git1
11+
repoName: <+pipeline.variables.target_repo>
12+
build:
13+
type: branch
14+
spec:
15+
branch: <+pipeline.variables.target_branch>
16+
stages:
17+
- stage:
18+
name: Detect changes and output images build matrix
19+
identifier: Detect_changes_and_output_images_build_matrix
20+
description: ""
21+
type: CI
22+
spec:
23+
cloneCodebase: true
24+
caching:
25+
enabled: true
26+
override: false
27+
buildIntelligence:
28+
enabled: true
29+
platform:
30+
os: Linux
31+
arch: Amd64
32+
runtime:
33+
type: Cloud
34+
spec: {}
35+
execution:
36+
steps:
37+
- step:
38+
type: Run
39+
name: Build params matrix
40+
identifier: Build_params_matrix
41+
spec:
42+
shell: Bash
43+
command: |-
44+
echo "Hello"
45+
pwd
46+
ls -la
47+
IFS=',' read -ra DIR_ARRAY <<< "<+pipeline.variables.envs_folders>"
48+
49+
json_array="[]"
50+
51+
for DIR in "${DIR_ARRAY[@]}"; do
52+
cd "${DIR}" || exit 1
53+
echo "Processing dir: ${DIR}"
54+
55+
# Read fields from env_info.json
56+
REPO_NAME=$(jq -r '.imageRepository' ./env_info.json)
57+
ENV_VERSION_ID=$(jq -r '.environmentVersionId' ./env_info.json)
58+
59+
# Create JSON object for this env
60+
env_json=$(jq -n \
61+
--arg path "${DIR}" \
62+
--arg dockerfile "Dockerfile" \
63+
--arg repo "${REPO_NAME}" \
64+
--arg tag "${ENV_VERSION_ID}" \
65+
'{path: $path, repository: $repo, tag: $tag, dockerfile: $dockerfile}')
66+
67+
# Append to the JSON array
68+
json_array=$(echo "${json_array}" | jq --argjson obj "${env_json}" '. + [$obj]')
69+
70+
# repeat if Dockerfile.local exists and add local suffix to dockerfile and tag
71+
if [ -f "./Dockerfile.local" ]; then
72+
env_json=$(jq -n \
73+
--arg path "${DIR}" \
74+
--arg dockerfile "Dockerfile.local" \
75+
--arg repo "${REPO_NAME}" \
76+
--arg tag "${ENV_VERSION_ID}.local" \
77+
'{path: $path, repository: $repo, tag: $tag, dockerfile: $dockerfile}')
78+
79+
# Append to the JSON array
80+
json_array=$(echo "${json_array}" | jq --argjson obj "${env_json}" '. + [$obj]')
81+
fi
82+
83+
84+
cd - || exit 1
85+
done
86+
87+
matrix_json=$(jq -n --argjson arr "${json_array}" '{images: $arr}' | jq -c .)
88+
89+
# Print final JSON array
90+
echo "${matrix_json}"
91+
export matrix_json
92+
outputVariables:
93+
- name: matrix_json
94+
type: String
95+
value: matrix_json
96+
when:
97+
pipelineStatus: Success
98+
condition: <+pipeline.variables.envs_folders>!=""
99+
- stage:
100+
name: build images
101+
identifier: build_images
102+
description: ""
103+
type: CI
104+
spec:
105+
cloneCodebase: true
106+
caching:
107+
enabled: true
108+
override: false
109+
buildIntelligence:
110+
enabled: true
111+
platform:
112+
os: Linux
113+
arch: Amd64
114+
runtime:
115+
type: Cloud
116+
spec: {}
117+
execution:
118+
steps:
119+
- step:
120+
type: BuildAndPushDockerRegistry
121+
name: Build and push to dockerhub
122+
identifier: Build_and_push_to_dockerhub
123+
spec:
124+
connectorRef: datarobot_user_models_read_write
125+
repo: datarobotdev/<+matrix.image.repository>
126+
tags:
127+
- <+matrix.image.tag>
128+
caching: true
129+
dockerfile: <+matrix.image.path>/<+matrix.image.dockerfile>
130+
context: <+matrix.image.path>
131+
when:
132+
pipelineStatus: Success
133+
condition: <+pipeline.variables.envs_folders>!=""
134+
strategy:
135+
matrix:
136+
image: <+json.list("images", <+pipeline.stages.Detect_changes_and_output_images_build_matrix.spec.execution.steps.Build_params_matrix.output.outputVariables.matrix_json>)>
137+
nodeName: <+matrix.image.repository>:<+matrix.image.tag>
138+
description: |-
139+
This pipeline can be used in other repositories to:
140+
* detect which environments have changed. Environment is a folder with env_info.json
141+
* build images according to repo name and environment version ID from env_info.json
142+
Should receive:
143+
Repo
144+
Branch
145+
Comma separated list of paths to envs to build
146+
variables:
147+
- name: target_repo
148+
type: String
149+
description: "Target repo: e.g. datarobot-user-models"
150+
required: false
151+
value: <+input>
152+
- name: target_branch
153+
type: String
154+
description: E.g. master
155+
required: false
156+
value: <+input>
157+
- name: envs_folders
158+
type: String
159+
description: Paths to envs' folders to build images from (usually changed envs from prev step)
160+
required: false
161+
value: <+input>

‎.harness/test_functional_by_framework_multisteps.yaml‎

Lines changed: 75 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,50 @@ pipeline:
22
projectIdentifier: datarobotusermodels
33
orgIdentifier: Custom_Models
44
tags: {}
5-
properties:
6-
ci:
7-
codebase:
8-
connectorRef: account.svc_harness_git1
9-
repoName: <+input>
10-
build: <+input>
11-
sparseCheckout: []
125
stages:
6+
- stage:
7+
name: Reconcile envVersionIds requirements
8+
identifier: Reconcile_envVersionIds_requirements
9+
template:
10+
templateRef: org.Execution_Environments_Reconcile_Stage
11+
versionLabel: v1
12+
templateInputs:
13+
type: CI
14+
variables:
15+
- name: target_repo
16+
type: String
17+
value: <+pipeline.variables.target_repo>
18+
- name: target_repo_branch
19+
type: String
20+
value: <+pipeline.variables.target_branch>
21+
- name: envs_parent_folders
22+
type: String
23+
value: <+pipeline.variables.envs_parent_folders_to_include>
24+
- name: force_requirements_upgrade
25+
type: String
26+
value: <+pipeline.variables.force_requirements_upgrade>
27+
- stage:
28+
name: Build changed images for changed environments
29+
identifier: Build_changed_images
30+
description: ""
31+
type: Pipeline
32+
spec:
33+
org: Custom_Models
34+
pipeline: WIP_build_images
35+
project: datarobotusermodels
36+
inputs:
37+
identifier: WIP_build_images
38+
variables:
39+
- name: target_repo
40+
type: String
41+
value: <+pipeline.variables.target_repo>
42+
- name: target_branch
43+
type: String
44+
value: <+pipeline.variables.target_branch>
45+
- name: envs_folders
46+
type: String
47+
value: <+pipeline.stages.Reconcile_envVersionIds_requirements.spec.execution.steps.find_changed.output.outputVariables.CHANGED_DIRS>
48+
tags: {}
1349
- stage:
1450
name: Build list of envs to test
1551
identifier: build_list_of_envs_to_test
@@ -42,6 +78,8 @@ pipeline:
4278
spec: {}
4379
buildIntelligence:
4480
enabled: true
81+
when:
82+
pipelineStatus: Success
4583
- stage:
4684
name: Test functional by framework
4785
identifier: check_by_framework
@@ -90,3 +128,33 @@ pipeline:
90128
If only DRUM changed in the PR, latest image is taken for every framework, DRUM installed and tests run.
91129
If environment itself has changed, temporary image is built.
92130
name: Test functional by framework - multisteps
131+
variables:
132+
- name: target_repo
133+
type: String
134+
description: ""
135+
required: true
136+
value: <+input>
137+
- name: target_branch
138+
type: String
139+
description: ""
140+
required: false
141+
value: <+input>
142+
- name: envs_parent_folders_to_include
143+
type: String
144+
description: |-
145+
Comma separated list of envs parent folders to update; all if empty.
146+
E.g. if changes are in: parent1/env1, parent2/env2; and parent2 is provided, only parent2/env2 is included
147+
required: false
148+
value: <+input>
149+
- name: force_requirements_upgrade
150+
type: String
151+
description: Whether to force all requirements update to the latest versions
152+
required: false
153+
value: <+input>.default(false).allowedValues(true,false)
154+
properties:
155+
ci:
156+
codebase:
157+
connectorRef: account.svc_harness_git1
158+
repoName: datarobot-user-models
159+
build: <+input>
160+
sparseCheckout: []

0 commit comments

Comments
 (0)