Skip to content

Commit 9512c08

Browse files
yakov-gyakov-g
authored
[RAPTOR-12416] add new reconciliation pipeline (#1415)
1 parent 9f6c8f3 commit 9512c08

File tree

2 files changed

+175
-7
lines changed

2 files changed

+175
-7
lines changed

‎.harness/reconcile_dependencies_for_all_envs_based_on_parent_folder.yaml‎

Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
pipeline:
2+
projectIdentifier: datarobotusermodels
3+
orgIdentifier: Custom_Models
4+
tags: {}
5+
stages:
6+
- stage:
7+
name: Reconcile Python Dependencies
8+
identifier: Reconcile_Python_Dependencies
9+
description: ""
10+
type: CI
11+
spec:
12+
cloneCodebase: false
13+
caching:
14+
enabled: true
15+
buildIntelligence:
16+
enabled: true
17+
execution:
18+
steps:
19+
- step:
20+
type: Run
21+
name: reconcile_dependencies
22+
identifier: reconcile_dependencies
23+
spec:
24+
connectorRef: account.dockerhub_datarobot_read
25+
image: datarobotdev/mirror_chainguard_datarobot.com_python-fips:3.11-dev
26+
shell: Bash
27+
command: |-
28+
# Ensure Git is installed
29+
if ! command -v git &> /dev/null; then
30+
echo "Git is not installed. Installing..."
31+
apt-get update && apt-get install -y git
32+
fi
33+
34+
# Configure Git
35+
git config --global user.name "svc-harness-git2"
36+
git config --global user.email "svc-harness-git2@datarobot.com"
37+
git config --global url."https://${GITHUB_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
38+
git config --global --add safe.directory /harness # Mark /harness as a safe directory
39+
40+
pip install bson
41+
42+
# Clone the datarobot-user-models repository
43+
git init
44+
git clone https://github.com/datarobot/datarobot-user-models.git
45+
cd datarobot-user-models || exit 1
46+
47+
ROOT_DIR=$(pwd)
48+
49+
# Get the current branch name
50+
current_branch=$(git branch --show-current)
51+
echo "Current branch: $current_branch"
52+
53+
# Define the target branch (replace <+pipeline.branch> with the actual target branch name)
54+
target_branch="<+trigger.branch>"
55+
echo "Target branch: $target_branch"
56+
57+
# Compare the current branch with the target branch
58+
echo "Listing all branches..."
59+
git branch -a
60+
if [ "$current_branch" != "$target_branch" ]; then
61+
echo "Switching to branch $target_branch..."
62+
git checkout "$target_branch"
63+
else
64+
# If already on the target branch, just print the current branch again
65+
echo "Already on the target branch. No need to switch."
66+
echo "Current branch: $current_branch"
67+
fi
68+
69+
# r_lang specific setup
70+
export RPY2_CFFI_MODE=ABI
71+
72+
# Navigate to environment directory
73+
ENV_DIR=<+pipeline.variables.env_folder>
74+
TARGET_DIR=${ENV_DIR}
75+
76+
# Capture the filtered directories into a variable
77+
CHANGED_DIRS=$(git diff --name-only "$current_branch" "$target_branch" \
78+
| grep '/' \
79+
| xargs -n1 dirname \
80+
| grep "^$TARGET_DIR" \
81+
| while read -r DIR; do
82+
REST=${DIR#"$TARGET_DIR/"}
83+
if [ -z "$REST" ]; then
84+
echo "$DIR"
85+
else
86+
FIRST_LEVEL=$(echo "$REST" | cut -d'/' -f1)
87+
echo "$TARGET_DIR/$FIRST_LEVEL"
88+
fi
89+
done \
90+
| sort -u)
91+
92+
# Output the list stored in the variable
93+
echo "=== Changed environments ==="
94+
echo "$CHANGED_DIRS"
95+
echo "=== === === === === === ==="
96+
97+
98+
# Iterate over each directory in the list
99+
for DIR in $CHANGED_DIRS; do
100+
# Run the script for each directory
101+
cd ${DIR} || exit 1
102+
echo "Running: bash ${ROOT_DIR}/tools/reconcile_dependencies.sh $DIR"
103+
bash ${ROOT_DIR}/tools/reconcile_dependencies.sh "$DIR"
104+
105+
# This could be a separate step, but we already iterate over folders here.
106+
echo "Checking whether Environment Version ID should be updated"
107+
git show master:./env_info.json | grep -n 'environmentVersionId' > /tmp/old_env_version_id.txt
108+
git show HEAD:./env_info.json | grep -n 'environmentVersionId' > /tmp/new_env_version_id.txt
109+
# If env version was not updated, update
110+
if diff /tmp/old_env_version_id.txt /tmp/new_env_version_id.txt > /dev/null; then
111+
echo "Updating Environment Version ID"
112+
python3 ${ROOT_DIR}/tools/env_version_update.py --file ./env_info.json
113+
fi
114+
115+
116+
cd -
117+
done
118+
119+
# Commit and push changes if any
120+
if [[ -n $(git status --porcelain) ]]; then
121+
git status --porcelain
122+
git commit -a -m "Reconcile dependencies for $ENV_DIR"
123+
git config pull.rebase true
124+
git pull origin "$(git branch --show-current)" --rebase
125+
git push --set-upstream origin "$(git branch --show-current)"
126+
127+
echo "Reconciled dependencied commited to branch"
128+
exit 0
129+
else
130+
echo "No changes detected in Git."
131+
fi
132+
envVariables:
133+
GITHUB_ACCESS_TOKEN: <+secrets.getValue("account.githubpatsvcharnessgit2")>
134+
resources:
135+
limits:
136+
memory: 5Gi
137+
description: Run the tools/reconcile_dependencies.sh script to auto-generate requirements.txt file base on its requirements.in file.
138+
platform:
139+
os: Linux
140+
arch: Amd64
141+
runtime:
142+
type: Cloud
143+
spec: {}
144+
variables:
145+
- name: env_folder
146+
type: String
147+
description: The folder containing the environment to reconcile, e.g. "public_dropin_environments"
148+
required: true
149+
value: <+input>.allowedValues(public_dropin_environments)
150+
identifier: reconcile_dependencies_for_all_envs_based_on_parent_folder
151+
description: |-
152+
On PR, for every drop in env in parent envs folder, e.g. public_dropin_environments:
153+
* Enforce getting requirements.in and generate requirements.txt.
154+
* Update environmentVersionId in env_info.json
155+
name: reconcile dependencies for all envs based on parent

‎tools/env_version_update.py‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,15 @@
2424
ENV_INFO_JSON = "env_info.json"
2525

2626

27+
def update_env_info_file(env_info_file):
28+
with open(env_info_file) as json_file:
29+
metadata = json.load(json_file)
30+
metadata["environmentVersionId"] = str(ObjectId())
31+
with open(env_info_file, "w") as json_file:
32+
json.dump(metadata, json_file, indent=2)
33+
json_file.write("\n")
34+
35+
2736
def main(dir_to_scan, env=None):
2837
"""
2938
Iterate over directories in dir_to_scan, load json from env._info.json,
@@ -40,12 +49,7 @@ def main(dir_to_scan, env=None):
4049
item_abs_path = os.path.abspath(os.path.join(dir_to_scan, item))
4150
if os.path.isdir(item_abs_path):
4251
env_info_json = os.path.join(item_abs_path, ENV_INFO_JSON)
43-
with open(env_info_json) as json_file:
44-
metadata = json.load(json_file)
45-
metadata["environmentVersionId"] = str(ObjectId())
46-
with open(env_info_json, "w") as json_file:
47-
json.dump(metadata, json_file, indent=2)
48-
json_file.write("\n")
52+
update_env_info_file(env_info_json)
4953

5054

5155
if __name__ == "__main__":
@@ -62,6 +66,15 @@ def main(dir_to_scan, env=None):
6266
default=None,
6367
help="Name of the environment to update",
6468
)
69+
parser.add_argument(
70+
"-f",
71+
"--file",
72+
default=None,
73+
help="Path of the env_info.json to update. If provided, other arguments are ignored.",
74+
)
6575

6676
args = parser.parse_args()
67-
main(args.dir, args.env)
77+
if args.file:
78+
update_env_info_file(args.file)
79+
else:
80+
main(args.dir, args.env)

0 commit comments

Comments
 (0)