[RAPTOR-12473] Update pipeline test functional by framework (#1426)

Author: datarobot-harness[bot]

.harness/env_image_publish.yaml

@@ -28,7 +28,7 @@ pipeline:
                     stageStatus: Success
                   spec:
                     connectorRef: datarobot_user_models_read_write
-                    repo: datarobotdev/datarobot-user-models
+                    repo: <+pipeline.variables.image_namespace_repo>
                     tags:
                       - <+pipeline.variables.image_tag>
                     dockerfile: <+pipeline.variables.dockerfile_path>
@@ -54,6 +54,11 @@ pipeline:
       type: String
       required: true
       value: <+input>
+    - name: image_namespace_repo
+      type: String
+      description: ""
+      required: true
+      value: <+input>.default(datarobotdev/datarobot-user-models)
     - name: image_tag
       type: String
       description: ""

.harness/reconcile_dependencies_for_all_envs_based_on_parent_folder.yaml

@@ -16,13 +16,25 @@ pipeline:
             enabled: true
           execution:
             steps:
+              - step:
+                  type: GitClone
+                  name: Get DR Release Version
+                  identifier: Get_DR_Release_Version
+                  spec:
+                    connectorRef: account.svc_harness_git1
+                    repoName: DataRobot
+                    sparseCheckout:
+                      - config/RELEASE_VERSION
+                    depth: 1
+                    build:
+                      type: branch
+                      spec:
+                        branch: master
               - step:
                   type: Run
                   name: reconcile_dependencies
                   identifier: reconcile_dependencies
                   spec:
-                    connectorRef: account.dockerhub_datarobot_read
-                    image: datarobotdev/mirror_chainguard_datarobot.com_python-fips:3.11-dev
                     shell: Bash
                     command: |-
                       # Ensure Git is installed
@@ -31,13 +43,20 @@ pipeline:
                           apt-get update && apt-get install -y git
                       fi
 
+                      echo "Getting current DataRobot release version"
+                      RELEASE_VERSION=$(cat DataRobot/config/RELEASE_VERSION)
+                      # if [ "$TARGET_BRANCH" != "master" ]; then
+                      #  RELEASE_VERSION=$(echo $TARGET_BRANCH | sed 's|release/||')
+                      # fi
+                      echo "Current DataRobot release version: ${RELEASE_VERSION}"
+
                       # Configure Git
                       git config --global user.name "svc-harness-git2"
                       git config --global user.email "svc-harness-git2@datarobot.com"
                       git config --global url."https://${GITHUB_ACCESS_TOKEN}@github.com/".insteadOf "https://github.com/"
                       git config --global --add safe.directory /harness  # Mark /harness as a safe directory
 
-                      pip install bson
+                      pip install bson pyyaml
 
                       # Clone the datarobot-user-models repository
                       git init
@@ -108,23 +127,44 @@ pipeline:
                         git show HEAD:./env_info.json | grep -n 'environmentVersionId' > /tmp/new_env_version_id.txt
                         # If env version was not updated, update
                         if diff /tmp/old_env_version_id.txt /tmp/new_env_version_id.txt > /dev/null; then
-                          echo "Updating Environment Version ID"
+                          echo "Updating Environment Version IDi --"
                           python3 ${ROOT_DIR}/tools/env_version_update.py --file ./env_info.json
+                          echo "DONE Updating Environment Version IDi --"
                         fi
 
+                        # convert env_info.json to values.yaml
+                        # echo "Updating values.yaml with info from env_info.json"
+                        # python3 ${ROOT_DIR}/tools/build_values_yaml.py from-json --json ./env_info.json --yaml ./values.yaml
+
+                        # echo "Getting env version id from yaml"
+                        # ENV_VERSION_ID=$(yq '.environmentVersionId' ./values.yaml)
+                        # IMAGE_REPOSITORY=$(yq '.imageRepository' ./values.yaml)
+
+                        # IMAGE_TAG="v$RELEASE_VERSION-$ENV_VERSION_ID"
+                        # TAGS="$IMAGE_TAG,v$RELEASE_VERSION-latest"
+                        # if [ "$TARGET_BRANCH" = "master" ]; then
+                        #   TAGS="$TAGS,latest"
+                        # fi
+
+                        # python3 ${ROOT_DIR}/tools/build_values_yaml.py set-values \
+                        #                                   --yaml ./values.yaml  \
+                        #                                   tags=${TAGS} \
+                        #                                   image=datarobotdev/${IMAGE_REPOSITORY}:${IMAGE_TAG}
+
+                        # git add ./values.yaml
 
                         cd -
                       done
 
                       # Commit and push changes if any
                       if [[ -n $(git status --porcelain) ]]; then
                         git status --porcelain
-                        git commit -a -m "Reconcile dependencies for $ENV_DIR"
+                        git commit -a -m "Reconcile dependencies, updated IDs, tags, values.yaml for $DIR"
                         git config pull.rebase true
                         git pull origin "$(git branch --show-current)" --rebase
                         git push --set-upstream origin "$(git branch --show-current)"
 
-                        echo "Reconciled dependencied commited to branch"
+                        echo "Reconciled dependencies commited to branch"
                         exit 0
                       else
                         echo "No changes detected in Git."

.harness/test_functional_by_framework.yaml

@@ -52,19 +52,22 @@ pipeline:
                       . ./harness_scripts/functional_by_framework/check_env_changed_check_diff.sh
 
                       dockerfile_local="./<+pipeline.variables.env_folder>/<+pipeline.variables.framework>/Dockerfile.local"
-                      ls -la ${dockefile_local}
+
+                      image_uri=${test_image_namespace}/${test_image_repository}:${test_image_tag}
                       if [ -f "${dockerfile_local}" ]; then
                         dockerfile_local_exists=true
-                        matrix_values="${test_image_tag},${test_image_tag}.local"
+                        matrix_values="${image_uri},datarobotdev/${test_image_repository}:${test_image_tag}.local"
                       else
                         dockerfile_local_exists=false
-                        matrix_values="${test_image_tag}"
+                        matrix_values="${image_uri}"
                       fi
                       export dockerfile_local_exists
                     outputVariables:
                       - name: dockerfile_local_exists
                       - name: matrix_values
                       - name: changed_deps
+                      - name: test_image_namespace
+                      - name: test_image_repository
                       - name: test_image_tag
           caching:
             enabled: false
@@ -98,6 +101,9 @@ pipeline:
                   - name: dockerfile_path
                     type: String
                     value: <+pipeline.variables.env_folder>/<+pipeline.variables.framework>/Dockerfile
+                  - name: image_namespace_repo
+                    type: String
+                    value: <+pipeline.stages.check_env_changed.spec.execution.steps.check_diff.output.outputVariables.test_image_namespace>/<+pipeline.stages.check_env_changed.spec.execution.steps.check_diff.output.outputVariables.test_image_repository>
                   - name: image_tag
                     type: String
                     value: <+pipeline.stages.check_env_changed.spec.execution.steps.check_diff.output.outputVariables.test_image_tag>
@@ -127,6 +133,9 @@ pipeline:
                   - name: dockerfile_path
                     type: String
                     value: <+pipeline.variables.env_folder>/<+pipeline.variables.framework>/Dockerfile.local
+                  - name: image_namespace_repo
+                    type: String
+                    value: <+pipeline.stages.check_env_changed.spec.execution.steps.check_diff.output.outputVariables.test_image_namespace>/<+pipeline.stages.check_env_changed.spec.execution.steps.check_diff.output.outputVariables.test_image_repository>
                   - name: image_tag
                     type: String
                     value: <+pipeline.stages.check_env_changed.spec.execution.steps.check_diff.output.outputVariables.test_image_tag>.local
@@ -149,10 +158,10 @@ pipeline:
                   identifier: Run_1
                   strategy:
                     matrix:
-                      image_tag: <+<+pipeline.stages.check_env_changed.spec.execution.steps.check_diff.output.outputVariables.matrix_values>.split(',')>
+                      image_uri: <+<+pipeline.stages.check_env_changed.spec.execution.steps.check_diff.output.outputVariables.matrix_values>.split(',')>
                   spec:
                     connectorRef: datarobot_user_models_read_write
-                    image: datarobotdev/datarobot-user-models:<+matrix.image_tag>
+                    image: <+matrix.image_uri>
                     shell: Sh
                     command: |-
                       echo "Checking if test for local should run"

harness_scripts/functional_by_framework/check_env_changed_check_diff.sh

@@ -16,32 +16,67 @@ git fetch origin master ${CODEBASE_BRANCH}
 git branch -a
 
 changed_paths=$(git diff --name-only origin/master...HEAD)
-echo "changed_paths: $changed_paths"
+echo "--- changed paths ---"
+echo "${changed_paths}"
+echo "--- --- --- --- ---"
+
+# by default define namespace, repo, and tag for the existing flow without changes
+# e.g. datarobotdev/datarobot-user-models:public_dropin_envs_python3_sklearn_latest
+env_info="${ENV_FOLDER}/${FRAMEWORK}/env_info.json"
+ENV_VERSION_ID=$(jq -r '.environmentVersionId' ${env_info})
+
+test_image_namespace=datarobotdev
+test_image_repository=datarobot-user-models
+
+test_image_tag_base=${ENV_FOLDER}_${FRAMEWORK}_${ENV_VERSION_ID}
+test_image_tag=${test_image_tag_base}
+
+changed_deps=false;
+
+
+IMAGE_REPOSITORY=$(jq -r '.imageRepository' ${env_info})
+if [ "${IMAGE_REPOSITORY}" = "null" ]; then
+  echo "Image repository is not defined in env_info.json"
+else
+  # if env_info has imageRepository
+  # point test_image_namespace to datarobot
+  # point test_image_repository to defined repo
+  # point tag to ENV_VERSION_ID
+  # e.g. datarobot/env-python-sklearn:12355123abc918234
+  echo "read ${IMAGE_REPOSITORY}"
+  # after promotion work, change to datarobot
+  test_image_namespace=datarobotdev
+  test_image_repository=${IMAGE_REPOSITORY}
+  test_image_tag_base=${ENV_VERSION_ID}
+  test_image_tag=${ENV_VERSION_ID}
+fi
+
 
-test_image_tag_base=${ENV_FOLDER}_${FRAMEWORK}
 if echo "${changed_paths}" | grep "${ENV_FOLDER}/${FRAMEWORK}" > /dev/null; then
     changed_deps=true;
+    # if env changed, means we want to push tmp image into datarobotdev
+    test_image_namespace=datarobotdev
     if [ -n $TRIGGER_PR_NUMBER ] && [ "$TRIGGER_PR_NUMBER" != "null" ]; then
-        test_image_tag=${test_image_tag_base}_${TRIGGER_PR_NUMBER};
+        # datarobotdev/env-python-sklearn:12355123abc918234_PR_NUM
+        # or
+        # datarobotdev/datarobot-user-models:public_dropin_envs_python3_sklearn_PR_NUM
+        test_image_tag=${test_image_tag_base}
     else
         test_image_tag=${test_image_tag_base}_${CODEBASE_BRANCH}
         # If the test_image_tag may contain a slash, replace it with an underscore (POSIX compliant)
         while case $test_image_tag in */*) true;; *) false;; esac; do
           test_image_tag=${test_image_tag%%/*}_${test_image_tag#*/}
         done
     fi
-else
-    changed_deps=false;
-    test_image_tag=${test_image_tag_base}_latest;
 fi
 
-# If the environment variable 'USE_LOCAL_DOCKERFILE' is set to "true", than add '.local'
-#if [ -n $USE_LOCAL_DOCKERFILE ] && [ "$USE_LOCAL_DOCKERFILE" = "true" ]; then
-#    test_image_tag=${test_image_tag}.local
-#fi
-
 # Required by the Harness step
 export changed_deps
+export test_image_namespace
+export test_image_repository
 export test_image_tag
 
-echo "changed_deps: $changed_deps, test_image_tag: $test_image_tag"
+echo "changed_deps: $changed_deps"
+echo "test_image_namespace: $test_image_namespace"
+echo "test_image_repository: $test_image_repository"
+echo "test_image_tag: $test_image_tag"