[YOLO] create a new pipeline to run functional tests per framework (#1451)

Author: datarobot-harness[bot]

.harness/pipelines/test_functional_by_framework_multisteps/input_sets/default_on_pr_input.yaml

@@ -0,0 +1,15 @@
+inputSet:
+  name: default_on_pr_input
+  identifier: default_on_pr_input
+  orgIdentifier: Custom_Models
+  projectIdentifier: datarobotusermodels
+  pipeline:
+    identifier: Test_functional_by_framework_multisteps
+    properties:
+      ci:
+        codebase:
+          repoName: datarobot-user-models
+          build:
+            type: PR
+            spec:
+              number: <+trigger.prNumber>

.harness/test_functional_by_framework_multisteps.yaml

@@ -0,0 +1,92 @@
+pipeline:
+  projectIdentifier: datarobotusermodels
+  orgIdentifier: Custom_Models
+  tags: {}
+  properties:
+    ci:
+      codebase:
+        connectorRef: account.svc_harness_git1
+        repoName: <+input>
+        build: <+input>
+        sparseCheckout: []
+  stages:
+    - stage:
+        name: Build list of envs to test
+        identifier: build_list_of_envs_to_test
+        description: ""
+        type: CI
+        spec:
+          cloneCodebase: true
+          execution:
+            steps:
+              - step:
+                  type: Run
+                  name: build_list_of_environments_to_test_on
+                  identifier: build_list_of_environments_to_test_on
+                  spec:
+                    connectorRef: account.dockerhub_datarobot_read
+                    shell: Sh
+                    command: |-
+                      matrix_values=$(python3 ./harness_scripts/functional_by_framework_multistep/generate_testlist.py ./tests/functional/test_per_framework.json ./)
+                      echo ${matrix_values}
+                    outputVariables:
+                      - name: matrix_values
+          caching:
+            enabled: true
+            paths: []
+          platform:
+            os: Linux
+            arch: Amd64
+          runtime:
+            type: Cloud
+            spec: {}
+          buildIntelligence:
+            enabled: true
+    - stage:
+        name: Test functional by framework
+        identifier: check_by_framework
+        description: ""
+        type: CI
+        spec:
+          cloneCodebase: true
+          execution:
+            steps:
+              - step:
+                  type: Run
+                  name: Run functional test in provided framework
+                  identifier: Run_1
+                  spec:
+                    connectorRef: datarobot_user_models_read_write
+                    image: datarobotdev/<+matrix.env.repo>:<+matrix.env.tag>
+                    shell: Sh
+                    command: |-
+                      ./harness_scripts/functional_by_framework/check_by_framework_run1_step_entrypoint.sh \
+                        <+matrix.env.framework> <+matrix.env.env_folder>
+                    resources:
+                      limits:
+                        memory: 3G
+                  when:
+                    stageStatus: Success
+          caching:
+            enabled: true
+            paths: []
+          platform:
+            os: Linux
+            arch: Amd64
+          runtime:
+            type: Cloud
+            spec: {}
+          buildIntelligence:
+            enabled: true
+        when:
+          pipelineStatus: Success
+        strategy:
+          matrix:
+            env: <+json.list("environments", <+pipeline.stages.build_list_of_envs_to_test.spec.execution.steps.build_list_of_environments_to_test_on.output.outputVariables.matrix_values>)>
+            nodeName: <+matrix.env.env_folder>/<+matrix.env.framework> - <+matrix.env.tag>
+  identifier: Test_functional_by_framework_multisteps
+  description: |-
+    Runs functional tests for each framework in a corresponding image.
+    If only DRUM changed in the PR, latest image is taken for every framework, DRUM installed and tests run.
+    If environment itself has changed, temporary image is built.
+  name: Test functional by framework - multisteps

harness_scripts/functional_by_framework_multistep/generate_testlist.py

@@ -0,0 +1,96 @@
+import json
+import os
+import sys
+import logging
+
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="[%(levelname)s] %(asctime)s - %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+
+logger = logging.getLogger(__name__)
+
+
+def load_tests_list(tests_list_path):
+    try:
+        logger.info(f"Loading tests list from {tests_list_path}")
+        with open(tests_list_path, "r") as f:
+            data = json.load(f)
+        return data.get("environments", [])
+    except FileNotFoundError:
+        logger.error(f"Tests list file not found: {tests_list_path}")
+        sys.exit(1)
+    except json.JSONDecodeError:
+        logger.error("Invalid JSON in tests list.")
+        sys.exit(1)
+
+
+def load_env_info(env_info_path):
+    try:
+        logger.info(f"Reading env_info from {env_info_path}")
+        with open(env_info_path, "r") as f:
+            data = json.load(f)
+        return {"repo": data["imageRepository"], "tag": data["environmentVersionId"]}
+    except FileNotFoundError:
+        logger.error(f"env_info.json not found at path: {env_info_path}")
+        return None
+    except KeyError as e:
+        logger.error(f"Missing key in env_info.json: {e}")
+        return None
+    except json.JSONDecodeError:
+        logger.error("Invalid JSON in env_info.json.")
+        return None
+
+
+def build_environments(env_list, root_path):
+    output_envs = []
+
+    for env in env_list:
+        env_folder = env["env_folder"]
+        framework = env["framework"]
+        with_local = env.get("with_local", False)
+
+        env_info_path = os.path.join(root_path, env_folder, framework, "env_info.json")
+        env_info = load_env_info(env_info_path)
+
+        if not env_info:
+            logger.warning(f"Skipping environment due to invalid env_info: {env}")
+            continue
+
+        base_record = {
+            "env_folder": env_folder,
+            "framework": framework,
+            "repo": env_info["repo"],
+            "tag": env_info["tag"],
+        }
+
+        output_envs.append(base_record)
+
+        if with_local:
+            local_record = base_record.copy()
+            local_record["tag"] = f"{env_info['tag']}.local"
+            output_envs.append(local_record)
+            logger.info(f"Added local version for environment: {env_folder}/{framework}")
+
+    return output_envs
+
+
+def main():
+    if len(sys.argv) != 3:
+        logger.error("Usage: python build_env_output.py <tests-list.json> <root-path>")
+        sys.exit(1)
+
+    tests_list_path = sys.argv[1]
+    root_path = sys.argv[2]
+
+    env_list = load_tests_list(tests_list_path)
+    final_envs = build_environments(env_list, root_path)
+
+    output = {"environments": final_envs}
+    print(json.dumps(output, separators=(",", ":")))  # Minified JSON
+
+
+if __name__ == "__main__":
+    main()

tests/functional/test_per_framework.json

@@ -0,0 +1,15 @@
+{
+  "environments" : [
+    {"env_folder":  "public_dropin_environments", "framework":  "java_codegen", "with_local":  true},
+    {"env_folder":  "public_dropin_environments", "framework":  "python3_keras", "with_local":  true},
+    {"env_folder":  "public_dropin_environments", "framework":  "python3_onnx", "with_local":  true},
+    {"env_folder":  "public_dropin_environments", "framework":  "python3_pmml", "with_local":  true},
+    {"env_folder":  "public_dropin_environments", "framework":  "python3_pytorch", "with_local":  true},
+    {"env_folder":  "public_dropin_environments", "framework":  "python3_sklearn", "with_local":  true},
+    {"env_folder":  "public_dropin_environments", "framework":  "python3_xgboost", "with_local":  true},
+    {"env_folder":  "public_dropin_environments", "framework":  "python311", "with_local":  true},
+    {"env_folder":  "public_dropin_environments", "framework":  "python311_genai", "with_local":  true},
+    {"env_folder":  "public_dropin_environments", "framework":  "r_lang", "with_local":  true},
+    {"env_folder":  "public_dropin_gpu_environments", "framework":  "vllm"}
+  ]
+}