Update qe tests

tests/interop/README.md

@@ -0,0 +1,23 @@
+# Running tests
+
+## Prerequisites
+
+* Openshift cluster with coco-pattern installed
+* kubeconfig file for Openshift cluster
+* oc client installed at ~/oc_client/oc
+
+## Steps
+
+* create python3 venv, clone multicloud-gitops repository
+* export KUBECONFIG=\<path to hub kubeconfig file>
+* export INFRA_PROVIDER=azure
+* (optional) export WORKSPACE=\<dir to save test results to> (defaults to /tmp)
+* cd coco-pattern/tests/interop
+* pip install -r requirements.txt
+* ./run_tests.sh
+
+## Results
+
+* results .xml files will be placed at $WORKSPACE
+* test logs will be placed at $WORKSPACE/.results/test_execution_logs/
+* CI badge file will be placed at $WORKSPACE

tests/interop/create_ci_badge.py

@@ -0,0 +1,84 @@
+import json
+import os
+import subprocess
+from datetime import datetime
+
+from junitparser import JUnitXml
+
+oc = os.environ["HOME"] + "/oc_client/oc"
+
+ci_badge = {
+    "schemaVersion": 1,
+    "label": "Community test",
+    "message": "",
+    "color": "red",
+    "openshiftVersion": "",
+    "infraProvider": os.environ.get("INFRA_PROVIDER"),
+    "patternName": os.environ.get("PATTERN_NAME"),
+    "patternRepo": "",
+    "patternBranch": "",
+    "date": datetime.today().strftime("%Y-%m-%d"),
+    "testSource": "Community",
+    "debugInfo": None,
+}
+
+
+def get_openshift_version():
+    try:
+        version_ret = subprocess.run([oc, "version", "-o", "json"], capture_output=True)
+        version_out = version_ret.stdout.decode("utf-8")
+        openshift_version = json.loads(version_out)["openshiftVersion"]
+        major_minor = ".".join(openshift_version.split(".")[:-1])
+        return openshift_version, major_minor
+    except KeyError as e:
+        print("KeyError:" + str(e))
+        return None
+
+
+if __name__ == "__main__":
+    versions = get_openshift_version()
+    ci_badge["openshiftVersion"] = versions[0]
+
+    pattern_repo = subprocess.run(
+        ["git", "config", "--get", "remote.origin.url"], capture_output=True, text=True
+    )
+    pattern_branch = subprocess.run(
+        ["git", "branch", "--show-current"], capture_output=True, text=True
+    )
+
+    ci_badge["patternRepo"] = pattern_repo.stdout.strip()
+    ci_badge["patternBranch"] = pattern_branch.stdout.strip()
+
+    # Check each xml file for failures
+    results_dir = os.environ.get("WORKSPACE")
+    failures = 0
+
+    for file in os.listdir(results_dir):
+        if file.startswith("test_") and file.endswith(".xml"):
+            with open(os.path.join(results_dir, file), "r") as result_file:  # type: ignore
+                xml = JUnitXml.fromfile(result_file)  # type: ignore
+                for suite in xml:
+                    for case in suite:
+                        if case.result:
+                            failures += 1
+
+    # Determine badge color from results
+    if failures == 0:
+        ci_badge["color"] = "green"
+
+    # For now we assume `message` is the same as patternBranch
+    ci_badge["message"] = ci_badge["patternBranch"]
+
+    ci_badge_json_basename = (
+        os.environ.get("PATTERN_SHORTNAME")  # type: ignore
+        + "-"
+        + os.environ.get("INFRA_PROVIDER")
+        + "-"
+        + versions[1]
+        + "-stable-badge.json"
+    )
+    ci_badge_json_filename = os.path.join(results_dir, ci_badge_json_basename)  # type: ignore
+    print(f"Creating CI badge file at: {ci_badge_json_filename}")
+
+    with open(ci_badge_json_filename, "w") as ci_badge_file:
+        json.dump(ci_badge, ci_badge_file)

tests/interop/requirements.txt

@@ -0,0 +1,6 @@
+pytest
+kubernetes
+openshift
+openshift-python-wrapper
+junitparser
+git+https://github.com/validatedpatterns/vp-qe-test-common.git@development#egg=vp-qe-test-common

tests/interop/run_tests.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/bash
+
+export EXTERNAL_TEST="true"
+export PATTERN_NAME="CoCoPattern"
+export PATTERN_SHORTNAME="coco"
+
+if [ -z "${KUBECONFIG}" ]; then
+    echo "No kubeconfig file set for hub cluster"
+    exit 1
+fi
+
+if [ -z "${INFRA_PROVIDER}" ]; then
+    echo "INFRA_PROVIDER is not defined"
+    exit 1
+fi
+
+if [ -z "${WORKSPACE}" ]; then
+    export WORKSPACE=/tmp
+fi
+
+pytest -lv --disable-warnings test_subscription_status_hub.py --kubeconfig $KUBECONFIG --junit-xml $WORKSPACE/test_subscription_status_hub.xml
+
+pytest -lv --disable-warnings test_validate_hub_site_components.py --kubeconfig $KUBECONFIG --junit-xml $WORKSPACE/test_validate_hub_site_components.xml
+
+python3 create_ci_badge.py

tests/interop/test_subscription_status_hub.py

@@ -1,8 +1,4 @@
-import difflib
 import logging
-import os
-import re
-import subprocess
 
 import pytest
 from validatedpatterns_tests.interop import subscription
@@ -19,102 +15,16 @@ def test_subscription_status_hub(openshift_dyn_client):
         "openshift-gitops-operator": ["openshift-operators"],
         "advanced-cluster-management": ["open-cluster-management"],
         "multicluster-engine": ["multicluster-engine"],
+        "openshift-cert-manager-operator": ["cert-manager-operator"],
+        "sandboxed-containers-operator": ["openshift-sandboxed-containers-operator"],
+        "trustee-operator": ["trustee-operator-system"],
     }
 
-    (
-        operator_versions,
-        missing_subs,
-        unhealthy_subs,
-        missing_installplans,
-        upgrades_pending,
-    ) = subscription.subscription_status(openshift_dyn_client, expected_subs)
-
-    if missing_subs:
-        logger.error(f"FAIL: The following subscriptions are missing: {missing_subs}")
-    if unhealthy_subs:
-        logger.error(
-            f"FAIL: The following subscriptions are unhealthy: {unhealthy_subs}"
-        )
-    if missing_installplans:
-        logger.error(
-            f"FAIL: The install plan for the following subscriptions is missing: {missing_installplans}"
-        )
-    if upgrades_pending:
-        logger.error(
-            f"FAIL: The following subscriptions are in UpgradePending state: {upgrades_pending}"
-        )
-
-    cluster_version = subscription.openshift_version(openshift_dyn_client)
-    logger.info(f"Openshift version:\n{cluster_version.instance.status.history}")
-
-    if os.getenv("EXTERNAL_TEST") != "true":
-        shortversion = re.sub("(.[0-9]+$)", "", os.getenv("OPENSHIFT_VER"))
-        currentfile = os.getcwd() + "/operators_hub_current"
-        sourceFile = open(currentfile, "w")
-        for line in operator_versions:
-            logger.info(line)
-            print(line, file=sourceFile)
-        sourceFile.close()
-
-        logger.info("Clone operator-versions repo")
-        try:
-            operator_versions_repo = (
-                "git@gitlab.cee.redhat.com:mpqe/mps/vp/operator-versions.git"
-            )
-            clone = subprocess.run(
-                ["git", "clone", operator_versions_repo], capture_output=True, text=True
-            )
-            logger.info(clone.stdout)
-            logger.info(clone.stderr)
-        except Exception:
-            pass
-
-        previouspath = os.getcwd() + f"/operator-versions/mcgitops_hub_{shortversion}"
-        previousfile = f"mcgitops_hub_{shortversion}"
-
-        logger.info("Ensure previous file exists")
-        checkpath = os.path.exists(previouspath)
-        logger.info(checkpath)
-
-        if checkpath is True:
-            logger.info("Diff current operator list with previous file")
-            diff = opdiff(open(previouspath).readlines(), open(currentfile).readlines())
-            diffstring = "".join(diff)
-            logger.info(diffstring)
-
-            logger.info("Write diff to file")
-            sourceFile = open("operator_diffs_hub.log", "w")
-            print(diffstring, file=sourceFile)
-            sourceFile.close()
-        else:
-            logger.info("Skipping operator diff - previous file not found")
-
-    if missing_subs or unhealthy_subs or missing_installplans or upgrades_pending:
-        err_msg = "Subscription status check failed"
+    err_msg = subscription.subscription_status(
+        openshift_dyn_client, expected_subs, diff=True
+    )
+    if err_msg:
         logger.error(f"FAIL: {err_msg}")
         assert False, err_msg
     else:
-        # Only push the new operarator list if the test passed
-        # and we are not testing a pre-release operator nor
-        # running externally
-        if os.getenv("EXTERNAL_TEST") != "true":
-            if checkpath is True and not os.environ["INDEX_IMAGE"]:
-                os.remove(previouspath)
-                os.rename(currentfile, previouspath)
-
-                cwd = os.getcwd() + "/operator-versions"
-                logger.info(f"CWD: {cwd}")
-
-                logger.info("Push new operator list")
-                subprocess.run(["git", "add", previousfile], cwd=cwd)
-                subprocess.run(
-                    ["git", "commit", "-m", "Update operator versions list"],
-                    cwd=cwd,
-                )
-                subprocess.run(["git", "push"], cwd=cwd)
-
         logger.info("PASS: Subscription status check passed")
-
-
-def opdiff(*args):
-    return filter(lambda x: not x.startswith(" "), difflib.ndiff(*args))