From 09b97a346dbab0f8d7373d709c0647449285039f Mon Sep 17 00:00:00 2001 From: Mark LaBonte Date: Thu, 24 Apr 2025 16:47:23 -0400 Subject: [PATCH 1/2] Update QE tests --- .../interop/test_subscription_status_edge.py | 37 ---- tests/interop/test_subscription_status_hub.py | 104 +--------- .../test_validate_edge_site_components.py | 187 ------------------ .../test_validate_hub_site_components.py | 124 ++---------- 4 files changed, 28 insertions(+), 424 deletions(-) delete mode 100644 tests/interop/test_subscription_status_edge.py delete mode 100644 tests/interop/test_validate_edge_site_components.py diff --git a/tests/interop/test_subscription_status_edge.py b/tests/interop/test_subscription_status_edge.py deleted file mode 100644 index b4e4b2df..00000000 --- a/tests/interop/test_subscription_status_edge.py +++ /dev/null @@ -1,37 +0,0 @@ -import logging - -import pytest -from validatedpatterns_tests.interop import subscription - -from . import __loggername__ - -logger = logging.getLogger(__loggername__) - - -@pytest.mark.subscription_status_edge -def test_subscription_status_edge(openshift_dyn_client): - # These are the operator subscriptions and their associated namespaces - expected_subs = { - "openshift-gitops-operator": ["openshift-operators"], - } - - ( - operator_versions, - missing_subs, - unhealthy_subs, - missing_installplans, - upgrades_pending, - ) = subscription.subscription_status(openshift_dyn_client, expected_subs) - - for line in operator_versions: - logger.info(line) - - cluster_version = subscription.openshift_version(openshift_dyn_client) - logger.info(f"Openshift version:\n{cluster_version.instance.status.history}") - - if missing_subs or unhealthy_subs or missing_installplans or upgrades_pending: - err_msg = "Subscription status check failed" - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - else: - logger.info("PASS: Subscription status check passed") diff --git a/tests/interop/test_subscription_status_hub.py b/tests/interop/test_subscription_status_hub.py index 6a575d69..9f50d5b7 100644 --- a/tests/interop/test_subscription_status_hub.py +++ b/tests/interop/test_subscription_status_hub.py @@ -1,8 +1,4 @@ -import difflib import logging -import os -import re -import subprocess import pytest from validatedpatterns_tests.interop import subscription @@ -19,102 +15,16 @@ def test_subscription_status_hub(openshift_dyn_client): "openshift-gitops-operator": ["openshift-operators"], "advanced-cluster-management": ["open-cluster-management"], "multicluster-engine": ["multicluster-engine"], + "openshift-cert-manager-operator": ["cert-manager-operator"], + "sandboxed-containers-operator": ["openshift-sandboxed-containers-operator"], + "trustee-operator": ["trustee-operator-system"], } - ( - operator_versions, - missing_subs, - unhealthy_subs, - missing_installplans, - upgrades_pending, - ) = subscription.subscription_status(openshift_dyn_client, expected_subs) - - if missing_subs: - logger.error(f"FAIL: The following subscriptions are missing: {missing_subs}") - if unhealthy_subs: - logger.error( - f"FAIL: The following subscriptions are unhealthy: {unhealthy_subs}" - ) - if missing_installplans: - logger.error( - f"FAIL: The install plan for the following subscriptions is missing: {missing_installplans}" - ) - if upgrades_pending: - logger.error( - f"FAIL: The following subscriptions are in UpgradePending state: {upgrades_pending}" - ) - - cluster_version = subscription.openshift_version(openshift_dyn_client) - logger.info(f"Openshift version:\n{cluster_version.instance.status.history}") - - if os.getenv("EXTERNAL_TEST") != "true": - shortversion = re.sub("(.[0-9]+$)", "", os.getenv("OPENSHIFT_VER")) - currentfile = os.getcwd() + "/operators_hub_current" - sourceFile = open(currentfile, "w") - for line in operator_versions: - logger.info(line) - print(line, file=sourceFile) - sourceFile.close() - - logger.info("Clone operator-versions repo") - try: - operator_versions_repo = ( - "git@gitlab.cee.redhat.com:mpqe/mps/vp/operator-versions.git" - ) - clone = subprocess.run( - ["git", "clone", operator_versions_repo], capture_output=True, text=True - ) - logger.info(clone.stdout) - logger.info(clone.stderr) - except Exception: - pass - - previouspath = os.getcwd() + f"/operator-versions/mcgitops_hub_{shortversion}" - previousfile = f"mcgitops_hub_{shortversion}" - - logger.info("Ensure previous file exists") - checkpath = os.path.exists(previouspath) - logger.info(checkpath) - - if checkpath is True: - logger.info("Diff current operator list with previous file") - diff = opdiff(open(previouspath).readlines(), open(currentfile).readlines()) - diffstring = "".join(diff) - logger.info(diffstring) - - logger.info("Write diff to file") - sourceFile = open("operator_diffs_hub.log", "w") - print(diffstring, file=sourceFile) - sourceFile.close() - else: - logger.info("Skipping operator diff - previous file not found") - - if missing_subs or unhealthy_subs or missing_installplans or upgrades_pending: - err_msg = "Subscription status check failed" + err_msg = subscription.subscription_status( + openshift_dyn_client, expected_subs, diff=True + ) + if err_msg: logger.error(f"FAIL: {err_msg}") assert False, err_msg else: - # Only push the new operarator list if the test passed - # and we are not testing a pre-release operator nor - # running externally - if os.getenv("EXTERNAL_TEST") != "true": - if checkpath is True and not os.environ["INDEX_IMAGE"]: - os.remove(previouspath) - os.rename(currentfile, previouspath) - - cwd = os.getcwd() + "/operator-versions" - logger.info(f"CWD: {cwd}") - - logger.info("Push new operator list") - subprocess.run(["git", "add", previousfile], cwd=cwd) - subprocess.run( - ["git", "commit", "-m", "Update operator versions list"], - cwd=cwd, - ) - subprocess.run(["git", "push"], cwd=cwd) - logger.info("PASS: Subscription status check passed") - - -def opdiff(*args): - return filter(lambda x: not x.startswith(" "), difflib.ndiff(*args)) diff --git a/tests/interop/test_validate_edge_site_components.py b/tests/interop/test_validate_edge_site_components.py deleted file mode 100644 index 343f97c8..00000000 --- a/tests/interop/test_validate_edge_site_components.py +++ /dev/null @@ -1,187 +0,0 @@ -import logging -import os - -import pytest -from ocp_resources.route import Route -from validatedpatterns_tests.interop import components -from validatedpatterns_tests.interop.crd import ArgoCD -from validatedpatterns_tests.interop.edge_util import ( - get_long_live_bearer_token, - get_site_response, -) - -from . import __loggername__ - -logger = logging.getLogger(__loggername__) - -oc = os.environ["HOME"] + "/oc_client/oc" - -""" -Validate following multicloud-gitops components pods and endpoints on edge site (line server): - -1) argocd -2) ACM agents -3) applications health (Applications deployed through argocd) -""" - - -@pytest.mark.test_validate_edge_site_components -def test_validate_edge_site_components(): - logger.info("Checking Openshift version on edge site") - version_out = components.dump_openshift_version() - logger.info(f"Openshift version:\n{version_out}") - - -@pytest.mark.validate_edge_site_reachable -def test_validate_edge_site_reachable(kube_config, openshift_dyn_client): - logger.info("Check if edge site API end point is reachable") - edge_api_url = kube_config.host - if not edge_api_url: - err_msg = "Edge site url is missing in kubeconfig file" - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - else: - logger.info(f"EDGE api url : {edge_api_url}") - - bearer_token = get_long_live_bearer_token( - dyn_client=openshift_dyn_client, - namespace="openshift-gitops", - sub_string="argocd-dex-server-token", - ) - - if not bearer_token: - assert False, "Bearer token is missing for argocd-dex-server" - - edge_api_response = get_site_response( - site_url=edge_api_url, bearer_token=bearer_token - ) - - if edge_api_response.status_code != 200: - err_msg = "Edge site is not reachable. Please check the deployment." - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - else: - logger.info("PASS: Edge site is reachable") - - -@pytest.mark.check_pod_status_edge -def test_check_pod_status(openshift_dyn_client): - logger.info("Checking pod status") - - err_msg = [] - projects = [ - "openshift-operators", - "open-cluster-management-agent", - "open-cluster-management-agent-addon", - "openshift-gitops", - ] - - missing_projects = components.check_project_absense(openshift_dyn_client, projects) - missing_pods = [] - failed_pods = [] - - for project in projects: - missing_pods += components.check_pod_absence(openshift_dyn_client, project) - failed_pods += components.check_pod_status(openshift_dyn_client, project) - - if missing_projects: - err_msg.append(f"The following namespaces are missing: {missing_projects}") - - if missing_pods: - err_msg.append( - f"The following namespaces have no pods deployed: {missing_pods}" - ) - - if failed_pods: - err_msg.append(f"The following pods are failed: {failed_pods}") - - if err_msg: - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - else: - logger.info("PASS: Pod status check succeeded.") - - -@pytest.mark.validate_argocd_reachable_edge_site -def test_validate_argocd_reachable_edge_site(openshift_dyn_client): - namespace = "openshift-gitops" - - try: - for route in Route.get( - dyn_client=openshift_dyn_client, - namespace=namespace, - name="openshift-gitops-server", - ): - argocd_route_url = route.instance.spec.host - except StopIteration: - err_msg = f"Argocd url/route is missing in {namespace} namespace" - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - - logger.info("Check if argocd route/url on hub site is reachable") - if not argocd_route_url: - err_msg = f"Argocd url/route is missing in {namespace} namespace" - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - else: - final_argocd_url = f"{'https://'}{argocd_route_url}" - logger.info(f"Argocd route/url : {final_argocd_url}") - - bearer_token = get_long_live_bearer_token( - dyn_client=openshift_dyn_client, - namespace=namespace, - sub_string="argocd-dex-server-token", - ) - if not bearer_token: - err_msg = "Bearer token is missing for argocd-dex-server" - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - else: - logger.debug(f"Argocd bearer token : {bearer_token}") - - argocd_route_response = get_site_response( - site_url=final_argocd_url, bearer_token=bearer_token - ) - - logger.info(f"Argocd route response : {argocd_route_response}") - - if argocd_route_response.status_code != 200: - err_msg = "Argocd is not reachable. Please check the deployment." - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - else: - logger.info("PASS: Argocd is reachable") - - -@pytest.mark.validate_argocd_applications_health_edge_site -def test_validate_argocd_applications_health_edge_site(openshift_dyn_client): - unhealthy_apps = [] - logger.info("Get all applications deployed by argocd on edge site") - projects = ["openshift-gitops"] - for project in projects: - logger.info(f"PROJECT: {project}") - for app in ArgoCD.get(dyn_client=openshift_dyn_client, namespace=project): - app_name = app.instance.metadata.name - app_health = app.instance.status.health.status - app_sync = app.instance.status.sync.status - - logger.info(f"Status for {app_name} : {app_health} : {app_sync}") - - if "Healthy" != app_health or "Synced" != app_sync: - logger.info(f"Dumping failed resources for app: {app_name}") - unhealthy_apps.append(app_name) - try: - for res in app.instance.status.resources: - if ( - res.health and res.health.status != "Healthy" - ) or res.status != "Synced": - logger.info(f"\n{res}") - except TypeError: - logger.info(f"No resources found for app: {app_name}") - - if unhealthy_apps: - err_msg = "Some or all applications deployed on edge site are unhealthy" - logger.error(f"FAIL: {err_msg}:\n{unhealthy_apps}") - assert False, err_msg - else: - logger.info("PASS: All applications deployed on edge site are healthy.") diff --git a/tests/interop/test_validate_hub_site_components.py b/tests/interop/test_validate_hub_site_components.py index d70cc174..0770ae7b 100644 --- a/tests/interop/test_validate_hub_site_components.py +++ b/tests/interop/test_validate_hub_site_components.py @@ -2,10 +2,8 @@ import os import pytest -import yaml from ocp_resources.storage_class import StorageClass from validatedpatterns_tests.interop import application, components -from validatedpatterns_tests.interop.crd import ManagedCluster from . import __loggername__ @@ -13,15 +11,6 @@ oc = os.environ["HOME"] + "/oc_client/oc" -""" -Validate following multicloud-gitops components pods and endpoints on hub site (central server): - -1) ACM (Advanced Cluster Manager) and self-registration -2) argocd -3) openshift operators -4) applications health (Applications deployed through argocd) -""" - @pytest.mark.test_validate_hub_site_components def test_validate_hub_site_components(openshift_dyn_client): @@ -40,19 +29,8 @@ def test_validate_hub_site_components(openshift_dyn_client): @pytest.mark.validate_hub_site_reachable def test_validate_hub_site_reachable(kube_config, openshift_dyn_client): logger.info("Check if hub site API end point is reachable") - namespace = "openshift-gitops" - sub_string = "argocd-dex-server-token" - try: - hub_api_url = application.get_site_api_url(kube_config) - hub_api_response = application.get_site_api_response( - openshift_dyn_client, namespace, sub_string, hub_api_url - ) - except AssertionError as e: - logger.error(f"FAIL: {e}") - assert False, e - - if hub_api_response.status_code != 200: - err_msg = "Hub site is not reachable. Please check the deployment." + err_msg = components.validate_site_reachable(kube_config, openshift_dyn_client) + if err_msg: logger.error(f"FAIL: {err_msg}") assert False, err_msg else: @@ -62,36 +40,15 @@ def test_validate_hub_site_reachable(kube_config, openshift_dyn_client): @pytest.mark.check_pod_status_hub def test_check_pod_status(openshift_dyn_client): logger.info("Checking pod status") - - err_msg = [] projects = [ "openshift-operators", "open-cluster-management", "open-cluster-management-hub", "openshift-gitops", + "coco-pattern-simple", "vault", ] - - missing_projects = components.check_project_absense(openshift_dyn_client, projects) - missing_pods = [] - failed_pods = [] - - for project in projects: - logger.info(f"Checking pods in namespace '{project}'") - missing_pods += components.check_pod_absence(openshift_dyn_client, project) - failed_pods += components.check_pod_status(openshift_dyn_client, projects) - - if missing_projects: - err_msg.append(f"The following namespaces are missing: {missing_projects}") - - if missing_pods: - err_msg.append( - f"The following namespaces have no pods deployed: {missing_pods}" - ) - - if failed_pods: - err_msg.append(f"The following pods are failed: {failed_pods}") - + err_msg = components.check_pod_status(openshift_dyn_client, projects) if err_msg: logger.error(f"FAIL: {err_msg}") assert False, err_msg @@ -99,62 +56,25 @@ def test_check_pod_status(openshift_dyn_client): logger.info("PASS: Pod status check succeeded.") -@pytest.mark.validate_acm_self_registration_managed_clusters -def test_validate_acm_self_registration_managed_clusters(openshift_dyn_client): - logger.info("Check ACM self registration for edge site") - - kubefile = os.getenv("KUBECONFIG_EDGE") - kubefile_exp = os.path.expandvars(kubefile) - with open(kubefile_exp) as stream: - try: - out = yaml.safe_load(stream) - site_name = out["clusters"][0]["name"] - except yaml.YAMLError: - err_msg = "Failed to load kubeconfig file" - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - - clusters = ManagedCluster.get(dyn_client=openshift_dyn_client, name=site_name) - cluster = next(clusters) - is_managed_cluster_joined, managed_cluster_status = cluster.self_registered - - logger.info(f"Cluster Managed : {is_managed_cluster_joined}") - logger.info(f"Managed Cluster Status : {managed_cluster_status}") - - if not is_managed_cluster_joined: - err_msg = f"{site_name} is not self registered" - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - else: - logger.info(f"PASS: {site_name} is self registered") +# @pytest.mark.validate_acm_self_registration_managed_clusters +# def test_validate_acm_self_registration_managed_clusters(openshift_dyn_client): +# logger.info("Check ACM self registration for edge site") +# kubefiles = [os.getenv("KUBECONFIG_EDGE")] +# err_msg = components.validate_acm_self_registration_managed_clusters( +# openshift_dyn_client, kubefiles +# ) +# if err_msg: +# logger.error(f"FAIL: {err_msg}") +# assert False, err_msg +# else: +# logger.info("PASS: Edge site is self registered") @pytest.mark.validate_argocd_reachable_hub_site def test_validate_argocd_reachable_hub_site(openshift_dyn_client): - namespace = "openshift-gitops" - name = "openshift-gitops-server" - sub_string = "argocd-dex-server-token" logger.info("Check if argocd route/url on hub site is reachable") - try: - argocd_route_url = application.get_argocd_route_url( - openshift_dyn_client, namespace, name - ) - argocd_route_response = application.get_site_api_response( - openshift_dyn_client, namespace, sub_string, argocd_route_url - ) - except StopIteration: - err_msg = "Argocd url/route is missing in open-cluster-management namespace" - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - except AssertionError: - err_msg = "Bearer token is missing for argocd-dex-server" - logger.error(f"FAIL: {err_msg}") - assert False, err_msg - - logger.info(f"Argocd route response : {argocd_route_response}") - - if argocd_route_response.status_code != 200: - err_msg = "Argocd is not reachable. Please check the deployment" + err_msg = components.validate_argocd_reachable(openshift_dyn_client) + if err_msg: logger.error(f"FAIL: {err_msg}") assert False, err_msg else: @@ -163,13 +83,11 @@ def test_validate_argocd_reachable_hub_site(openshift_dyn_client): @pytest.mark.validate_argocd_applications_health_hub_site def test_validate_argocd_applications_health_hub_site(openshift_dyn_client): - unhealthy_apps = [] logger.info("Get all applications deployed by argocd on hub site") projects = ["openshift-gitops", "multicloud-gitops-hub"] - for project in projects: - unhealthy_apps += application.get_argocd_application_status( - openshift_dyn_client, project - ) + unhealthy_apps = application.get_argocd_application_status( + openshift_dyn_client, projects + ) if unhealthy_apps: err_msg = "Some or all applications deployed on hub site are unhealthy" logger.error(f"FAIL: {err_msg}:\n{unhealthy_apps}") From 966d426e0b020a99b79e823f8d95652565cd34a0 Mon Sep 17 00:00:00 2001 From: Mark LaBonte Date: Thu, 24 Apr 2025 16:54:04 -0400 Subject: [PATCH 2/2] Add support file for QE tests --- tests/interop/README.md | 23 +++++++++ tests/interop/create_ci_badge.py | 84 ++++++++++++++++++++++++++++++++ tests/interop/requirements.txt | 6 +++ tests/interop/run_tests.sh | 25 ++++++++++ 4 files changed, 138 insertions(+) create mode 100644 tests/interop/README.md create mode 100644 tests/interop/create_ci_badge.py create mode 100644 tests/interop/requirements.txt create mode 100755 tests/interop/run_tests.sh diff --git a/tests/interop/README.md b/tests/interop/README.md new file mode 100644 index 00000000..8a4d6f6e --- /dev/null +++ b/tests/interop/README.md @@ -0,0 +1,23 @@ +# Running tests + +## Prerequisites + +* Openshift cluster with coco-pattern installed +* kubeconfig file for Openshift cluster +* oc client installed at ~/oc_client/oc + +## Steps + +* create python3 venv, clone multicloud-gitops repository +* export KUBECONFIG=\ +* export INFRA_PROVIDER=azure +* (optional) export WORKSPACE=\ (defaults to /tmp) +* cd coco-pattern/tests/interop +* pip install -r requirements.txt +* ./run_tests.sh + +## Results + +* results .xml files will be placed at $WORKSPACE +* test logs will be placed at $WORKSPACE/.results/test_execution_logs/ +* CI badge file will be placed at $WORKSPACE diff --git a/tests/interop/create_ci_badge.py b/tests/interop/create_ci_badge.py new file mode 100644 index 00000000..8ed179a4 --- /dev/null +++ b/tests/interop/create_ci_badge.py @@ -0,0 +1,84 @@ +import json +import os +import subprocess +from datetime import datetime + +from junitparser import JUnitXml + +oc = os.environ["HOME"] + "/oc_client/oc" + +ci_badge = { + "schemaVersion": 1, + "label": "Community test", + "message": "", + "color": "red", + "openshiftVersion": "", + "infraProvider": os.environ.get("INFRA_PROVIDER"), + "patternName": os.environ.get("PATTERN_NAME"), + "patternRepo": "", + "patternBranch": "", + "date": datetime.today().strftime("%Y-%m-%d"), + "testSource": "Community", + "debugInfo": None, +} + + +def get_openshift_version(): + try: + version_ret = subprocess.run([oc, "version", "-o", "json"], capture_output=True) + version_out = version_ret.stdout.decode("utf-8") + openshift_version = json.loads(version_out)["openshiftVersion"] + major_minor = ".".join(openshift_version.split(".")[:-1]) + return openshift_version, major_minor + except KeyError as e: + print("KeyError:" + str(e)) + return None + + +if __name__ == "__main__": + versions = get_openshift_version() + ci_badge["openshiftVersion"] = versions[0] + + pattern_repo = subprocess.run( + ["git", "config", "--get", "remote.origin.url"], capture_output=True, text=True + ) + pattern_branch = subprocess.run( + ["git", "branch", "--show-current"], capture_output=True, text=True + ) + + ci_badge["patternRepo"] = pattern_repo.stdout.strip() + ci_badge["patternBranch"] = pattern_branch.stdout.strip() + + # Check each xml file for failures + results_dir = os.environ.get("WORKSPACE") + failures = 0 + + for file in os.listdir(results_dir): + if file.startswith("test_") and file.endswith(".xml"): + with open(os.path.join(results_dir, file), "r") as result_file: # type: ignore + xml = JUnitXml.fromfile(result_file) # type: ignore + for suite in xml: + for case in suite: + if case.result: + failures += 1 + + # Determine badge color from results + if failures == 0: + ci_badge["color"] = "green" + + # For now we assume `message` is the same as patternBranch + ci_badge["message"] = ci_badge["patternBranch"] + + ci_badge_json_basename = ( + os.environ.get("PATTERN_SHORTNAME") # type: ignore + + "-" + + os.environ.get("INFRA_PROVIDER") + + "-" + + versions[1] + + "-stable-badge.json" + ) + ci_badge_json_filename = os.path.join(results_dir, ci_badge_json_basename) # type: ignore + print(f"Creating CI badge file at: {ci_badge_json_filename}") + + with open(ci_badge_json_filename, "w") as ci_badge_file: + json.dump(ci_badge, ci_badge_file) diff --git a/tests/interop/requirements.txt b/tests/interop/requirements.txt new file mode 100644 index 00000000..3b20852f --- /dev/null +++ b/tests/interop/requirements.txt @@ -0,0 +1,6 @@ +pytest +kubernetes +openshift +openshift-python-wrapper +junitparser +git+https://github.com/validatedpatterns/vp-qe-test-common.git@development#egg=vp-qe-test-common \ No newline at end of file diff --git a/tests/interop/run_tests.sh b/tests/interop/run_tests.sh new file mode 100755 index 00000000..ee33cdda --- /dev/null +++ b/tests/interop/run_tests.sh @@ -0,0 +1,25 @@ +#!/usr/bin/bash + +export EXTERNAL_TEST="true" +export PATTERN_NAME="CoCoPattern" +export PATTERN_SHORTNAME="coco" + +if [ -z "${KUBECONFIG}" ]; then + echo "No kubeconfig file set for hub cluster" + exit 1 +fi + +if [ -z "${INFRA_PROVIDER}" ]; then + echo "INFRA_PROVIDER is not defined" + exit 1 +fi + +if [ -z "${WORKSPACE}" ]; then + export WORKSPACE=/tmp +fi + +pytest -lv --disable-warnings test_subscription_status_hub.py --kubeconfig $KUBECONFIG --junit-xml $WORKSPACE/test_subscription_status_hub.xml + +pytest -lv --disable-warnings test_validate_hub_site_components.py --kubeconfig $KUBECONFIG --junit-xml $WORKSPACE/test_validate_hub_site_components.xml + +python3 create_ci_badge.py