diff --git a/scripts/run-tests b/scripts/run-tests new file mode 100755 index 0000000..4d13827 --- /dev/null +++ b/scripts/run-tests @@ -0,0 +1,409 @@ +#!/usr/bin/env python +# vim: filetype=python syntax=python tabstop=4 expandtab + +# WARNING +# No not modify this file, the source of truth is +# https://github.com/stackabletech/operator-templating/blob/main/template/scripts/run-tests +# WARNING + +import argparse +import collections.abc +import contextlib +import logging +import os +import re +import shutil +import subprocess +import sys +import tempfile + +__version__ = "0.0.1" + +DESCRIPTION = """ +Run integration tests. Call this script from the root of the repository. + +Exits with 0 on success, 1 on failure. + +Requires the following commands to be installed: +* beku +* stackablectl +* kubectl +* kubectl-kuttl + +Examples: + +1. Install operators, run all tests and clean up test namespaces: + + ./scripts/run-tests --parallel 4 + +2. Install operators but for Airflow use version "0.0.0-pr123" instead of "0.0.0-dev" and run all tests as above: + + ./scripts/run-tests --operator airflow=0.0.0-pr123 --parallel 4 + +3. Do not install any operators, run the smoke test suite and keep namespace: + + ./scripts/run-tests --skip-release --skip-delete --test-suite smoke-latest + +4. Run the ldap test(s) from the openshift test suite and keep namespace: + + ./scripts/run-tests --skip-release --skip-delete --test-suite openshift --test ldap + +5. Run the smoke test suite in the namespace "smoke". The namespace will be + created if it doesn't exist and will not be deleted when the tests end. + + ./scripts/run-tests --test-suite smoke-latest --namespace smoke +""" + + +class TestRunnerException(Exception): + pass + + +def parse_args(argv: list[str]) -> argparse.Namespace: + """Parse command line args.""" + parser = argparse.ArgumentParser( + description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--version", + help="Display application version", + action="version", + version=f"%(prog)s {__version__}", + ) + + parser.add_argument( + "--skip-delete", + help="Do not delete test namespaces.", + action="store_true", + ) + + parser.add_argument( + "--skip-tests", + help="Do not actually run the tests.", + action="store_true", + ) + + parser.add_argument( + "--skip-release", + help="Do not install operators.", + action="store_true", + ) + + parser.add_argument( + "--parallel", + help="How many tests to run in parallel. Default 2.", + type=int, + required=False, + default=2, + ) + + parser.add_argument( + "--operator", + help="Patch operator version in release.yaml. Format =", + action="append", + type=cli_parse_operator_args, + default=[], + ) + + parser.add_argument( + "--skip-operator", + help="Skip given operator(s) when installing a release.", + action="append", + default=[], + ) + + parser.add_argument( + "--test", + help="Kuttl test to run.", + type=str, + required=False, + ) + + parser.add_argument( + "--test-suite", + help="Name of the test suite to expand. Default: default", + type=str, + required=False, + ) + + parser.add_argument( + "--log-level", + help="Set log level.", + type=cli_log_level, + required=False, + default=logging.INFO, + ) + + parser.add_argument( + "--namespace", + help="Namespace to run the tests in. It will be created if it doesn't already exist.", + type=str, + required=False, + ) + + return parser.parse_args(argv) + + +def cli_parse_operator_args(args: str) -> tuple[str, str]: + if "=" not in args: + raise argparse.ArgumentTypeError( + f"Invalid operator argument: {args}. Must be in format =" + ) + op, version = args.split("=", maxsplit=1) + return op, version + + +def cli_log_level(cli_arg: str) -> int: + match cli_arg: + case "debug": + return logging.DEBUG + case "info": + return logging.INFO + case "error": + return logging.ERROR + case "warning": + return logging.WARNING + case "critical": + return logging.CRITICAL + case _: + raise argparse.ArgumentTypeError("Invalid log level") + + +def have_requirements() -> None: + commands = [ + ("beku", "https://github.com/stackabletech/beku.py"), + ( + "stackablectl", + "https://github.com/stackabletech/stackable-cockpit/blob/main/rust/stackablectl/README.md", + ), + ("kubectl", "https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/"), + ("kubectl-kuttl", "https://kuttl.dev/"), + ] + + err = False + for command, url in commands: + if not shutil.which(command): + logging.error(f'Command "{command}" not found, please install from {url}') + err = True + if err: + raise TestRunnerException() + + +@contextlib.contextmanager +def release_file( + operators: list[tuple[str, str]], skip_ops: list[str] +) -> collections.abc.Generator[str, None, None]: + """Generate a (possibly modified) copy of the release.yaml file. + + Operator versions passed as --operator take precedence over the release.yaml contents. + + Operators passed as --skip-operator are excluded from the resulting release.yaml contents. + + If an invalid operator name is provided (i.e. one that doesn't exist in the + original release file), a TestRunnerException is raised. + + Yields the name of the (potentially patched) release file. This is a temporary + file that will be deleted when the context manager exits. + """ + + def _patch(): + release_file = os.path.join("tests", "release.yaml") + # A marker to validate that all ops were patched + patched_release = [] + with open(release_file, "r") as f: + patched_ops = [] + patch_version = "" + for line in f: + if patch_version: + line = re.sub(":.+$", f": {patch_version}", line) + patch_version = "" + else: + for op, version in operators: + if op in line: + patch_version = version + patched_ops.append(op) + break + patched_release.append(line.rstrip("\n")) + + # Sanity test that cli didn't contain garbage that is silently discarded + ops_not_patched = set([op for op, _ in operators]) - set(patched_ops) + if ops_not_patched: + logging.error( + f"Patched operators [{', '.join(ops_not_patched)}] not found in {release_file}" + ) + raise TestRunnerException() + + # Filter out skip operators + release_contents = [] + skip_lines = 0 + valid_skip_ops = [] + for line in patched_release: + if skip_lines: + skip_lines -= 1 + continue + for op in skip_ops: + if op in line: + # Every product section has 1 line of additional config to skip + skip_lines = 1 + valid_skip_ops.append(op) + break + else: + release_contents.append(line) + # Sanity test that cli didn't contain garbage that is silently discarded + ops_not_skipped = set(skip_ops) - set(valid_skip_ops) + if ops_not_skipped: + logging.error( + f"Skipped operators [{', '.join(ops_not_skipped)}] not found in {release_file}" + ) + raise TestRunnerException() + + with tempfile.NamedTemporaryFile( + mode="w", + delete=False, + prefix="patched", + ) as f: + pcontents = "\n".join(release_contents) + logging.debug(f"Writing patched release to {f.name}: {pcontents}\n") + f.write(pcontents) + return f.name + + release_file = _patch() + try: + yield release_file + except TestRunnerException as e: + logging.error(f"Caught exception: {e}") + raise + finally: + if "patched" in release_file: + try: + logging.debug(f"Removing patched release file : {release_file}") + os.remove(release_file) + except FileNotFoundError | OSError: + logging.error(f"Failed to delete patched release file: {release_file}") + + +def maybe_install_release(skip_release: bool, release_file: str) -> None: + if skip_release: + logging.debug("Skip release installation") + return + stackablectl_err = "" + try: + stackablectl_cmd = [ + "stackablectl", + "release", + "install", + "--release-file", + release_file, + "tests", + ] + logging.debug(f"Running : {stackablectl_cmd}") + + completed_proc = subprocess.run( + stackablectl_cmd, + capture_output=True, + check=True, + ) + # stackablectl doesn't return a non-zero exit code on failure + # so we need to check stderr for errors + stackablectl_err = completed_proc.stderr.decode("utf-8") + if "error" in stackablectl_err.lower(): + logging.error(stackablectl_err) + logging.error("stackablectl failed") + raise TestRunnerException() + + except subprocess.CalledProcessError as e: + # in case stackablectl starts returning non-zero exit codes + logging.error(e.stderr.decode("utf-8")) + logging.error("stackablectl failed") + raise TestRunnerException() + + +def gen_tests(test_suite: str, namespace: str) -> None: + try: + beku_cmd = [ + "beku", + "--test_definition", + os.path.join("tests", "test-definition.yaml"), + "--kuttl_test", + os.path.join("tests", "kuttl-test.yaml.jinja2"), + "--template_dir", + os.path.join("tests", "templates", "kuttl"), + "--output_dir", + os.path.join("tests", "_work"), + ] + if test_suite: + beku_cmd.extend(["--suite", test_suite]) + if namespace: + beku_cmd.extend(["--namespace", namespace]) + + logging.debug(f"Running : {beku_cmd}") + subprocess.run( + beku_cmd, + check=True, + ) + except subprocess.CalledProcessError: + logging.error("beku failed") + raise TestRunnerException() + + +def run_tests(test: str, parallel: int, namespace: str, skip_delete: bool) -> None: + try: + kuttl_cmd = ["kubectl-kuttl", "test"] + if test: + kuttl_cmd.extend(["--test", test]) + if parallel: + kuttl_cmd.extend(["--parallel", str(parallel)]) + if skip_delete: + kuttl_cmd.extend(["--skip-delete"]) + if namespace: + kuttl_cmd.extend(["--namespace", namespace]) + # kuttl doesn't create the namespace so we need to do it ourselves + create_ns_cmd = ["kubectl", "create", "namespace", namespace] + try: + logging.debug(f"Running : {create_ns_cmd}") + subprocess.run( + create_ns_cmd, + check=True, + capture_output=True, + ) + except subprocess.CalledProcessError as e: + stderr = e.stderr.decode("utf-8") + # If the namespace already exists, this will fail and we ignore the + # error. If it fails for any other reason, we raise an exception. + if "already exists" not in stderr: + logging.error(stderr) + logging.error("namespace creation failed") + raise TestRunnerException() + + logging.debug(f"Running : {kuttl_cmd}") + + subprocess.run( + kuttl_cmd, + cwd="tests/_work", + check=True, + ) + except subprocess.CalledProcessError: + logging.error("kuttl failed") + raise TestRunnerException() + + +def main(argv) -> int: + ret = 0 + try: + opts = parse_args(argv[1:]) + logging.basicConfig(encoding="utf-8", level=opts.log_level) + have_requirements() + gen_tests(opts.test_suite, opts.namespace) + with release_file(opts.operator, opts.skip_operator) as f: + maybe_install_release(opts.skip_release, f) + if opts.skip_tests: + logging.info("Skip running tests.") + else: + run_tests(opts.test, opts.parallel, opts.namespace, opts.skip_delete) + except TestRunnerException: + ret = 1 + return ret + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh new file mode 100755 index 0000000..a31a138 --- /dev/null +++ b/scripts/run_tests.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +./scripts/run-tests "$@" diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..e96b7d6 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +_work/ diff --git a/tests/kuttl-test.yaml.jinja2 b/tests/kuttl-test.yaml.jinja2 new file mode 100644 index 0000000..8a5620c --- /dev/null +++ b/tests/kuttl-test.yaml.jinja2 @@ -0,0 +1,28 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +testDirs: + {% for testcase in testinput.tests %} + - ./tests/{{ testcase.name }} + {% endfor %} + +startKIND: false +suppress: ["events"] +parallel: 2 + +# The timeout (in seconds) is used when namespaces are created or +# deleted, and, if not overridden, in TestSteps, TestAsserts, and +# Commands. If not set, the timeout is 30 seconds by default. +# +# The deletion of a namespace can take a while until all resources, +# especially PersistentVolumeClaims, are gracefully shut down. If the +# timeout is reached in the meantime, even a successful test case is +# considered a failure. +# +# For instance, the termination grace period of the Vector aggregator in +# the logging tests is set to 60 seconds. If there are logs entries +# which could not be forwarded yet to the external aggregator defined in +# the VECTOR_AGGREGATOR environment variable, then the test aggregator +# uses this period of time by trying to forward the events. In this +# case, deleting a namespace with several Pods takes about 90 seconds. +timeout: 300 diff --git a/tests/release.yaml b/tests/release.yaml new file mode 100644 index 0000000..4093b6e --- /dev/null +++ b/tests/release.yaml @@ -0,0 +1,16 @@ +# Contains all operators required to run the test suite. +--- +releases: + # Do not change the name of the release as it's referenced from run-tests + tests: + releaseDate: 1970-01-01 + description: Integration test + products: + commons: + operatorVersion: 0.0.0-dev + secret: + operatorVersion: 0.0.0-dev + listener: + operatorVersion: 0.0.0-dev + trino: + operatorVersion: 0.0.0-dev diff --git a/tests/templates/kuttl/smoke/00-assert.yaml.j2 b/tests/templates/kuttl/smoke/00-assert.yaml.j2 new file mode 100644 index 0000000..50b1d4c --- /dev/null +++ b/tests/templates/kuttl/smoke/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 0000000..2d6a0df --- /dev/null +++ b/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 b/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 new file mode 100644 index 0000000..67185ac --- /dev/null +++ b/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/smoke/00-rbac.yaml.j2 b/tests/templates/kuttl/smoke/00-rbac.yaml.j2 new file mode 100644 index 0000000..4abbf4f --- /dev/null +++ b/tests/templates/kuttl/smoke/00-rbac.yaml.j2 @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: trino-lb +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: trino-lb +subjects: + - kind: ServiceAccount + name: trino-lb +roleRef: + kind: Role + name: trino-lb + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: trino-lb + labels: + app.kubernetes.io/name: trino-lb +rules: + - apiGroups: + - trino.stackable.tech + resources: + - trinoclusters + verbs: + - get + - list + - watch + - patch + - apiGroups: + - trino.stackable.tech + resources: + - trinoclusters/status + verbs: + - get diff --git a/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml b/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml new file mode 100644 index 0000000..3a69c2b --- /dev/null +++ b/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml @@ -0,0 +1,58 @@ +apiVersion: v1 +kind: Secret +metadata: + name: trino-lb-certificates +# Copied from example-configs/self-singed-certs +stringData: + cert.pem: | + -----BEGIN CERTIFICATE----- + MIIDkzCCAnugAwIBAgIUXVYkRCrM/ge03DVymDtXCuybp7gwDQYJKoZIhvcNAQEL + BQAwWTELMAkGA1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM + GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MB4X + DTIxMDczMTE0MjIxMloXDTIyMDczMTE0MjIxMlowWTELMAkGA1UEBhMCVVMxEzAR + BgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5 + IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A + MIIBCgKCAQEA02V5ZjmqLB/VQwTarrz/35qsa83L+DbAoa0001+jVmmC+G9Nufi0 + daroFWj/Uicv2fZWETU8JoZKUrX4BK9og5cg5rln/CtBRWCUYIwRgY9R/CdBGPn4 + kp+XkSJaCw74ZIyLy/Zfux6h8ES1m9YRnBza+s7U+ImRBRf4MRPtXQ3/mqJxAZYq + dOnKnvssRyD2qutgVTAxwMUvJWIivRhRYDj7WOpS4CEEeQxP1iH1/T5P7FdtTGdT + bVBABCA8JhL96uFGPpOYHcM/7R5EIA3yZ5FNg931QzoDITjtXGtQ6y9/l/IYkWm6 + J67RWcN0IoTsZhz0WNU4gAeslVtJLofn8QIDAQABo1MwUTAdBgNVHQ4EFgQUzFnK + NfS4LAYuKeWwHbzooER0yZ0wHwYDVR0jBBgwFoAUzFnKNfS4LAYuKeWwHbzooER0 + yZ0wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAk4O+e9jia59W + ZwetN4GU7OWcYhmOgSizRSs6u7mTfp62LDMt96WKU3THksOnZ44HnqWQxsSfdFVU + XJD12tjvVU8Z4FWzQajcHeemUYiDze8EAh6TnxnUcOrU8IcwiKGxCWRY/908jnWg + +MMscfMCMYTRdeTPqD8fGzAlUCtmyzH6KLE3s4Oo/r5+NR+Uvrwpdvb7xe0MwwO9 + Q/zR4N8ep/HwHVEObcaBofE1ssZLksX7ZgCP9wMgXRWpNAtC5EWxMbxYjBfWFH24 + fDJlBMiGJWg8HHcxK7wQhFh+fuyNzE+xEWPsI9VL1zDftd9x8/QsOagyEOnY8Vxr + AopvZ09uEQ== + -----END CERTIFICATE----- + key.pem: | + -----BEGIN PRIVATE KEY----- + MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDTZXlmOaosH9VD + BNquvP/fmqxrzcv4NsChrTTTX6NWaYL4b025+LR1qugVaP9SJy/Z9lYRNTwmhkpS + tfgEr2iDlyDmuWf8K0FFYJRgjBGBj1H8J0EY+fiSn5eRIloLDvhkjIvL9l+7HqHw + RLWb1hGcHNr6ztT4iZEFF/gxE+1dDf+aonEBlip06cqe+yxHIPaq62BVMDHAxS8l + YiK9GFFgOPtY6lLgIQR5DE/WIfX9Pk/sV21MZ1NtUEAEIDwmEv3q4UY+k5gdwz/t + HkQgDfJnkU2D3fVDOgMhOO1ca1DrL3+X8hiRabonrtFZw3QihOxmHPRY1TiAB6yV + W0kuh+fxAgMBAAECggEADltu8k1qTFLhJgsXWxTFAAe+PBgfCT2WuaRM2So+qqjB + 12Of0MieYPt5hbK63HaC3nfHgqWt7yPhulpXfOH45C8IcgMXl93MMg0MJr58leMI + +2ojFrIrerHSFm5R1TxwDEwrVm/mMowzDWFtQCc6zPJ8wNn5RuP48HKfTZ3/2fjw + zEjSwPO2wFMfo1EJNTjlI303lFbdFBs67NaX6puh30M7Tn+gznHKyO5a7F57wkIt + fkgnEy/sgMedQlwX7bRpUoD6f0fZzV8Qz4cHFywtYErczZJh3VGitJoO/VCIDdty + RPXOAqVDd7EpP1UUehZlKVWZ0OZMEfRgKbRCel5abQKBgQDwgwrIQ5+BiZv6a0VT + ETeXB+hRbvBinRykNo/RvLc3j1enRh9/zO/ShadZIXgOAiM1Jnr5Gp8KkNGca6K1 + myhtad7xYPODYzNXXp6T1OPgZxHZLIYzVUj6ypXeV64Te5ZiDaJ1D49czsq+PqsQ + XRcgBJSNpFtDFiXWpjXWfx8PxwKBgQDhAnLY5Sl2eeQo+ud0MvjwftB/mN2qCzJY + 5AlQpRI4ThWxJgGPuHTR29zVa5iWNYuA5LWrC1y/wx+t5HKUwq+5kxvs+npYpDJD + ZX/w0Glc6s0Jc/mFySkbw9B2LePedL7lRF5OiAyC6D106Sc9V2jlL4IflmOzt4CD + ZTNbLtC6hwKBgHfIzBXxl/9sCcMuqdg1Ovp9dbcZCaATn7ApfHd5BccmHQGyav27 + k7XF2xMJGEHhzqcqAxUNrSgV+E9vTBomrHvRvrd5Ec7eGTPqbBA0d0nMC5eeFTh7 + wV0miH20LX6Gjt9G6yJiHYSbeV5G1+vOcTYBEft5X/qJjU7aePXbWh0BAoGBAJlV + 5tgCCuhvFloK6fHYzqZtdT6O+PfpW20SMXrgkvMF22h2YvgDFrDwqKRUB47NfHzg + 3yBpxNH1ccA5/w97QO8w3gX3h6qicpJVOAPusu6cIBACFZfjRv1hyszOZwvw+Soa + Fj5kHkqTY1YpkREPYS9V2dIW1Wjic1SXgZDw7VM/AoGAP/cZ3ZHTSCDTFlItqy5C + rIy2AiY0WJsx+K0qcvtosPOOwtnGjWHb1gdaVdfX/IRkSsX4PAOdnsyidNC5/l/m + y8oa+5WEeGFclWFhr4dnTA766o8HrM2UjIgWWYBF2VKdptGnHxFeJWFUmeQC/xeW + w37pCS7ykL+7gp7V0WShYsw= + -----END PRIVATE KEY----- diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml new file mode 100644 index 0000000..7ebc7d9 --- /dev/null +++ b/tests/templates/kuttl/smoke/10-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 720 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-s-1 --timeout 719s + - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-m-1 --timeout 719s diff --git a/tests/templates/kuttl/smoke/10-install-trino-catalogs.j2 b/tests/templates/kuttl/smoke/10-install-trino-catalogs.j2 new file mode 100644 index 0000000..00aa754 --- /dev/null +++ b/tests/templates/kuttl/smoke/10-install-trino-catalogs.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCatalog +metadata: + name: tpch + labels: + trino: trino +spec: + connector: + tpch: {} diff --git a/tests/templates/kuttl/smoke/10-install-trinos.yaml.j2 b/tests/templates/kuttl/smoke/10-install-trinos.yaml.j2 new file mode 100644 index 0000000..5fc62a6 --- /dev/null +++ b/tests/templates/kuttl/smoke/10-install-trinos.yaml.j2 @@ -0,0 +1,72 @@ +{% for name in ['trino-s-1', 'trino-m-1'] %} +--- +apiVersion: trino.stackable.tech/v1alpha1 +kind: TrinoCluster +metadata: + name: {{ name }} +spec: + image: +{% if test_scenario['values']['trino'].find(",") > 0 %} + custom: "{{ test_scenario['values']['trino'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['trino'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['trino'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + catalogLabelSelector: + matchLabels: + trino: trino + authentication: + - authenticationClass: trino-users-auth +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + coordinators: + config: + resources: + cpu: + min: 250m + max: "1" + memory: + limit: 2Gi + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + workers: + config: + resources: + cpu: + min: 250m + max: "1" + memory: + limit: 3Gi + gracefulShutdownTimeout: 60s # Let the test run faster + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +{% endfor %} +--- +apiVersion: authentication.stackable.tech/v1alpha1 +kind: AuthenticationClass +metadata: + name: trino-users-auth +spec: + provider: + static: + userCredentialsSecret: + name: trino-users +--- +apiVersion: v1 +kind: Secret +metadata: + name: trino-users +type: kubernetes.io/opaque +stringData: + admin: adminadmin + alice: alicealice + bob: bobbob diff --git a/tests/templates/kuttl/smoke/20-assert.yaml b/tests/templates/kuttl/smoke/20-assert.yaml new file mode 100644 index 0000000..61126f0 --- /dev/null +++ b/tests/templates/kuttl/smoke/20-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trino-lb +status: + replicas: 3 + readyReplicas: 3 diff --git a/tests/templates/kuttl/smoke/20-install-trino-lb.j2 b/tests/templates/kuttl/smoke/20-install-trino-lb.j2 new file mode 100644 index 0000000..a57517c --- /dev/null +++ b/tests/templates/kuttl/smoke/20-install-trino-lb.j2 @@ -0,0 +1,74 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl -n $NAMESPACE create secret generic trino-lb-config --from-file=trino-lb-config.yaml=20_trino-lb-config.yaml +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trino-lb +spec: + # I'm a bit surprised that 3 replicas work in combination with inMemory persistence :) + # Maybe Kubernetes is sticky for some reason, maybe trino-clients will retry call until they get + # to the correct trino-lb instance... + replicas: 3 + selector: + matchLabels: + app.kubernetes.io/name: trino-lb + template: + metadata: + labels: + app.kubernetes.io/name: trino-lb + spec: + containers: + - name: trino-lb + image: {{ test_scenario['values']['trino-lb'] }} + imagePullPolicy: IfNotPresent + command: ["trino-lb", "--config-file", "/etc/stackable/trino-lb/config/trino-lb-config.yaml"] + ports: + - containerPort: 8080 + - containerPort: 8443 + - containerPort: 9090 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 256Mi + volumeMounts: + - mountPath: /etc/stackable/trino-lb/config/ + name: config + - mountPath: /certificates/ + name: certificates + volumes: + - name: config + secret: + secretName: trino-lb-config + - name: certificates + secret: + secretName: trino-lb-certificates + serviceAccountName: trino-lb +--- +apiVersion: v1 +kind: Service +metadata: + name: trino-lb +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: trino-lb + ports: + - protocol: TCP + port: 8080 + targetPort: 8080 + name: http + - protocol: TCP + port: 8443 + targetPort: 8443 + name: https + - protocol: TCP + port: 9090 + targetPort: 9090 + name: metrics diff --git a/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2 b/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2 new file mode 100644 index 0000000..2b15508 --- /dev/null +++ b/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2 @@ -0,0 +1,22 @@ +trinoLb: + externalAddress: https://trino-lb:8443 + tls: + enabled: true + certPemFile: /certificates/cert.pem + keyPemFile: /certificates/key.pem + persistence: + inMemory: {} +trinoClusterGroups: + default: + maxRunningQueries: 1 + trinoClusters: + - name: trino-s-1 + endpoint: https://trino-s-1-coordinator:8443 + credentials: + username: admin + password: adminadmin +trinoClusterGroupsIgnoreCert: true + +# Route all queries to the "default" cluster group +routers: [] +routingFallback: default diff --git a/tests/templates/kuttl/smoke/30-assert.yaml b/tests/templates/kuttl/smoke/30-assert.yaml new file mode 100644 index 0000000..7f2cc57 --- /dev/null +++ b/tests/templates/kuttl/smoke/30-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-queries +status: + succeeded: 1 diff --git a/tests/templates/kuttl/smoke/30-test-queries.yaml.j2 b/tests/templates/kuttl/smoke/30-test-queries.yaml.j2 new file mode 100644 index 0000000..b335050 --- /dev/null +++ b/tests/templates/kuttl/smoke/30-test-queries.yaml.j2 @@ -0,0 +1,54 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-queries +spec: + template: + spec: + containers: + - name: test-queries + image: oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino'] }}-stackable0.0.0-dev + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c + - | + # Query Trinos and trino-lb + COORDINATORS=( + "https://trino-s-1-coordinator:8443" + "https://trino-m-1-coordinator:8443" + "https://trino-lb:8443" + ) + + export TRINO_USER="alice" + export TRINO_PASSWORD="alicealice" + QUERY="select count(*) from tpch.sf1.customer" + + for COORDINATOR in "${COORDINATORS[@]}"; do + echo "$QUERY" | java -jar trino-cli-executable.jar --server $COORDINATOR --insecure --user $TRINO_USER --password + done + + # Send multiple queries in parallel to trino-lb + NUM_REQUESTS=10 + TRINO_LB_ADDRESS="https://trino-lb:8443" + + pids=() + + for ((i = 1; i <= NUM_REQUESTS; i++)); do + echo "$QUERY" | java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user $TRINO_USER --password & + pids+=("$!") + done + + # Wait for all processes to complete and check exit codes + for pid in "${pids[@]}"; do + if ! wait "$pid"; then + echo "One of the requests failed with a non-zero exit code." + exit 1 + fi + done + + echo "All queries completed successfully." + restartPolicy: OnFailure diff --git a/tests/templates/kuttl/smoke/40-change-routing.j2 b/tests/templates/kuttl/smoke/40-change-routing.j2 new file mode 100644 index 0000000..5d08ea8 --- /dev/null +++ b/tests/templates/kuttl/smoke/40-change-routing.j2 @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl -n $NAMESPACE delete secret trino-lb-config || true + - script: kubectl -n $NAMESPACE create secret generic trino-lb-config --from-file=trino-lb-config.yaml=40_trino-lb-config.yaml diff --git a/tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2 b/tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2 new file mode 100644 index 0000000..8f81c64 --- /dev/null +++ b/tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2 @@ -0,0 +1,52 @@ +trinoLb: + externalAddress: https://trino-lb:8443 + tls: + enabled: true + certPemFile: /certificates/cert.pem + keyPemFile: /certificates/key.pem + persistence: + inMemory: {} +trinoClusterGroups: + s: + maxRunningQueries: 1 + trinoClusters: + - name: trino-s-1 + endpoint: https://trino-s-1-coordinator:8443 + credentials: + username: admin + password: adminadmin + m: + maxRunningQueries: 1 + trinoClusters: + - name: trino-m-1 + endpoint: https://trino-m-1-coordinator:8443 + credentials: + username: admin + password: adminadmin +trinoClusterGroupsIgnoreCert: true + +routers: + - trinoRoutingGroupHeader: + headerName: X-Trino-Routing-Group + - clientTags: + oneOf: ["s"] + trinoClusterGroup: s + - clientTags: + oneOf: ["m"] + trinoClusterGroup: m + - pythonScript: + script: | + from typing import Optional + + def targetClusterGroup(query: str, headers: dict[str, str]) -> Optional[str]: + user = get_user(headers) + if user == "alice": + return "s" + elif user == "bob": + return "m" + else: + return None + + def get_user(headers: dict[str, str]) -> Optional[str]: + return headers.get("x-trino-user") +routingFallback: s diff --git a/tests/templates/kuttl/smoke/41-restart-trino-lb.yaml b/tests/templates/kuttl/smoke/41-restart-trino-lb.yaml new file mode 100644 index 0000000..0e36a0b --- /dev/null +++ b/tests/templates/kuttl/smoke/41-restart-trino-lb.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl -n $NAMESPACE rollout restart deployment trino-lb + - script: kubectl -n $NAMESPACE rollout status deployment trino-lb --watch + - script: sleep 2 diff --git a/tests/templates/kuttl/smoke/50-assert.yaml b/tests/templates/kuttl/smoke/50-assert.yaml new file mode 100644 index 0000000..32690b9 --- /dev/null +++ b/tests/templates/kuttl/smoke/50-assert.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-routing +status: + succeeded: 1 diff --git a/tests/templates/kuttl/smoke/50-test-routing.yaml.j2 b/tests/templates/kuttl/smoke/50-test-routing.yaml.j2 new file mode 100644 index 0000000..deb69b1 --- /dev/null +++ b/tests/templates/kuttl/smoke/50-test-routing.yaml.j2 @@ -0,0 +1,34 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-routing +spec: + template: + spec: + containers: + - name: test-routing + image: oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino'] }}-stackable0.0.0-dev + command: + - /bin/bash + - -x + - -euo + - pipefail + - -c + - | + TRINO_LB_ADDRESS="https://trino-lb:8443" + COORDINATOR_NAME_QUERY="select regexp_extract(url_extract_host(http_uri), '[a-z0-9-]+') from "system".runtime.nodes where coordinator = true" + + # admin lands in s + echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=adminadmin java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user admin --password | grep -qx '"trino-s-1-coordinator-default-0"' || exit 1 + # alice lands in s + echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user alice --password | grep -qx '"trino-s-1-coordinator-default-0"' || exit 1 + # bob lands in m + echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=bobbob java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user bob --password | grep -qx '"trino-m-1-coordinator-default-0"' || exit 1 + + # We can also set client tags explicitly + echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=adminadmin java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user admin --password --client-tags=s | grep -qx '"trino-s-1-coordinator-default-0"' || exit 1 + echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=adminadmin java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user admin --password --client-tags=m | grep -qx '"trino-m-1-coordinator-default-0"' || exit 1 + + echo "All queries completed successfully." + restartPolicy: OnFailure diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml new file mode 100644 index 0000000..8210c0c --- /dev/null +++ b/tests/test-definition.yaml @@ -0,0 +1,32 @@ +--- +dimensions: + - name: trino-lb + values: + - oci.stackable.tech/stackable/trino-lb:0.5.0 + - name: trino + values: + - "451" + - "470" + - "476" + # To use a custom image, add a comma and the full name after the product version + # - 470,oci.stackable.tech/sdp/trino:470-stackable0.0.0-dev + # However, watch out, you need to tweak the trino-cli image + - name: trino-latest + values: + - "470" + # To use a custom image, add a comma and the full name after the product version + # - 470,oci.stackable.tech/sdp/trino:470-stackable0.0.0-dev + # However, watch out, you need to tweak the trino-cli image + - name: trino-lb-https + values: + - "true" + - "false" +tests: + - name: smoke + dimensions: + - trino-lb + - trino + +# TODOS +# 1. Test storage backend redis and postgres +# a. Also restart trino-lb deployment to make sure persistence is kept