From c4b51399083e6e212dfd1dedd191bc906687666a Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Thu, 27 Mar 2025 10:58:13 +0100
Subject: [PATCH 1/4] chore: Add smoke kuttl test

---
 scripts/run-tests                             | 409 ++++++++++++++++++
 scripts/run_tests.sh                          |   3 +
 tests/.gitignore                              |   1 +
 tests/kuttl-test.yaml.jinja2                  |  28 ++
 tests/release.yaml                            |  16 +
 tests/templates/kuttl/smoke/00-assert.yaml.j2 |  10 +
 ...tor-aggregator-discovery-configmap.yaml.j2 |   9 +
 .../templates/kuttl/smoke/00-patch-ns.yaml.j2 |   9 +
 tests/templates/kuttl/smoke/00-rbac.yaml.j2   |  39 ++
 .../kuttl/smoke/00-trino-lb-certificates.yaml |  60 +++
 tests/templates/kuttl/smoke/10-assert.yaml    |   6 +
 .../kuttl/smoke/10-install-trino-catalogs.j2  |  10 +
 .../kuttl/smoke/10-install-trino.yaml.j2      |  58 +++
 tests/templates/kuttl/smoke/20-assert.yaml    |   7 +
 .../kuttl/smoke/20-install-trino-lb.j2        |  71 +++
 .../kuttl/smoke/20_trino-lb-config.yaml.j2    |  22 +
 tests/templates/kuttl/smoke/30-assert.yaml    |   7 +
 .../kuttl/smoke/30-send-queries.yaml.j2       |  53 +++
 tests/test-definition.yaml                    |  36 ++
 19 files changed, 854 insertions(+)
 create mode 100755 scripts/run-tests
 create mode 100755 scripts/run_tests.sh
 create mode 100644 tests/.gitignore
 create mode 100644 tests/kuttl-test.yaml.jinja2
 create mode 100644 tests/release.yaml
 create mode 100644 tests/templates/kuttl/smoke/00-assert.yaml.j2
 create mode 100644 tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2
 create mode 100644 tests/templates/kuttl/smoke/00-patch-ns.yaml.j2
 create mode 100644 tests/templates/kuttl/smoke/00-rbac.yaml.j2
 create mode 100644 tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml
 create mode 100644 tests/templates/kuttl/smoke/10-assert.yaml
 create mode 100644 tests/templates/kuttl/smoke/10-install-trino-catalogs.j2
 create mode 100644 tests/templates/kuttl/smoke/10-install-trino.yaml.j2
 create mode 100644 tests/templates/kuttl/smoke/20-assert.yaml
 create mode 100644 tests/templates/kuttl/smoke/20-install-trino-lb.j2
 create mode 100644 tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2
 create mode 100644 tests/templates/kuttl/smoke/30-assert.yaml
 create mode 100644 tests/templates/kuttl/smoke/30-send-queries.yaml.j2
 create mode 100644 tests/test-definition.yaml

diff --git a/scripts/run-tests b/scripts/run-tests
new file mode 100755
index 0000000..4d13827
--- /dev/null
+++ b/scripts/run-tests
@@ -0,0 +1,409 @@
+#!/usr/bin/env python
+# vim: filetype=python syntax=python tabstop=4 expandtab
+
+# WARNING
+# No not modify this file, the source of truth is
+# https://github.com/stackabletech/operator-templating/blob/main/template/scripts/run-tests
+# WARNING
+
+import argparse
+import collections.abc
+import contextlib
+import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+
+__version__ = "0.0.1"
+
+DESCRIPTION = """
+Run integration tests. Call this script from the root of the repository.
+
+Exits with 0 on success, 1 on failure.
+
+Requires the following commands to be installed:
+* beku
+* stackablectl
+* kubectl
+* kubectl-kuttl
+
+Examples:
+
+1. Install operators, run all tests and clean up test namespaces:
+
+    ./scripts/run-tests --parallel 4
+
+2. Install operators but for Airflow use version "0.0.0-pr123" instead of "0.0.0-dev" and run all tests as above:
+
+    ./scripts/run-tests --operator airflow=0.0.0-pr123 --parallel 4
+
+3. Do not install any operators, run the smoke test suite and keep namespace:
+
+    ./scripts/run-tests --skip-release --skip-delete --test-suite smoke-latest
+
+4. Run the ldap test(s) from the openshift test suite and keep namespace:
+
+    ./scripts/run-tests --skip-release --skip-delete --test-suite openshift --test ldap
+
+5. Run the smoke test suite in the namespace "smoke". The namespace will be
+   created if it doesn't exist and will not be deleted when the tests end.
+
+    ./scripts/run-tests --test-suite smoke-latest --namespace smoke
+"""
+
+
+class TestRunnerException(Exception):
+    pass
+
+
+def parse_args(argv: list[str]) -> argparse.Namespace:
+    """Parse command line args."""
+    parser = argparse.ArgumentParser(
+        description=DESCRIPTION, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    parser.add_argument(
+        "--version",
+        help="Display application version",
+        action="version",
+        version=f"%(prog)s {__version__}",
+    )
+
+    parser.add_argument(
+        "--skip-delete",
+        help="Do not delete test namespaces.",
+        action="store_true",
+    )
+
+    parser.add_argument(
+        "--skip-tests",
+        help="Do not actually run the tests.",
+        action="store_true",
+    )
+
+    parser.add_argument(
+        "--skip-release",
+        help="Do not install operators.",
+        action="store_true",
+    )
+
+    parser.add_argument(
+        "--parallel",
+        help="How many tests to run in parallel. Default 2.",
+        type=int,
+        required=False,
+        default=2,
+    )
+
+    parser.add_argument(
+        "--operator",
+        help="Patch operator version in release.yaml. Format <operator>=<version>",
+        action="append",
+        type=cli_parse_operator_args,
+        default=[],
+    )
+
+    parser.add_argument(
+        "--skip-operator",
+        help="Skip given operator(s) when installing a release.",
+        action="append",
+        default=[],
+    )
+
+    parser.add_argument(
+        "--test",
+        help="Kuttl test to run.",
+        type=str,
+        required=False,
+    )
+
+    parser.add_argument(
+        "--test-suite",
+        help="Name of the test suite to expand. Default: default",
+        type=str,
+        required=False,
+    )
+
+    parser.add_argument(
+        "--log-level",
+        help="Set log level.",
+        type=cli_log_level,
+        required=False,
+        default=logging.INFO,
+    )
+
+    parser.add_argument(
+        "--namespace",
+        help="Namespace to run the tests in. It will be created if it doesn't already exist.",
+        type=str,
+        required=False,
+    )
+
+    return parser.parse_args(argv)
+
+
+def cli_parse_operator_args(args: str) -> tuple[str, str]:
+    if "=" not in args:
+        raise argparse.ArgumentTypeError(
+            f"Invalid operator argument: {args}. Must be in format <operator>=<version>"
+        )
+    op, version = args.split("=", maxsplit=1)
+    return op, version
+
+
+def cli_log_level(cli_arg: str) -> int:
+    match cli_arg:
+        case "debug":
+            return logging.DEBUG
+        case "info":
+            return logging.INFO
+        case "error":
+            return logging.ERROR
+        case "warning":
+            return logging.WARNING
+        case "critical":
+            return logging.CRITICAL
+        case _:
+            raise argparse.ArgumentTypeError("Invalid log level")
+
+
+def have_requirements() -> None:
+    commands = [
+        ("beku", "https://github.com/stackabletech/beku.py"),
+        (
+            "stackablectl",
+            "https://github.com/stackabletech/stackable-cockpit/blob/main/rust/stackablectl/README.md",
+        ),
+        ("kubectl", "https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/"),
+        ("kubectl-kuttl", "https://kuttl.dev/"),
+    ]
+
+    err = False
+    for command, url in commands:
+        if not shutil.which(command):
+            logging.error(f'Command "{command}" not found, please install from {url}')
+            err = True
+    if err:
+        raise TestRunnerException()
+
+
+@contextlib.contextmanager
+def release_file(
+    operators: list[tuple[str, str]], skip_ops: list[str]
+) -> collections.abc.Generator[str, None, None]:
+    """Generate a (possibly modified) copy of the release.yaml file.
+
+    Operator versions passed as --operator take precedence over the release.yaml contents.
+
+    Operators passed as --skip-operator are excluded from the resulting release.yaml contents.
+
+    If an invalid operator name is provided (i.e. one that doesn't exist in the
+    original release file), a TestRunnerException is raised.
+
+    Yields the name of the (potentially patched) release file. This is a temporary
+    file that will be deleted when the context manager exits.
+    """
+
+    def _patch():
+        release_file = os.path.join("tests", "release.yaml")
+        # A marker to validate that all ops were patched
+        patched_release = []
+        with open(release_file, "r") as f:
+            patched_ops = []
+            patch_version = ""
+            for line in f:
+                if patch_version:
+                    line = re.sub(":.+$", f": {patch_version}", line)
+                    patch_version = ""
+                else:
+                    for op, version in operators:
+                        if op in line:
+                            patch_version = version
+                            patched_ops.append(op)
+                            break
+                patched_release.append(line.rstrip("\n"))
+
+            # Sanity test that cli didn't contain garbage that is silently discarded
+            ops_not_patched = set([op for op, _ in operators]) - set(patched_ops)
+            if ops_not_patched:
+                logging.error(
+                    f"Patched operators [{', '.join(ops_not_patched)}] not found in {release_file}"
+                )
+                raise TestRunnerException()
+
+        # Filter out skip operators
+        release_contents = []
+        skip_lines = 0
+        valid_skip_ops = []
+        for line in patched_release:
+            if skip_lines:
+                skip_lines -= 1
+                continue
+            for op in skip_ops:
+                if op in line:
+                    # Every product section has 1 line of additional config to skip
+                    skip_lines = 1
+                    valid_skip_ops.append(op)
+                    break
+            else:
+                release_contents.append(line)
+        # Sanity test that cli didn't contain garbage that is silently discarded
+        ops_not_skipped = set(skip_ops) - set(valid_skip_ops)
+        if ops_not_skipped:
+            logging.error(
+                f"Skipped operators [{', '.join(ops_not_skipped)}] not found in {release_file}"
+            )
+            raise TestRunnerException()
+
+        with tempfile.NamedTemporaryFile(
+            mode="w",
+            delete=False,
+            prefix="patched",
+        ) as f:
+            pcontents = "\n".join(release_contents)
+            logging.debug(f"Writing patched release to {f.name}: {pcontents}\n")
+            f.write(pcontents)
+            return f.name
+
+    release_file = _patch()
+    try:
+        yield release_file
+    except TestRunnerException as e:
+        logging.error(f"Caught exception: {e}")
+        raise
+    finally:
+        if "patched" in release_file:
+            try:
+                logging.debug(f"Removing patched release file : {release_file}")
+                os.remove(release_file)
+            except FileNotFoundError | OSError:
+                logging.error(f"Failed to delete patched release file: {release_file}")
+
+
+def maybe_install_release(skip_release: bool, release_file: str) -> None:
+    if skip_release:
+        logging.debug("Skip release installation")
+        return
+    stackablectl_err = ""
+    try:
+        stackablectl_cmd = [
+            "stackablectl",
+            "release",
+            "install",
+            "--release-file",
+            release_file,
+            "tests",
+        ]
+        logging.debug(f"Running : {stackablectl_cmd}")
+
+        completed_proc = subprocess.run(
+            stackablectl_cmd,
+            capture_output=True,
+            check=True,
+        )
+        # stackablectl doesn't return a non-zero exit code on failure
+        # so we need to check stderr for errors
+        stackablectl_err = completed_proc.stderr.decode("utf-8")
+        if "error" in stackablectl_err.lower():
+            logging.error(stackablectl_err)
+            logging.error("stackablectl failed")
+            raise TestRunnerException()
+
+    except subprocess.CalledProcessError as e:
+        # in case stackablectl starts returning non-zero exit codes
+        logging.error(e.stderr.decode("utf-8"))
+        logging.error("stackablectl failed")
+        raise TestRunnerException()
+
+
+def gen_tests(test_suite: str, namespace: str) -> None:
+    try:
+        beku_cmd = [
+            "beku",
+            "--test_definition",
+            os.path.join("tests", "test-definition.yaml"),
+            "--kuttl_test",
+            os.path.join("tests", "kuttl-test.yaml.jinja2"),
+            "--template_dir",
+            os.path.join("tests", "templates", "kuttl"),
+            "--output_dir",
+            os.path.join("tests", "_work"),
+        ]
+        if test_suite:
+            beku_cmd.extend(["--suite", test_suite])
+        if namespace:
+            beku_cmd.extend(["--namespace", namespace])
+
+        logging.debug(f"Running : {beku_cmd}")
+        subprocess.run(
+            beku_cmd,
+            check=True,
+        )
+    except subprocess.CalledProcessError:
+        logging.error("beku failed")
+        raise TestRunnerException()
+
+
+def run_tests(test: str, parallel: int, namespace: str, skip_delete: bool) -> None:
+    try:
+        kuttl_cmd = ["kubectl-kuttl", "test"]
+        if test:
+            kuttl_cmd.extend(["--test", test])
+        if parallel:
+            kuttl_cmd.extend(["--parallel", str(parallel)])
+        if skip_delete:
+            kuttl_cmd.extend(["--skip-delete"])
+        if namespace:
+            kuttl_cmd.extend(["--namespace", namespace])
+            # kuttl doesn't create the namespace so we need to do it ourselves
+            create_ns_cmd = ["kubectl", "create", "namespace", namespace]
+            try:
+                logging.debug(f"Running : {create_ns_cmd}")
+                subprocess.run(
+                    create_ns_cmd,
+                    check=True,
+                    capture_output=True,
+                )
+            except subprocess.CalledProcessError as e:
+                stderr = e.stderr.decode("utf-8")
+                # If the namespace already exists, this will fail and we ignore the
+                # error. If it fails for any other reason, we raise an exception.
+                if "already exists" not in stderr:
+                    logging.error(stderr)
+                    logging.error("namespace creation failed")
+                    raise TestRunnerException()
+
+        logging.debug(f"Running : {kuttl_cmd}")
+
+        subprocess.run(
+            kuttl_cmd,
+            cwd="tests/_work",
+            check=True,
+        )
+    except subprocess.CalledProcessError:
+        logging.error("kuttl failed")
+        raise TestRunnerException()
+
+
+def main(argv) -> int:
+    ret = 0
+    try:
+        opts = parse_args(argv[1:])
+        logging.basicConfig(encoding="utf-8", level=opts.log_level)
+        have_requirements()
+        gen_tests(opts.test_suite, opts.namespace)
+        with release_file(opts.operator, opts.skip_operator) as f:
+            maybe_install_release(opts.skip_release, f)
+        if opts.skip_tests:
+            logging.info("Skip running tests.")
+        else:
+            run_tests(opts.test, opts.parallel, opts.namespace, opts.skip_delete)
+    except TestRunnerException:
+        ret = 1
+    return ret
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))
diff --git a/scripts/run_tests.sh b/scripts/run_tests.sh
new file mode 100755
index 0000000..a31a138
--- /dev/null
+++ b/scripts/run_tests.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+./scripts/run-tests "$@"
diff --git a/tests/.gitignore b/tests/.gitignore
new file mode 100644
index 0000000..e96b7d6
--- /dev/null
+++ b/tests/.gitignore
@@ -0,0 +1 @@
+_work/
diff --git a/tests/kuttl-test.yaml.jinja2 b/tests/kuttl-test.yaml.jinja2
new file mode 100644
index 0000000..8a5620c
--- /dev/null
+++ b/tests/kuttl-test.yaml.jinja2
@@ -0,0 +1,28 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestSuite
+testDirs:
+  {% for testcase in testinput.tests %}
+  - ./tests/{{ testcase.name }}
+  {% endfor %}
+
+startKIND: false
+suppress: ["events"]
+parallel: 2
+
+# The timeout (in seconds) is used when namespaces are created or
+# deleted, and, if not overridden, in TestSteps, TestAsserts, and
+# Commands. If not set, the timeout is 30 seconds by default.
+#
+# The deletion of a namespace can take a while until all resources,
+# especially PersistentVolumeClaims, are gracefully shut down. If the
+# timeout is reached in the meantime, even a successful test case is
+# considered a failure.
+#
+# For instance, the termination grace period of the Vector aggregator in
+# the logging tests is set to 60 seconds. If there are logs entries
+# which could not be forwarded yet to the external aggregator defined in
+# the VECTOR_AGGREGATOR environment variable, then the test aggregator
+# uses this period of time by trying to forward the events. In this
+# case, deleting a namespace with several Pods takes about 90 seconds.
+timeout: 300
diff --git a/tests/release.yaml b/tests/release.yaml
new file mode 100644
index 0000000..4093b6e
--- /dev/null
+++ b/tests/release.yaml
@@ -0,0 +1,16 @@
+# Contains all operators required to run the test suite.
+---
+releases:
+  # Do not change the name of the release as it's referenced from run-tests
+  tests:
+    releaseDate: 1970-01-01
+    description: Integration test
+    products:
+      commons:
+        operatorVersion: 0.0.0-dev
+      secret:
+        operatorVersion: 0.0.0-dev
+      listener:
+        operatorVersion: 0.0.0-dev
+      trino:
+        operatorVersion: 0.0.0-dev
diff --git a/tests/templates/kuttl/smoke/00-assert.yaml.j2 b/tests/templates/kuttl/smoke/00-assert.yaml.j2
new file mode 100644
index 0000000..50b1d4c
--- /dev/null
+++ b/tests/templates/kuttl/smoke/00-assert.yaml.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+{% endif %}
diff --git a/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2
new file mode 100644
index 0000000..2d6a0df
--- /dev/null
+++ b/tests/templates/kuttl/smoke/00-install-vector-aggregator-discovery-configmap.yaml.j2
@@ -0,0 +1,9 @@
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vector-aggregator-discovery
+data:
+  ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }}
+{% endif %}
diff --git a/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2 b/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2
new file mode 100644
index 0000000..67185ac
--- /dev/null
+++ b/tests/templates/kuttl/smoke/00-patch-ns.yaml.j2
@@ -0,0 +1,9 @@
+{% if test_scenario['values']['openshift'] == 'true' %}
+# see https://github.com/stackabletech/issues/issues/566
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}'
+    timeout: 120
+{% endif %}
diff --git a/tests/templates/kuttl/smoke/00-rbac.yaml.j2 b/tests/templates/kuttl/smoke/00-rbac.yaml.j2
new file mode 100644
index 0000000..4abbf4f
--- /dev/null
+++ b/tests/templates/kuttl/smoke/00-rbac.yaml.j2
@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: trino-lb
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: trino-lb
+subjects:
+  - kind: ServiceAccount
+    name: trino-lb
+roleRef:
+  kind: Role
+  name: trino-lb
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: trino-lb
+  labels:
+    app.kubernetes.io/name: trino-lb
+rules:
+  - apiGroups:
+      - trino.stackable.tech
+    resources:
+      - trinoclusters
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+  - apiGroups:
+      - trino.stackable.tech
+    resources:
+      - trinoclusters/status
+    verbs:
+      - get
diff --git a/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml b/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml
new file mode 100644
index 0000000..473c5d0
--- /dev/null
+++ b/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml
@@ -0,0 +1,60 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: trino-lb-certificates
+data:
+  cert.pem: |
+    LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURrekNDQW51Z0F3SUJBZ0lVWFZZa1JDck0v
+    Z2UwM0RWeW1EdFhDdXlicDdnd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1dURUxNQWtHQTFVRUJoTUNW
+    Vk14RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeElUQWZCZ05WQkFvTQpHRWx1ZEdWeWJtVjBJ
+    RmRwWkdkcGRITWdVSFI1SUV4MFpERVNNQkFHQTFVRUF3d0piRzlqWVd4b2IzTjBNQjRYCkRUSXhN
+    RGN6TVRFME1qSXhNbG9YRFRJeU1EY3pNVEUwTWpJeE1sb3dXVEVMTUFrR0ExVUVCaE1DVlZNeEV6
+    QVIKQmdOVkJBZ01DbE52YldVdFUzUmhkR1V4SVRBZkJnTlZCQW9NR0VsdWRHVnlibVYwSUZkcFpH
+    ZHBkSE1nVUhSNQpJRXgwWkRFU01CQUdBMVVFQXd3SmJHOWpZV3hvYjNOME1JSUJJakFOQmdrcWhr
+    aUc5dzBCQVFFRkFBT0NBUThBCk1JSUJDZ0tDQVFFQTAyVjVaam1xTEIvVlF3VGFycnovMzVxc2E4
+    M0wrRGJBb2EwMDAxK2pWbW1DK0c5TnVmaTAKZGFyb0ZXai9VaWN2MmZaV0VUVThKb1pLVXJYNEJL
+    OW9nNWNnNXJsbi9DdEJSV0NVWUl3UmdZOVIvQ2RCR1BuNAprcCtYa1NKYUN3NzRaSXlMeS9aZnV4
+    Nmg4RVMxbTlZUm5CemErczdVK0ltUkJSZjRNUlB0WFEzL21xSnhBWllxCmRPbktudnNzUnlEMnF1
+    dGdWVEF4d01VdkpXSWl2UmhSWURqN1dPcFM0Q0VFZVF4UDFpSDEvVDVQN0ZkdFRHZFQKYlZCQUJD
+    QThKaEw5NnVGR1BwT1lIY00vN1I1RUlBM3laNUZOZzkzMVF6b0RJVGp0WEd0UTZ5OS9sL0lZa1dt
+    NgpKNjdSV2NOMElvVHNaaHowV05VNGdBZXNsVnRKTG9mbjhRSURBUUFCbzFNd1VUQWRCZ05WSFE0
+    RUZnUVV6Rm5LCk5mUzRMQVl1S2VXd0hiem9vRVIweVowd0h3WURWUjBqQkJnd0ZvQVV6Rm5LTmZT
+    NExBWXVLZVd3SGJ6b29FUjAKeVowd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBTkJna3Foa2lHOXcw
+    QkFRc0ZBQU9DQVFFQWs0TytlOWppYTU5Vwpad2V0TjRHVTdPV2NZaG1PZ1NpelJTczZ1N21UZnA2
+    MkxETXQ5NldLVTNUSGtzT25aNDRIbnFXUXhzU2ZkRlZVClhKRDEydGp2VlU4WjRGV3pRYWpjSGVl
+    bVVZaUR6ZThFQWg2VG54blVjT3JVOEljd2lLR3hDV1JZLzkwOGpuV2cKK01Nc2NmTUNNWVRSZGVU
+    UHFEOGZHekFsVUN0bXl6SDZLTEUzczRPby9yNStOUitVdnJ3cGR2Yjd4ZTBNd3dPOQpRL3pSNE44
+    ZXAvSHdIVkVPYmNhQm9mRTFzc1pMa3NYN1pnQ1A5d01nWFJXcE5BdEM1RVd4TWJ4WWpCZldGSDI0
+    CmZESmxCTWlHSldnOEhIY3hLN3dRaEZoK2Z1eU56RSt4RVdQc0k5VkwxekRmdGQ5eDgvUXNPYWd5
+    RU9uWThWeHIKQW9wdlowOXVFUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+  key.pem: |
+    LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZB
+    QVNDQktjd2dnU2pBZ0VBQW9JQkFRRFRaWGxtT2Fvc0g5VkQKQk5xdXZQL2ZtcXhyemN2NE5zQ2hy
+    VFRUWDZOV2FZTDRiMDI1K0xSMXF1Z1ZhUDlTSnkvWjlsWVJOVHdtaGtwUwp0ZmdFcjJpRGx5RG11
+    V2Y4SzBGRllKUmdqQkdCajFIOEowRVkrZmlTbjVlUklsb0xEdmhrakl2TDlsKzdIcUh3ClJMV2Ix
+    aEdjSE5yNnp0VDRpWkVGRi9neEUrMWREZithb25FQmxpcDA2Y3FlK3l4SElQYXE2MkJWTURIQXhT
+    OGwKWWlLOUdGRmdPUHRZNmxMZ0lRUjVERS9XSWZYOVBrL3NWMjFNWjFOdFVFQUVJRHdtRXYzcTRV
+    WStrNWdkd3ovdApIa1FnRGZKbmtVMkQzZlZET2dNaE9PMWNhMURyTDMrWDhoaVJhYm9ucnRGWncz
+    UWloT3htSFBSWTFUaUFCNnlWClcwa3VoK2Z4QWdNQkFBRUNnZ0VBRGx0dThrMXFURkxoSmdzWFd4
+    VEZBQWUrUEJnZkNUMld1YVJNMlNvK3FxakIKMTJPZjBNaWVZUHQ1aGJLNjNIYUMzbmZIZ3FXdDd5
+    UGh1bHBYZk9INDVDOEljZ01YbDkzTU1nME1KcjU4bGVNSQorMm9qRnJJcmVySFNGbTVSMVR4d0RF
+    d3JWbS9tTW93ekRXRnRRQ2M2elBKOHdObjVSdVA0OEhLZlRaMy8yZmp3CnpFalN3UE8yd0ZNZm8x
+    RUpOVGpsSTMwM2xGYmRGQnM2N05hWDZwdWgzME03VG4rZ3puSEt5TzVhN0Y1N3drSXQKZmtnbkV5
+    L3NnTWVkUWx3WDdiUnBVb0Q2ZjBmWnpWOFF6NGNIRnl3dFlFcmN6WkpoM1ZHaXRKb08vVkNJRGR0
+    eQpSUFhPQXFWRGQ3RXBQMVVVZWhabEtWV1owT1pNRWZSZ0tiUkNlbDVhYlFLQmdRRHdnd3JJUTUr
+    QmladjZhMFZUCkVUZVhCK2hSYnZCaW5SeWtOby9SdkxjM2oxZW5SaDkvek8vU2hhZFpJWGdPQWlN
+    MUpucjVHcDhLa05HY2E2SzEKbXlodGFkN3hZUE9EWXpOWFhwNlQxT1BnWnhIWkxJWXpWVWo2eXBY
+    ZVY2NFRlNVppRGFKMUQ0OWN6c3ErUHFzUQpYUmNnQkpTTnBGdERGaVhXcGpYV2Z4OFB4d0tCZ1FE
+    aEFuTFk1U2wyZWVRbyt1ZDBNdmp3ZnRCL21OMnFDekpZCjVBbFFwUkk0VGhXeEpnR1B1SFRSMjl6
+    VmE1aVdOWXVBNUxXckMxeS93eCt0NUhLVXdxKzVreHZzK25wWXBESkQKWlgvdzBHbGM2czBKYy9t
+    RnlTa2J3OUIyTGVQZWRMN2xSRjVPaUF5QzZEMTA2U2M5VjJqbEw0SWZsbU96dDRDRApaVE5iTHRD
+    Nmh3S0JnSGZJekJYeGwvOXNDY011cWRnMU92cDlkYmNaQ2FBVG43QXBmSGQ1QmNjbUhRR3lhdjI3
+    Cms3WEYyeE1KR0VIaHpxY3FBeFVOclNnVitFOXZUQm9tckh2UnZyZDVFYzdlR1RQcWJCQTBkMG5N
+    QzVlZUZUaDcKd1YwbWlIMjBMWDZHanQ5RzZ5SmlIWVNiZVY1RzErdk9jVFlCRWZ0NVgvcUpqVTdh
+    ZVBYYldoMEJBb0dCQUpsVgo1dGdDQ3VodkZsb0s2ZkhZenFadGRUNk8rUGZwVzIwU01Ycmdrdk1G
+    MjJoMll2Z0RGckR3cUtSVUI0N05mSHpnCjN5QnB4TkgxY2NBNS93OTdRTzh3M2dYM2g2cWljcEpW
+    T0FQdXN1NmNJQkFDRlpmalJ2MWh5c3pPWnd2dytTb2EKRmo1a0hrcVRZMVlwa1JFUFlTOVYyZElX
+    MVdqaWMxU1hnWkR3N1ZNL0FvR0FQL2NaM1pIVFNDRFRGbEl0cXk1QwpySXkyQWlZMFdKc3grSzBx
+    Y3Z0b3NQT093dG5HaldIYjFnZGFWZGZYL0lSa1NzWDRQQU9kbnN5aWROQzUvbC9tCnk4b2ErNVdF
+    ZUdGY2xXRmhyNGRuVEE3NjZvOEhyTTJVaklnV1dZQkYyVktkcHRHbkh4RmVKV0ZVbWVRQy94ZVcK
+    dzM3cENTN3lrTCs3Z3A3VjBXU2hZc3c9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml
new file mode 100644
index 0000000..ad2cfd1
--- /dev/null
+++ b/tests/templates/kuttl/smoke/10-assert.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 720
+commands:
+  - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-default-1 --timeout 719s
diff --git a/tests/templates/kuttl/smoke/10-install-trino-catalogs.j2 b/tests/templates/kuttl/smoke/10-install-trino-catalogs.j2
new file mode 100644
index 0000000..00aa754
--- /dev/null
+++ b/tests/templates/kuttl/smoke/10-install-trino-catalogs.j2
@@ -0,0 +1,10 @@
+---
+apiVersion: trino.stackable.tech/v1alpha1
+kind: TrinoCatalog
+metadata:
+  name: tpch
+  labels:
+    trino: trino
+spec:
+  connector:
+    tpch: {}
diff --git a/tests/templates/kuttl/smoke/10-install-trino.yaml.j2 b/tests/templates/kuttl/smoke/10-install-trino.yaml.j2
new file mode 100644
index 0000000..7efb961
--- /dev/null
+++ b/tests/templates/kuttl/smoke/10-install-trino.yaml.j2
@@ -0,0 +1,58 @@
+---
+apiVersion: trino.stackable.tech/v1alpha1
+kind: TrinoCluster
+metadata:
+  name: trino-default-1
+spec:
+  image:
+{% if test_scenario['values']['trino'].find(",") > 0 %}
+    custom: "{{ test_scenario['values']['trino'].split(',')[1] }}"
+    productVersion: "{{ test_scenario['values']['trino'].split(',')[0] }}"
+{% else %}
+    productVersion: "{{ test_scenario['values']['trino'] }}"
+{% endif %}
+    pullPolicy: IfNotPresent
+  clusterConfig:
+    catalogLabelSelector:
+      matchLabels:
+        trino: trino
+    authentication:
+      - authenticationClass: trino-users-auth
+{% if lookup('env', 'VECTOR_AGGREGATOR') %}
+    vectorAggregatorConfigMapName: vector-aggregator-discovery
+{% endif %}
+  coordinators:
+    config:
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 1
+  workers:
+    config:
+      gracefulShutdownTimeout: 60s # Let the test run faster
+      logging:
+        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
+    roleGroups:
+      default:
+        replicas: 1
+---
+apiVersion: authentication.stackable.tech/v1alpha1
+kind: AuthenticationClass
+metadata:
+  name: trino-users-auth
+spec:
+  provider:
+    static:
+      userCredentialsSecret:
+        name: trino-users
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: trino-users
+type: kubernetes.io/opaque
+stringData:
+  admin: adminadmin
+  alice: alicealice
+  bob: bobbob
diff --git a/tests/templates/kuttl/smoke/20-assert.yaml b/tests/templates/kuttl/smoke/20-assert.yaml
new file mode 100644
index 0000000..61126f0
--- /dev/null
+++ b/tests/templates/kuttl/smoke/20-assert.yaml
@@ -0,0 +1,7 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trino-lb
+status:
+  replicas: 3
+  readyReplicas: 3
diff --git a/tests/templates/kuttl/smoke/20-install-trino-lb.j2 b/tests/templates/kuttl/smoke/20-install-trino-lb.j2
new file mode 100644
index 0000000..8736dc2
--- /dev/null
+++ b/tests/templates/kuttl/smoke/20-install-trino-lb.j2
@@ -0,0 +1,71 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl -n $NAMESPACE create secret generic trino-lb-config --from-file=trino-lb-config.yaml=20_trino-lb-config.yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trino-lb
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: trino-lb
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: trino-lb
+    spec:
+      containers:
+        - name: trino-lb
+          image: {{ test_scenario['values']['trino-lb'] }}
+          imagePullPolicy: IfNotPresent
+          command: ["trino-lb", "--config-file", "/etc/stackable/trino-lb/config/trino-lb-config.yaml"]
+          ports:
+            - containerPort: 8080
+            - containerPort: 8443
+            - containerPort: 9090
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 500m
+              memory: 256Mi
+          volumeMounts:
+            - mountPath: /etc/stackable/trino-lb/config/
+              name: config
+            - mountPath: /certificates/
+              name: certificates
+      volumes:
+        - name: config
+          secret:
+            secretName: trino-lb-config
+        - name: certificates
+          secret:
+            secretName: trino-lb-certificates
+      serviceAccountName: trino-lb
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: trino-lb
+spec:
+  type: ClusterIP
+  selector:
+    app.kubernetes.io/name: trino-lb
+  ports:
+    - protocol: TCP
+      port: 8080
+      targetPort: 8080
+      name: http
+    - protocol: TCP
+      port: 8443
+      targetPort: 8443
+      name: https
+    - protocol: TCP
+      port: 9090
+      targetPort: 9090
+      name: metrics
diff --git a/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2 b/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2
new file mode 100644
index 0000000..03e943f
--- /dev/null
+++ b/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2
@@ -0,0 +1,22 @@
+trinoLb:
+  externalAddress: https://trino-lb:8443
+  tls:
+    enabled: true
+    certPemFile: /certificates/cert.pem
+    keyPemFile: /certificates/key.pem
+  persistence:
+    inMemory: {}
+trinoClusterGroups:
+  default:
+    maxRunningQueries: 1
+    trinoClusters:
+      - name: trino-default-1
+        endpoint: https://trino-default-1-coordinator:8443
+        credentials:
+          username: admin
+          password: adminadmin
+trinoClusterGroupsIgnoreCert: true
+
+# Route all queries to the "default" cluster group
+routers: []
+routingFallback: default
diff --git a/tests/templates/kuttl/smoke/30-assert.yaml b/tests/templates/kuttl/smoke/30-assert.yaml
new file mode 100644
index 0000000..9f2d6e2
--- /dev/null
+++ b/tests/templates/kuttl/smoke/30-assert.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: send-queries
+status:
+  succeeded: 1
diff --git a/tests/templates/kuttl/smoke/30-send-queries.yaml.j2 b/tests/templates/kuttl/smoke/30-send-queries.yaml.j2
new file mode 100644
index 0000000..a6d74b1
--- /dev/null
+++ b/tests/templates/kuttl/smoke/30-send-queries.yaml.j2
@@ -0,0 +1,53 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: send-queries
+spec:
+  template:
+    spec:
+      containers:
+        - name: send-queries
+          image: oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino'] }}-stackable0.0.0-dev
+          command:
+            - /bin/bash
+            - -x
+            - -euo
+            - pipefail
+            - -c
+            - |
+              # Query Trino directly
+              echo "show catalogs" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server https://trino-default-1-coordinator:8443 --insecure --user alice --password
+              echo "select count(*) from tpch.sf1.customer" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server https://trino-default-1-coordinator:8443 --insecure --user alice --password
+
+              # Query trino-lb
+              echo "show catalogs" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server https://trino-lb:8443 --insecure --user alice --password
+              echo "select count(*) from tpch.sf1.customer" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server https://trino-lb:8443 --insecure --user alice --password
+
+              # Send multiple queries in parallel to trino-lb
+              # Number of parallel requests
+              NUM_REQUESTS=10
+
+              # Command to run the query
+              QUERY_COMMAND='echo "select count(*) from tpch.sf1.customer" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server https://trino-lb:8443 --insecure --user alice --password'
+
+              # Array to store process IDs (PIDs)
+              pids=()
+
+              # Run requests in parallel
+              for i in $(seq 1 $NUM_REQUESTS); do
+                  bash -c "$QUERY_COMMAND" &
+                  pids+=("$!")
+              done
+
+              # Wait for all processes to complete and check exit codes
+              for pid in "${pids[@]}"; do
+                  wait "$pid"
+                  if [ $? -ne 0 ]; then
+                      echo "One of the requests failed with a non-zero exit code."
+                      exit 1
+                  fi
+              done
+
+              echo "All requests completed successfully."
+      restartPolicy: OnFailure
diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
new file mode 100644
index 0000000..e9a7e32
--- /dev/null
+++ b/tests/test-definition.yaml
@@ -0,0 +1,36 @@
+---
+dimensions:
+  - name: trino-lb
+    values:
+      - oci.stackable.tech/stackable/trino-lb:0.5.0
+  - name: trino
+    values:
+      - "451"
+      - "455"
+      - "470"
+      # To use a custom image, add a comma and the full name after the product version
+      # - 470,oci.stackable.tech/sdp/trino:470-stackable0.0.0-dev
+      # However, watch out, you need to tweak the trino-cli image
+  - name: trino-latest
+    values:
+      - "470"
+      # To use a custom image, add a comma and the full name after the product version
+      # - 470,oci.stackable.tech/sdp/trino:470-stackable0.0.0-dev
+      # However, watch out, you need to tweak the trino-cli image
+  - name: keycloak
+    values:
+      - 25.0.0
+  - name: trino-lb-https
+    values:
+      - "true"
+      - "false"
+tests:
+  - name: smoke
+    dimensions:
+      - trino-lb
+      - trino
+      - trino-use-authentication
+  # - name: oidc
+  #   dimensions:
+  #     - trino-latest
+  #     - keycloak

From 0dc5b0c2d4b2f97f51827142adfd421d19ca025f Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Thu, 27 Mar 2025 12:59:25 +0100
Subject: [PATCH 2/4] Add kuttl test for routing

---
 tests/templates/kuttl/smoke/10-assert.yaml    |  3 +-
 ...rino.yaml.j2 => 10-install-trinos.yaml.j2} | 16 +++++-
 .../kuttl/smoke/20-install-trino-lb.j2        |  3 ++
 .../kuttl/smoke/20_trino-lb-config.yaml.j2    |  4 +-
 tests/templates/kuttl/smoke/30-assert.yaml    |  2 +-
 .../kuttl/smoke/30-send-queries.yaml.j2       | 53 ------------------
 .../kuttl/smoke/30-test-queries.yaml.j2       | 54 +++++++++++++++++++
 .../kuttl/smoke/40-change-routing.j2          |  6 +++
 .../kuttl/smoke/40_trino-lb-config.yaml.j2    | 52 ++++++++++++++++++
 .../kuttl/smoke/41-restart-trino-lb.yaml      |  7 +++
 tests/templates/kuttl/smoke/50-assert.yaml    |  7 +++
 .../kuttl/smoke/50-test-routing.yaml.j2       | 34 ++++++++++++
 tests/test-definition.yaml                    | 11 ++--
 13 files changed, 187 insertions(+), 65 deletions(-)
 rename tests/templates/kuttl/smoke/{10-install-trino.yaml.j2 => 10-install-trinos.yaml.j2} (83%)
 delete mode 100644 tests/templates/kuttl/smoke/30-send-queries.yaml.j2
 create mode 100644 tests/templates/kuttl/smoke/30-test-queries.yaml.j2
 create mode 100644 tests/templates/kuttl/smoke/40-change-routing.j2
 create mode 100644 tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2
 create mode 100644 tests/templates/kuttl/smoke/41-restart-trino-lb.yaml
 create mode 100644 tests/templates/kuttl/smoke/50-assert.yaml
 create mode 100644 tests/templates/kuttl/smoke/50-test-routing.yaml.j2

diff --git a/tests/templates/kuttl/smoke/10-assert.yaml b/tests/templates/kuttl/smoke/10-assert.yaml
index ad2cfd1..7ebc7d9 100644
--- a/tests/templates/kuttl/smoke/10-assert.yaml
+++ b/tests/templates/kuttl/smoke/10-assert.yaml
@@ -3,4 +3,5 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 timeout: 720
 commands:
-  - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-default-1 --timeout 719s
+  - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-s-1 --timeout 719s
+  - script: kubectl -n $NAMESPACE wait --for=condition=available=true trinoclusters.trino.stackable.tech/trino-m-1 --timeout 719s
diff --git a/tests/templates/kuttl/smoke/10-install-trino.yaml.j2 b/tests/templates/kuttl/smoke/10-install-trinos.yaml.j2
similarity index 83%
rename from tests/templates/kuttl/smoke/10-install-trino.yaml.j2
rename to tests/templates/kuttl/smoke/10-install-trinos.yaml.j2
index 7efb961..5fc62a6 100644
--- a/tests/templates/kuttl/smoke/10-install-trino.yaml.j2
+++ b/tests/templates/kuttl/smoke/10-install-trinos.yaml.j2
@@ -1,8 +1,9 @@
+{% for name in ['trino-s-1', 'trino-m-1'] %}
 ---
 apiVersion: trino.stackable.tech/v1alpha1
 kind: TrinoCluster
 metadata:
-  name: trino-default-1
+  name: {{ name }}
 spec:
   image:
 {% if test_scenario['values']['trino'].find(",") > 0 %}
@@ -23,6 +24,12 @@ spec:
 {% endif %}
   coordinators:
     config:
+      resources:
+        cpu:
+          min: 250m
+          max: "1"
+        memory:
+          limit: 2Gi
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
     roleGroups:
@@ -30,12 +37,19 @@ spec:
         replicas: 1
   workers:
     config:
+      resources:
+        cpu:
+          min: 250m
+          max: "1"
+        memory:
+          limit: 3Gi
       gracefulShutdownTimeout: 60s # Let the test run faster
       logging:
         enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
     roleGroups:
       default:
         replicas: 1
+{% endfor %}
 ---
 apiVersion: authentication.stackable.tech/v1alpha1
 kind: AuthenticationClass
diff --git a/tests/templates/kuttl/smoke/20-install-trino-lb.j2 b/tests/templates/kuttl/smoke/20-install-trino-lb.j2
index 8736dc2..a57517c 100644
--- a/tests/templates/kuttl/smoke/20-install-trino-lb.j2
+++ b/tests/templates/kuttl/smoke/20-install-trino-lb.j2
@@ -9,6 +9,9 @@ kind: Deployment
 metadata:
   name: trino-lb
 spec:
+  # I'm a bit surprised that 3 replicas work in combination with inMemory persistence :)
+  # Maybe Kubernetes is sticky for some reason, maybe trino-clients will retry call until they get
+  # to the correct trino-lb instance...
   replicas: 3
   selector:
     matchLabels:
diff --git a/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2 b/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2
index 03e943f..2b15508 100644
--- a/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2
+++ b/tests/templates/kuttl/smoke/20_trino-lb-config.yaml.j2
@@ -10,8 +10,8 @@ trinoClusterGroups:
   default:
     maxRunningQueries: 1
     trinoClusters:
-      - name: trino-default-1
-        endpoint: https://trino-default-1-coordinator:8443
+      - name: trino-s-1
+        endpoint: https://trino-s-1-coordinator:8443
         credentials:
           username: admin
           password: adminadmin
diff --git a/tests/templates/kuttl/smoke/30-assert.yaml b/tests/templates/kuttl/smoke/30-assert.yaml
index 9f2d6e2..7f2cc57 100644
--- a/tests/templates/kuttl/smoke/30-assert.yaml
+++ b/tests/templates/kuttl/smoke/30-assert.yaml
@@ -2,6 +2,6 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: send-queries
+  name: test-queries
 status:
   succeeded: 1
diff --git a/tests/templates/kuttl/smoke/30-send-queries.yaml.j2 b/tests/templates/kuttl/smoke/30-send-queries.yaml.j2
deleted file mode 100644
index a6d74b1..0000000
--- a/tests/templates/kuttl/smoke/30-send-queries.yaml.j2
+++ /dev/null
@@ -1,53 +0,0 @@
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: send-queries
-spec:
-  template:
-    spec:
-      containers:
-        - name: send-queries
-          image: oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino'] }}-stackable0.0.0-dev
-          command:
-            - /bin/bash
-            - -x
-            - -euo
-            - pipefail
-            - -c
-            - |
-              # Query Trino directly
-              echo "show catalogs" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server https://trino-default-1-coordinator:8443 --insecure --user alice --password
-              echo "select count(*) from tpch.sf1.customer" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server https://trino-default-1-coordinator:8443 --insecure --user alice --password
-
-              # Query trino-lb
-              echo "show catalogs" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server https://trino-lb:8443 --insecure --user alice --password
-              echo "select count(*) from tpch.sf1.customer" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server https://trino-lb:8443 --insecure --user alice --password
-
-              # Send multiple queries in parallel to trino-lb
-              # Number of parallel requests
-              NUM_REQUESTS=10
-
-              # Command to run the query
-              QUERY_COMMAND='echo "select count(*) from tpch.sf1.customer" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server https://trino-lb:8443 --insecure --user alice --password'
-
-              # Array to store process IDs (PIDs)
-              pids=()
-
-              # Run requests in parallel
-              for i in $(seq 1 $NUM_REQUESTS); do
-                  bash -c "$QUERY_COMMAND" &
-                  pids+=("$!")
-              done
-
-              # Wait for all processes to complete and check exit codes
-              for pid in "${pids[@]}"; do
-                  wait "$pid"
-                  if [ $? -ne 0 ]; then
-                      echo "One of the requests failed with a non-zero exit code."
-                      exit 1
-                  fi
-              done
-
-              echo "All requests completed successfully."
-      restartPolicy: OnFailure
diff --git a/tests/templates/kuttl/smoke/30-test-queries.yaml.j2 b/tests/templates/kuttl/smoke/30-test-queries.yaml.j2
new file mode 100644
index 0000000..b335050
--- /dev/null
+++ b/tests/templates/kuttl/smoke/30-test-queries.yaml.j2
@@ -0,0 +1,54 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: test-queries
+spec:
+  template:
+    spec:
+      containers:
+        - name: test-queries
+          image: oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino'] }}-stackable0.0.0-dev
+          command:
+            - /bin/bash
+            - -x
+            - -euo
+            - pipefail
+            - -c
+            - |
+              # Query Trinos and trino-lb
+              COORDINATORS=(
+                  "https://trino-s-1-coordinator:8443"
+                  "https://trino-m-1-coordinator:8443"
+                  "https://trino-lb:8443"
+              )
+
+              export TRINO_USER="alice"
+              export TRINO_PASSWORD="alicealice"
+              QUERY="select count(*) from tpch.sf1.customer"
+
+              for COORDINATOR in "${COORDINATORS[@]}"; do
+                  echo "$QUERY" | java -jar trino-cli-executable.jar --server $COORDINATOR --insecure --user $TRINO_USER --password
+              done
+
+              # Send multiple queries in parallel to trino-lb
+              NUM_REQUESTS=10
+              TRINO_LB_ADDRESS="https://trino-lb:8443"
+
+              pids=()
+
+              for ((i = 1; i <= NUM_REQUESTS; i++)); do
+                  echo "$QUERY" | java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user $TRINO_USER --password &
+                  pids+=("$!")
+              done
+
+              # Wait for all processes to complete and check exit codes
+              for pid in "${pids[@]}"; do
+                  if ! wait "$pid"; then
+                      echo "One of the requests failed with a non-zero exit code."
+                      exit 1
+                  fi
+              done
+
+              echo "All queries completed successfully."
+      restartPolicy: OnFailure
diff --git a/tests/templates/kuttl/smoke/40-change-routing.j2 b/tests/templates/kuttl/smoke/40-change-routing.j2
new file mode 100644
index 0000000..5d08ea8
--- /dev/null
+++ b/tests/templates/kuttl/smoke/40-change-routing.j2
@@ -0,0 +1,6 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl -n $NAMESPACE delete secret trino-lb-config || true
+  - script: kubectl -n $NAMESPACE create secret generic trino-lb-config --from-file=trino-lb-config.yaml=40_trino-lb-config.yaml
diff --git a/tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2 b/tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2
new file mode 100644
index 0000000..8f81c64
--- /dev/null
+++ b/tests/templates/kuttl/smoke/40_trino-lb-config.yaml.j2
@@ -0,0 +1,52 @@
+trinoLb:
+  externalAddress: https://trino-lb:8443
+  tls:
+    enabled: true
+    certPemFile: /certificates/cert.pem
+    keyPemFile: /certificates/key.pem
+  persistence:
+    inMemory: {}
+trinoClusterGroups:
+  s:
+    maxRunningQueries: 1
+    trinoClusters:
+      - name: trino-s-1
+        endpoint: https://trino-s-1-coordinator:8443
+        credentials:
+          username: admin
+          password: adminadmin
+  m:
+    maxRunningQueries: 1
+    trinoClusters:
+      - name: trino-m-1
+        endpoint: https://trino-m-1-coordinator:8443
+        credentials:
+          username: admin
+          password: adminadmin
+trinoClusterGroupsIgnoreCert: true
+
+routers:
+  - trinoRoutingGroupHeader:
+      headerName: X-Trino-Routing-Group
+  - clientTags:
+      oneOf: ["s"]
+      trinoClusterGroup: s
+  - clientTags:
+      oneOf: ["m"]
+      trinoClusterGroup: m
+  - pythonScript:
+      script: |
+        from typing import Optional
+
+        def targetClusterGroup(query: str, headers: dict[str, str]) -> Optional[str]:
+            user = get_user(headers)
+            if user == "alice":
+                return "s"
+            elif user == "bob":
+                return "m"
+            else:
+                return None
+
+        def get_user(headers: dict[str, str]) -> Optional[str]:
+            return headers.get("x-trino-user")
+routingFallback: s
diff --git a/tests/templates/kuttl/smoke/41-restart-trino-lb.yaml b/tests/templates/kuttl/smoke/41-restart-trino-lb.yaml
new file mode 100644
index 0000000..0e36a0b
--- /dev/null
+++ b/tests/templates/kuttl/smoke/41-restart-trino-lb.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl -n $NAMESPACE rollout restart deployment trino-lb
+  - script: kubectl -n $NAMESPACE rollout status deployment trino-lb --watch
+  - script: sleep 2
diff --git a/tests/templates/kuttl/smoke/50-assert.yaml b/tests/templates/kuttl/smoke/50-assert.yaml
new file mode 100644
index 0000000..32690b9
--- /dev/null
+++ b/tests/templates/kuttl/smoke/50-assert.yaml
@@ -0,0 +1,7 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: test-routing
+status:
+  succeeded: 1
diff --git a/tests/templates/kuttl/smoke/50-test-routing.yaml.j2 b/tests/templates/kuttl/smoke/50-test-routing.yaml.j2
new file mode 100644
index 0000000..deb69b1
--- /dev/null
+++ b/tests/templates/kuttl/smoke/50-test-routing.yaml.j2
@@ -0,0 +1,34 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: test-routing
+spec:
+  template:
+    spec:
+      containers:
+        - name: test-routing
+          image: oci.stackable.tech/sdp/trino-cli:{{ test_scenario['values']['trino'] }}-stackable0.0.0-dev
+          command:
+            - /bin/bash
+            - -x
+            - -euo
+            - pipefail
+            - -c
+            - |
+              TRINO_LB_ADDRESS="https://trino-lb:8443"
+              COORDINATOR_NAME_QUERY="select regexp_extract(url_extract_host(http_uri), '[a-z0-9-]+') from "system".runtime.nodes where coordinator = true"
+
+              # admin lands in s
+              echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=adminadmin java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user admin --password | grep -qx '"trino-s-1-coordinator-default-0"' || exit 1
+              # alice lands in s
+              echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=alicealice java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user alice --password | grep -qx '"trino-s-1-coordinator-default-0"' || exit 1
+              # bob lands in m
+              echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=bobbob java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user bob --password | grep -qx '"trino-m-1-coordinator-default-0"' || exit 1
+
+              # We can also set client tags explicitly
+              echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=adminadmin java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user admin --password --client-tags=s | grep -qx '"trino-s-1-coordinator-default-0"' || exit 1
+              echo "$COORDINATOR_NAME_QUERY" | TRINO_PASSWORD=adminadmin java -jar trino-cli-executable.jar --server $TRINO_LB_ADDRESS --insecure --user admin --password --client-tags=m | grep -qx '"trino-m-1-coordinator-default-0"' || exit 1
+
+              echo "All queries completed successfully."
+      restartPolicy: OnFailure
diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
index e9a7e32..62b6d0c 100644
--- a/tests/test-definition.yaml
+++ b/tests/test-definition.yaml
@@ -17,9 +17,6 @@ dimensions:
       # To use a custom image, add a comma and the full name after the product version
       # - 470,oci.stackable.tech/sdp/trino:470-stackable0.0.0-dev
       # However, watch out, you need to tweak the trino-cli image
-  - name: keycloak
-    values:
-      - 25.0.0
   - name: trino-lb-https
     values:
       - "true"
@@ -30,7 +27,7 @@ tests:
       - trino-lb
       - trino
       - trino-use-authentication
-  # - name: oidc
-  #   dimensions:
-  #     - trino-latest
-  #     - keycloak
+
+# TODOS
+# 1. Test storage backend redis and postgres
+#   a. Also restart trino-lb deployment to make sure persistence is kept

From a7f785d24f53bc8784a8440377d328cadea8287e Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Mon, 15 Sep 2025 11:44:37 +0200
Subject: [PATCH 3/4] Update Trino versions

---
 tests/test-definition.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
index 62b6d0c..8210c0c 100644
--- a/tests/test-definition.yaml
+++ b/tests/test-definition.yaml
@@ -6,8 +6,8 @@ dimensions:
   - name: trino
     values:
       - "451"
-      - "455"
       - "470"
+      - "476"
       # To use a custom image, add a comma and the full name after the product version
       # - 470,oci.stackable.tech/sdp/trino:470-stackable0.0.0-dev
       # However, watch out, you need to tweak the trino-cli image
@@ -26,7 +26,6 @@ tests:
     dimensions:
       - trino-lb
       - trino
-      - trino-use-authentication
 
 # TODOS
 # 1. Test storage backend redis and postgres

From c50b22a636e50757d92055e4c1f7b5515b285e8d Mon Sep 17 00:00:00 2001
From: Sebastian Bernauer <sebastian.bernauer@stackable.tech>
Date: Mon, 15 Sep 2025 13:26:17 +0200
Subject: [PATCH 4/4] Switch cert Secret to stringData

---
 .../kuttl/smoke/00-trino-lb-certificates.yaml | 106 +++++++++---------
 1 file changed, 52 insertions(+), 54 deletions(-)

diff --git a/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml b/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml
index 473c5d0..3a69c2b 100644
--- a/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml
+++ b/tests/templates/kuttl/smoke/00-trino-lb-certificates.yaml
@@ -2,59 +2,57 @@ apiVersion: v1
 kind: Secret
 metadata:
   name: trino-lb-certificates
-data:
+# Copied from example-configs/self-singed-certs
+stringData:
   cert.pem: |
-    LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURrekNDQW51Z0F3SUJBZ0lVWFZZa1JDck0v
-    Z2UwM0RWeW1EdFhDdXlicDdnd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1dURUxNQWtHQTFVRUJoTUNW
-    Vk14RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeElUQWZCZ05WQkFvTQpHRWx1ZEdWeWJtVjBJ
-    RmRwWkdkcGRITWdVSFI1SUV4MFpERVNNQkFHQTFVRUF3d0piRzlqWVd4b2IzTjBNQjRYCkRUSXhN
-    RGN6TVRFME1qSXhNbG9YRFRJeU1EY3pNVEUwTWpJeE1sb3dXVEVMTUFrR0ExVUVCaE1DVlZNeEV6
-    QVIKQmdOVkJBZ01DbE52YldVdFUzUmhkR1V4SVRBZkJnTlZCQW9NR0VsdWRHVnlibVYwSUZkcFpH
-    ZHBkSE1nVUhSNQpJRXgwWkRFU01CQUdBMVVFQXd3SmJHOWpZV3hvYjNOME1JSUJJakFOQmdrcWhr
-    aUc5dzBCQVFFRkFBT0NBUThBCk1JSUJDZ0tDQVFFQTAyVjVaam1xTEIvVlF3VGFycnovMzVxc2E4
-    M0wrRGJBb2EwMDAxK2pWbW1DK0c5TnVmaTAKZGFyb0ZXai9VaWN2MmZaV0VUVThKb1pLVXJYNEJL
-    OW9nNWNnNXJsbi9DdEJSV0NVWUl3UmdZOVIvQ2RCR1BuNAprcCtYa1NKYUN3NzRaSXlMeS9aZnV4
-    Nmg4RVMxbTlZUm5CemErczdVK0ltUkJSZjRNUlB0WFEzL21xSnhBWllxCmRPbktudnNzUnlEMnF1
-    dGdWVEF4d01VdkpXSWl2UmhSWURqN1dPcFM0Q0VFZVF4UDFpSDEvVDVQN0ZkdFRHZFQKYlZCQUJD
-    QThKaEw5NnVGR1BwT1lIY00vN1I1RUlBM3laNUZOZzkzMVF6b0RJVGp0WEd0UTZ5OS9sL0lZa1dt
-    NgpKNjdSV2NOMElvVHNaaHowV05VNGdBZXNsVnRKTG9mbjhRSURBUUFCbzFNd1VUQWRCZ05WSFE0
-    RUZnUVV6Rm5LCk5mUzRMQVl1S2VXd0hiem9vRVIweVowd0h3WURWUjBqQkJnd0ZvQVV6Rm5LTmZT
-    NExBWXVLZVd3SGJ6b29FUjAKeVowd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBTkJna3Foa2lHOXcw
-    QkFRc0ZBQU9DQVFFQWs0TytlOWppYTU5Vwpad2V0TjRHVTdPV2NZaG1PZ1NpelJTczZ1N21UZnA2
-    MkxETXQ5NldLVTNUSGtzT25aNDRIbnFXUXhzU2ZkRlZVClhKRDEydGp2VlU4WjRGV3pRYWpjSGVl
-    bVVZaUR6ZThFQWg2VG54blVjT3JVOEljd2lLR3hDV1JZLzkwOGpuV2cKK01Nc2NmTUNNWVRSZGVU
-    UHFEOGZHekFsVUN0bXl6SDZLTEUzczRPby9yNStOUitVdnJ3cGR2Yjd4ZTBNd3dPOQpRL3pSNE44
-    ZXAvSHdIVkVPYmNhQm9mRTFzc1pMa3NYN1pnQ1A5d01nWFJXcE5BdEM1RVd4TWJ4WWpCZldGSDI0
-    CmZESmxCTWlHSldnOEhIY3hLN3dRaEZoK2Z1eU56RSt4RVdQc0k5VkwxekRmdGQ5eDgvUXNPYWd5
-    RU9uWThWeHIKQW9wdlowOXVFUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    -----BEGIN CERTIFICATE-----
+    MIIDkzCCAnugAwIBAgIUXVYkRCrM/ge03DVymDtXCuybp7gwDQYJKoZIhvcNAQEL
+    BQAwWTELMAkGA1UEBhMCVVMxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+    GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MB4X
+    DTIxMDczMTE0MjIxMloXDTIyMDczMTE0MjIxMlowWTELMAkGA1UEBhMCVVMxEzAR
+    BgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5
+    IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+    MIIBCgKCAQEA02V5ZjmqLB/VQwTarrz/35qsa83L+DbAoa0001+jVmmC+G9Nufi0
+    daroFWj/Uicv2fZWETU8JoZKUrX4BK9og5cg5rln/CtBRWCUYIwRgY9R/CdBGPn4
+    kp+XkSJaCw74ZIyLy/Zfux6h8ES1m9YRnBza+s7U+ImRBRf4MRPtXQ3/mqJxAZYq
+    dOnKnvssRyD2qutgVTAxwMUvJWIivRhRYDj7WOpS4CEEeQxP1iH1/T5P7FdtTGdT
+    bVBABCA8JhL96uFGPpOYHcM/7R5EIA3yZ5FNg931QzoDITjtXGtQ6y9/l/IYkWm6
+    J67RWcN0IoTsZhz0WNU4gAeslVtJLofn8QIDAQABo1MwUTAdBgNVHQ4EFgQUzFnK
+    NfS4LAYuKeWwHbzooER0yZ0wHwYDVR0jBBgwFoAUzFnKNfS4LAYuKeWwHbzooER0
+    yZ0wDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAk4O+e9jia59W
+    ZwetN4GU7OWcYhmOgSizRSs6u7mTfp62LDMt96WKU3THksOnZ44HnqWQxsSfdFVU
+    XJD12tjvVU8Z4FWzQajcHeemUYiDze8EAh6TnxnUcOrU8IcwiKGxCWRY/908jnWg
+    +MMscfMCMYTRdeTPqD8fGzAlUCtmyzH6KLE3s4Oo/r5+NR+Uvrwpdvb7xe0MwwO9
+    Q/zR4N8ep/HwHVEObcaBofE1ssZLksX7ZgCP9wMgXRWpNAtC5EWxMbxYjBfWFH24
+    fDJlBMiGJWg8HHcxK7wQhFh+fuyNzE+xEWPsI9VL1zDftd9x8/QsOagyEOnY8Vxr
+    AopvZ09uEQ==
+    -----END CERTIFICATE-----
   key.pem: |
-    LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZB
-    QVNDQktjd2dnU2pBZ0VBQW9JQkFRRFRaWGxtT2Fvc0g5VkQKQk5xdXZQL2ZtcXhyemN2NE5zQ2hy
-    VFRUWDZOV2FZTDRiMDI1K0xSMXF1Z1ZhUDlTSnkvWjlsWVJOVHdtaGtwUwp0ZmdFcjJpRGx5RG11
-    V2Y4SzBGRllKUmdqQkdCajFIOEowRVkrZmlTbjVlUklsb0xEdmhrakl2TDlsKzdIcUh3ClJMV2Ix
-    aEdjSE5yNnp0VDRpWkVGRi9neEUrMWREZithb25FQmxpcDA2Y3FlK3l4SElQYXE2MkJWTURIQXhT
-    OGwKWWlLOUdGRmdPUHRZNmxMZ0lRUjVERS9XSWZYOVBrL3NWMjFNWjFOdFVFQUVJRHdtRXYzcTRV
-    WStrNWdkd3ovdApIa1FnRGZKbmtVMkQzZlZET2dNaE9PMWNhMURyTDMrWDhoaVJhYm9ucnRGWncz
-    UWloT3htSFBSWTFUaUFCNnlWClcwa3VoK2Z4QWdNQkFBRUNnZ0VBRGx0dThrMXFURkxoSmdzWFd4
-    VEZBQWUrUEJnZkNUMld1YVJNMlNvK3FxakIKMTJPZjBNaWVZUHQ1aGJLNjNIYUMzbmZIZ3FXdDd5
-    UGh1bHBYZk9INDVDOEljZ01YbDkzTU1nME1KcjU4bGVNSQorMm9qRnJJcmVySFNGbTVSMVR4d0RF
-    d3JWbS9tTW93ekRXRnRRQ2M2elBKOHdObjVSdVA0OEhLZlRaMy8yZmp3CnpFalN3UE8yd0ZNZm8x
-    RUpOVGpsSTMwM2xGYmRGQnM2N05hWDZwdWgzME03VG4rZ3puSEt5TzVhN0Y1N3drSXQKZmtnbkV5
-    L3NnTWVkUWx3WDdiUnBVb0Q2ZjBmWnpWOFF6NGNIRnl3dFlFcmN6WkpoM1ZHaXRKb08vVkNJRGR0
-    eQpSUFhPQXFWRGQ3RXBQMVVVZWhabEtWV1owT1pNRWZSZ0tiUkNlbDVhYlFLQmdRRHdnd3JJUTUr
-    QmladjZhMFZUCkVUZVhCK2hSYnZCaW5SeWtOby9SdkxjM2oxZW5SaDkvek8vU2hhZFpJWGdPQWlN
-    MUpucjVHcDhLa05HY2E2SzEKbXlodGFkN3hZUE9EWXpOWFhwNlQxT1BnWnhIWkxJWXpWVWo2eXBY
-    ZVY2NFRlNVppRGFKMUQ0OWN6c3ErUHFzUQpYUmNnQkpTTnBGdERGaVhXcGpYV2Z4OFB4d0tCZ1FE
-    aEFuTFk1U2wyZWVRbyt1ZDBNdmp3ZnRCL21OMnFDekpZCjVBbFFwUkk0VGhXeEpnR1B1SFRSMjl6
-    VmE1aVdOWXVBNUxXckMxeS93eCt0NUhLVXdxKzVreHZzK25wWXBESkQKWlgvdzBHbGM2czBKYy9t
-    RnlTa2J3OUIyTGVQZWRMN2xSRjVPaUF5QzZEMTA2U2M5VjJqbEw0SWZsbU96dDRDRApaVE5iTHRD
-    Nmh3S0JnSGZJekJYeGwvOXNDY011cWRnMU92cDlkYmNaQ2FBVG43QXBmSGQ1QmNjbUhRR3lhdjI3
-    Cms3WEYyeE1KR0VIaHpxY3FBeFVOclNnVitFOXZUQm9tckh2UnZyZDVFYzdlR1RQcWJCQTBkMG5N
-    QzVlZUZUaDcKd1YwbWlIMjBMWDZHanQ5RzZ5SmlIWVNiZVY1RzErdk9jVFlCRWZ0NVgvcUpqVTdh
-    ZVBYYldoMEJBb0dCQUpsVgo1dGdDQ3VodkZsb0s2ZkhZenFadGRUNk8rUGZwVzIwU01Ycmdrdk1G
-    MjJoMll2Z0RGckR3cUtSVUI0N05mSHpnCjN5QnB4TkgxY2NBNS93OTdRTzh3M2dYM2g2cWljcEpW
-    T0FQdXN1NmNJQkFDRlpmalJ2MWh5c3pPWnd2dytTb2EKRmo1a0hrcVRZMVlwa1JFUFlTOVYyZElX
-    MVdqaWMxU1hnWkR3N1ZNL0FvR0FQL2NaM1pIVFNDRFRGbEl0cXk1QwpySXkyQWlZMFdKc3grSzBx
-    Y3Z0b3NQT093dG5HaldIYjFnZGFWZGZYL0lSa1NzWDRQQU9kbnN5aWROQzUvbC9tCnk4b2ErNVdF
-    ZUdGY2xXRmhyNGRuVEE3NjZvOEhyTTJVaklnV1dZQkYyVktkcHRHbkh4RmVKV0ZVbWVRQy94ZVcK
-    dzM3cENTN3lrTCs3Z3A3VjBXU2hZc3c9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K
+    -----BEGIN PRIVATE KEY-----
+    MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDTZXlmOaosH9VD
+    BNquvP/fmqxrzcv4NsChrTTTX6NWaYL4b025+LR1qugVaP9SJy/Z9lYRNTwmhkpS
+    tfgEr2iDlyDmuWf8K0FFYJRgjBGBj1H8J0EY+fiSn5eRIloLDvhkjIvL9l+7HqHw
+    RLWb1hGcHNr6ztT4iZEFF/gxE+1dDf+aonEBlip06cqe+yxHIPaq62BVMDHAxS8l
+    YiK9GFFgOPtY6lLgIQR5DE/WIfX9Pk/sV21MZ1NtUEAEIDwmEv3q4UY+k5gdwz/t
+    HkQgDfJnkU2D3fVDOgMhOO1ca1DrL3+X8hiRabonrtFZw3QihOxmHPRY1TiAB6yV
+    W0kuh+fxAgMBAAECggEADltu8k1qTFLhJgsXWxTFAAe+PBgfCT2WuaRM2So+qqjB
+    12Of0MieYPt5hbK63HaC3nfHgqWt7yPhulpXfOH45C8IcgMXl93MMg0MJr58leMI
+    +2ojFrIrerHSFm5R1TxwDEwrVm/mMowzDWFtQCc6zPJ8wNn5RuP48HKfTZ3/2fjw
+    zEjSwPO2wFMfo1EJNTjlI303lFbdFBs67NaX6puh30M7Tn+gznHKyO5a7F57wkIt
+    fkgnEy/sgMedQlwX7bRpUoD6f0fZzV8Qz4cHFywtYErczZJh3VGitJoO/VCIDdty
+    RPXOAqVDd7EpP1UUehZlKVWZ0OZMEfRgKbRCel5abQKBgQDwgwrIQ5+BiZv6a0VT
+    ETeXB+hRbvBinRykNo/RvLc3j1enRh9/zO/ShadZIXgOAiM1Jnr5Gp8KkNGca6K1
+    myhtad7xYPODYzNXXp6T1OPgZxHZLIYzVUj6ypXeV64Te5ZiDaJ1D49czsq+PqsQ
+    XRcgBJSNpFtDFiXWpjXWfx8PxwKBgQDhAnLY5Sl2eeQo+ud0MvjwftB/mN2qCzJY
+    5AlQpRI4ThWxJgGPuHTR29zVa5iWNYuA5LWrC1y/wx+t5HKUwq+5kxvs+npYpDJD
+    ZX/w0Glc6s0Jc/mFySkbw9B2LePedL7lRF5OiAyC6D106Sc9V2jlL4IflmOzt4CD
+    ZTNbLtC6hwKBgHfIzBXxl/9sCcMuqdg1Ovp9dbcZCaATn7ApfHd5BccmHQGyav27
+    k7XF2xMJGEHhzqcqAxUNrSgV+E9vTBomrHvRvrd5Ec7eGTPqbBA0d0nMC5eeFTh7
+    wV0miH20LX6Gjt9G6yJiHYSbeV5G1+vOcTYBEft5X/qJjU7aePXbWh0BAoGBAJlV
+    5tgCCuhvFloK6fHYzqZtdT6O+PfpW20SMXrgkvMF22h2YvgDFrDwqKRUB47NfHzg
+    3yBpxNH1ccA5/w97QO8w3gX3h6qicpJVOAPusu6cIBACFZfjRv1hyszOZwvw+Soa
+    Fj5kHkqTY1YpkREPYS9V2dIW1Wjic1SXgZDw7VM/AoGAP/cZ3ZHTSCDTFlItqy5C
+    rIy2AiY0WJsx+K0qcvtosPOOwtnGjWHb1gdaVdfX/IRkSsX4PAOdnsyidNC5/l/m
+    y8oa+5WEeGFclWFhr4dnTA766o8HrM2UjIgWWYBF2VKdptGnHxFeJWFUmeQC/xeW
+    w37pCS7ykL+7gp7V0WShYsw=
+    -----END PRIVATE KEY-----