diff --git a/.gitignore b/.gitignore index c25a43231..5734e0467 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ testbin/* bundle/* bundle.Dockerfile config/operator/deployment/kustomization.yaml +# Generated certificate ConfigMaps for kustomize +config/samples/tls/custom_route_cert/placement-cert-data.yaml # Test binary, build with `go test -c` *.test diff --git a/config/samples/tls/custom_route_cert/kustomization.yaml b/config/samples/tls/custom_route_cert/kustomization.yaml index 94aa1cafb..e63da179f 100644 --- a/config/samples/tls/custom_route_cert/kustomization.yaml +++ b/config/samples/tls/custom_route_cert/kustomization.yaml @@ -1,5 +1,6 @@ resources: - ../../base/openstackcontrolplane +- placement-cert-data.yaml patches: - target: @@ -9,6 +10,37 @@ patches: - op: replace path: /metadata/name value: openstack -- target: - kind: OpenStackControlPlane - path: patch.yaml + +- path: patch.yaml + +# Use replacements to inject cert data from ConfigMap +replacements: +- source: + kind: ConfigMap + name: placement-cert-data + fieldPath: data.[tls.crt] + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.placement.apiOverride.route.spec.tls.certificate + +- source: + kind: ConfigMap + name: placement-cert-data + fieldPath: data.[tls.key] + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.placement.apiOverride.route.spec.tls.key + +- source: + kind: ConfigMap + name: placement-cert-data + fieldPath: data.[ca.crt] + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.placement.apiOverride.route.spec.tls.caCertificate diff --git a/config/samples/tls/custom_route_cert/patch.yaml b/config/samples/tls/custom_route_cert/patch.yaml index 436f87662..20ffd3865 100644 --- a/config/samples/tls/custom_route_cert/patch.yaml +++ b/config/samples/tls/custom_route_cert/patch.yaml @@ -12,10 +12,7 @@ spec: route: spec: tls: - certificate: | - CERT123 - key: | - KEY123 - caCertificate: | - CACERT123 + certificate: "" + key: "" + caCertificate: "" termination: reencrypt diff --git a/tests/kuttl/common/README.md b/tests/kuttl/common/README.md new file mode 100644 index 000000000..a609b53fc --- /dev/null +++ b/tests/kuttl/common/README.md @@ -0,0 +1,115 @@ +# Common Certificate Management Utilities + +This directory contains shared utilities for creating and managing custom TLS certificates using cert-manager in OpenStack operator kuttl tests. + +## Files + +- **`create_custom_cert.sh`** - Main script with bash functions for certificate creation +- **`osp_check_route_cert.sh`** - Verification script for route certificates +- **`verify_route_override_certs.sh`** - Verification script for OpenStackControlPlane overrides +- **`prepare_placement_certs.sh`** - Helper script to create ConfigMap from certificates +- **`custom-ingress-issuer.yaml`** - YAML template for custom ingress issuer +- **`custom-internal-issuer.yaml`** - YAML template for custom internal issuer +- **`custom-barbican-route.yaml`** - Pre-generated barbican route secret +- **`custom-ca.yaml`** - Custom CA bundle for testing + +## Quick Reference + +### Create Certificates for Barbican and Placement + +```bash +source ../../common/create_custom_cert.sh +INGRESS_DOMAIN=$(oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}') +create_barbican_placement_routes "${INGRESS_DOMAIN}" "${NAMESPACE}" +``` + +### Main Functions + +| Function | Usage | Description | +|----------|-------|-------------| +| `create_barbican_placement_routes` | ` [namespace]` | One-command setup for barbican and placement | +| `create_service_route_certificate` | ` [namespace]` | Create certificate for any service | +| `create_custom_issuer` | ` [namespace]` | Create root CA and issuer | +| `create_wildcard_certificate` | ` [issuer-name] [namespace]` | Create wildcard certificate | +| `setup_custom_certificate_infrastructure` | `[namespace]` | Setup complete cert infrastructure | +| `cleanup_custom_certificates` | `[namespace]` | Remove all custom certificates | + +### Verification Functions + +```bash +# Verify route certificate matches secret +bash ../../common/osp_check_route_cert.sh + +# Verify OpenStackControlPlane override matches secret +bash ../../common/verify_route_override_certs.sh +``` + +## Certificate Specifications + +### Root CA Certificate +- **Algorithm:** ECDSA P-256 +- **Duration:** 87600h (10 years) +- **Usage:** Certificate Sign, CRL Sign +- **IsCA:** true + +### Service Certificates +- **Algorithm:** ECDSA P-256 +- **Duration:** 8760h (1 year) +- **Renewal:** 720h (30 days) before expiration +- **Usage:** Server Auth, Client Auth +- **DNS Names:** `*.domain.com`, `domain.com` + +## Usage in Kuttl Tests + +```yaml +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + source ../../common/create_custom_cert.sh + INGRESS_DOMAIN=$(oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}') + create_barbican_placement_routes "${INGRESS_DOMAIN}" "${NAMESPACE}" +``` + +## Examples + +### Create Certificates for Any Service + +```bash +source ../../common/create_custom_cert.sh + +# For keystone +create_service_route_certificate "keystone" "apps-crc.testing" "openstack-kuttl-tests" + +# For glance +create_service_route_certificate "glance" "apps-crc.testing" "openstack-kuttl-tests" +``` + +### Setup Complete Infrastructure + +```bash +# Creates both ingress and internal issuers +setup_custom_certificate_infrastructure "openstack-kuttl-tests" +``` + +### Cleanup + +```bash +# Remove all custom certificates and issuers +cleanup_custom_certificates "openstack-kuttl-tests" +``` + +## Security Considerations + +⚠️ **Important:** These certificates are for **TESTING PURPOSES ONLY** + +## Complete Documentation + +For detailed documentation including test flow, architecture, troubleshooting, and implementation details, see: + +**[tests/kuttl/tests/ctlplane-tls-custom-route/README.md](../tests/ctlplane-tls-custom-route/README.md)** + +## Related Documentation + +- [Cert-Manager Documentation](https://cert-manager.io/docs/) +- [ctlplane-tls-custom-route Test](../tests/ctlplane-tls-custom-route/README.md) diff --git a/tests/kuttl/common/create_custom_cert.sh b/tests/kuttl/common/create_custom_cert.sh new file mode 100755 index 000000000..43999a85b --- /dev/null +++ b/tests/kuttl/common/create_custom_cert.sh @@ -0,0 +1,282 @@ +#!/bin/bash +# +# Helper functions to create custom issuers and wildcard certificates for kuttl tests +# + +# Wait for a resource to reach a specific state +function wait_for_state() { + local object="$1" + local state="$2" + local timeout="$3" + local namespace="${4:-openstack-kuttl-tests}" + + echo "Waiting for '${object}' in namespace '${namespace}' to become '${state}'..." + oc wait --for=${state} --timeout=${timeout} ${object} -n="${namespace}" + return $? +} + +# Create a self-signed issuer (prerequisite for CA certificate) +function create_selfsigned_issuer() { + local namespace="${1:-openstack-kuttl-tests}" + + echo "Creating self-signed issuer in namespace ${namespace}..." + oc apply -f - << EOF +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: selfsigned-issuer + namespace: ${namespace} +spec: + selfSigned: {} +EOF + + if wait_for_state "issuer/selfsigned-issuer" "condition=Ready" "2m" "${namespace}"; then + echo "Self-signed issuer is ready" + else + echo "Failed to create self-signed issuer" + oc describe issuer selfsigned-issuer -n ${namespace} + return 1 + fi +} + +# Create a custom root CA certificate and issuer +# Usage: create_custom_issuer +function create_custom_issuer() { + local issuer_name="${1:-rootca-custom}" + local namespace="${2:-openstack-kuttl-tests}" + + echo "Creating custom root CA and issuer: ${issuer_name} in namespace ${namespace}..." + + # Ensure self-signed issuer exists + if ! oc get issuer selfsigned-issuer -n ${namespace} &>/dev/null; then + create_selfsigned_issuer "${namespace}" || return 1 + fi + + # Create the root CA certificate + oc apply -f - << EOF +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ${issuer_name} + namespace: ${namespace} +spec: + commonName: ${issuer_name} + isCA: true + duration: 87600h # 10 years + privateKey: + algorithm: ECDSA + size: 256 + issuerRef: + name: selfsigned-issuer + kind: Issuer + secretName: ${issuer_name} +EOF + + if wait_for_state "certificate/${issuer_name}" "condition=Ready" "5m" "${namespace}"; then + echo "Root CA certificate ${issuer_name} is ready" + else + echo "Failed to create root CA certificate" + oc describe certificate ${issuer_name} -n ${namespace} + return 1 + fi + + # Create the issuer that uses this CA + oc apply -f - << EOF +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: ${issuer_name} + namespace: ${namespace} +spec: + ca: + secretName: ${issuer_name} +EOF + + if wait_for_state "issuer/${issuer_name}" "condition=Ready" "2m" "${namespace}"; then + echo "Custom issuer ${issuer_name} is ready" + else + echo "Failed to create custom issuer" + oc describe issuer ${issuer_name} -n ${namespace} + return 1 + fi +} + +# Create a wildcard certificate for ingress/routes +# Usage: create_wildcard_certificate +function create_wildcard_certificate() { + local cert_name="${1}" + local domain="${2}" + local issuer_name="${3:-rootca-custom}" + local namespace="${4:-openstack-kuttl-tests}" + + if [[ -z "${cert_name}" ]] || [[ -z "${domain}" ]]; then + echo "Error: cert_name and domain are required" + echo "Usage: create_wildcard_certificate [issuer-name] [namespace]" + return 1 + fi + + echo "Creating wildcard certificate ${cert_name} for *.${domain}..." + + oc apply -f - << EOF +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: ${cert_name} + namespace: ${namespace} +spec: + commonName: "*.${domain}" + dnsNames: + - "*.${domain}" + - "${domain}" + usages: + - server auth + - client auth + issuerRef: + kind: Issuer + name: ${issuer_name} + secretName: ${cert_name} + privateKey: + algorithm: ECDSA + size: 256 + duration: 8760h # 1 year + renewBefore: 720h # 30 days +EOF + + if wait_for_state "certificate/${cert_name}" "condition=Ready" "5m" "${namespace}"; then + echo "Wildcard certificate ${cert_name} is ready" + else + echo "Failed to create wildcard certificate" + oc describe certificate ${cert_name} -n ${namespace} + return 1 + fi +} + +# Create a wildcard certificate suitable for OpenStack service routes +# This creates both the certificate and extracts it to a route-compatible secret +# Usage: create_service_route_certificate +function create_service_route_certificate() { + local service_name="${1}" + local ingress_domain="${2}" + local namespace="${3:-openstack-kuttl-tests}" + local issuer_name="rootca-ingress-custom" + local cert_name="${service_name}-custom-route-cert" + local route_secret_name="${service_name}-custom-route" + + if [[ -z "${service_name}" ]] || [[ -z "${ingress_domain}" ]]; then + echo "Error: service_name and ingress_domain are required" + echo "Usage: create_service_route_certificate [namespace]" + return 1 + fi + + echo "Creating custom route certificate for ${service_name}..." + + # Ensure custom issuer exists + if ! oc get issuer ${issuer_name} -n ${namespace} &>/dev/null; then + echo "Custom issuer ${issuer_name} does not exist, creating..." + create_custom_issuer "${issuer_name}" "${namespace}" || return 1 + fi + + # Create wildcard certificate + create_wildcard_certificate "${cert_name}" "${ingress_domain}" "${issuer_name}" "${namespace}" || return 1 + + # Wait a moment for the secret to be fully populated + sleep 2 + + # Extract certificate data and create route-compatible secret + echo "Creating route-compatible secret ${route_secret_name}..." + + local tls_crt + local tls_key + local ca_crt + tls_crt=$(oc get secret ${cert_name} -n ${namespace} -o jsonpath='{.data.tls\.crt}') + tls_key=$(oc get secret ${cert_name} -n ${namespace} -o jsonpath='{.data.tls\.key}') + ca_crt=$(oc get secret ${cert_name} -n ${namespace} -o jsonpath='{.data.ca\.crt}') + + if [[ -z "${tls_crt}" ]] || [[ -z "${tls_key}" ]]; then + echo "Error: Failed to extract certificate data from secret ${cert_name}" + return 1 + fi + + # Create the route secret with the certificate + oc apply -f - << EOF +apiVersion: v1 +kind: Secret +type: kubernetes.io/tls +metadata: + name: ${route_secret_name} + namespace: ${namespace} +data: + tls.crt: ${tls_crt} + tls.key: ${tls_key} + ca.crt: ${ca_crt} +EOF + + echo "Route certificate secret ${route_secret_name} created successfully" +} + +# Setup complete custom certificate infrastructure for kuttl tests +# This creates ingress and internal issuers for comprehensive testing +# Usage: setup_custom_certificate_infrastructure +function setup_custom_certificate_infrastructure() { + local namespace="${1:-openstack-kuttl-tests}" + + echo "Setting up custom certificate infrastructure in namespace ${namespace}..." + + # Create self-signed issuer (root of trust) + create_selfsigned_issuer "${namespace}" || return 1 + + # Create custom ingress issuer (for public routes) + create_custom_issuer "rootca-ingress-custom" "${namespace}" || return 1 + + # Create custom internal issuer (for internal services) + create_custom_issuer "rootca-internal-custom" "${namespace}" || return 1 + + echo "Custom certificate infrastructure setup complete" +} + +# Create route certificates for barbican and placement +# Usage: create_barbican_placement_routes +function create_barbican_placement_routes() { + local ingress_domain="${1}" + local namespace="${2:-openstack-kuttl-tests}" + + if [[ -z "${ingress_domain}" ]]; then + echo "Error: ingress_domain is required" + echo "Usage: create_barbican_placement_routes [namespace]" + return 1 + fi + + echo "Creating route certificates for barbican and placement..." + + # Setup infrastructure if needed + if ! oc get issuer rootca-ingress-custom -n ${namespace} &>/dev/null; then + setup_custom_certificate_infrastructure "${namespace}" || return 1 + fi + + # Create certificates for both services + create_service_route_certificate "barbican" "${ingress_domain}" "${namespace}" || return 1 + create_service_route_certificate "placement" "${ingress_domain}" "${namespace}" || return 1 + + echo "Route certificates created successfully" + echo " - barbican-custom-route" + echo " - placement-custom-route" +} + +# Cleanup custom certificates and issuers +# Usage: cleanup_custom_certificates +function cleanup_custom_certificates() { + local namespace="${1:-openstack-kuttl-tests}" + + echo "Cleaning up custom certificates and issuers in namespace ${namespace}..." + + # Delete certificates + oc delete certificate --all -n ${namespace} --ignore-not-found=true + + # Delete issuers + oc delete issuer --all -n ${namespace} --ignore-not-found=true + + # Delete route secrets + oc delete secret -l custom-route=true -n ${namespace} --ignore-not-found=true + + echo "Cleanup complete" +} diff --git a/tests/kuttl/common/osp_check_route_cert.sh b/tests/kuttl/common/osp_check_route_cert.sh index 7ee8ad633..a24989fad 100755 --- a/tests/kuttl/common/osp_check_route_cert.sh +++ b/tests/kuttl/common/osp_check_route_cert.sh @@ -2,35 +2,8 @@ ROUTE_NAME=$1 -if [[ "$ROUTE_NAME" == "barbican" ]]; then - EXPECTED_CERTIFICATE="-----BEGIN CERTIFICATE----- -MIIBfjCCASUCFB2bFw2MfRB0vIAZmNe81aRJIj8GMAoGCCqGSM49BAMCMEIxCzAJ -BgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNVBAoME0RlZmF1 -bHQgQ29tcGFueSBMdGQwHhcNMjQwNDI5MDc0NzM4WhcNMjUwNDI5MDc0NzM4WjBC -MQswCQYDVQQGEwJYWDEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5MRwwGgYDVQQKDBNE -ZWZhdWx0IENvbXBhbnkgTHRkMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAExuPx -YX5SrJpymPX/j67XXiHiSte8S6wzbuJd0xM4HODaZbt0r0IWO8rRQUEeLnudFoF6 -Ju4R3QUT8d+t3zDZqTAKBggqhkjOPQQDAgNHADBEAiBYsG23KRf/3o+R9+imWJa4 -8d2zu9eMr6qkTg5Q4tj/sgIgLaox7QK2Ao4dY3nmlyg9ascsQNKV4bpdSXAX/drH -7DE= ------END CERTIFICATE-----" - - EXPECTED_CA_CERTIFICATE="-----BEGIN CERTIFICATE----- -MIIB2DCCAX+gAwIBAgIUWARgHwuaus9w7uQ1opRlDXRPRvswCgYIKoZIzj0EAwIw -QjELMAkGA1UEBhMCWFgxFTATBgNVBAcMDERlZmF1bHQgQ2l0eTEcMBoGA1UECgwT -RGVmYXVsdCBDb21wYW55IEx0ZDAeFw0yNDA0MjkwNzQ2NDdaFw0yNTA0MjkwNzQ2 -NDdaMEIxCzAJBgNVBAYTAlhYMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxHDAaBgNV -BAoME0RlZmF1bHQgQ29tcGFueSBMdGQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNC -AATg695MQRE7erKhHfNL0MA+2bve1TruxjNm5kUJaCk8CbQWififn9WjHlPnND6x -ftxDyis5MyTvdjpYnzZTxSEwo1MwUTAdBgNVHQ4EFgQUVkjtNnURJ4c+kvZn7/fF -EJhMz7QwHwYDVR0jBBgwFoAUVkjtNnURJ4c+kvZn7/fFEJhMz7QwDwYDVR0TAQH/ -BAUwAwEB/zAKBggqhkjOPQQDAgNHADBEAiAuExPY3JFBuZ1IFv5Sf+Ai3YHwtoSb -PzKC6Dq0YRpNqgIgTTK9yfJpiJSHwJqa2dWnD8dJkf8jKH+6/VJrRk3mpkY= ------END CERTIFICATE-----" -elif [[ "$ROUTE_NAME" == "placement" ]]; then - EXPECTED_CERTIFICATE="CERT123" - EXPECTED_CA_CERTIFICATE="CACERT123" -fi +EXPECTED_CERTIFICATE=$(oc get secret ${ROUTE_NAME}-custom-route -n $NAMESPACE -o jsonpath='{.data.tls\.crt}' | base64 -d) +EXPECTED_CA_CERTIFICATE=$(oc get secret ${ROUTE_NAME}-custom-route -n $NAMESPACE -o jsonpath='{.data.ca\.crt}' | base64 -d) TLS_DATA=$(oc get route ${ROUTE_NAME}-public -n $NAMESPACE -o jsonpath='{.spec.tls}') diff --git a/tests/kuttl/common/prepare_placement_certs.sh b/tests/kuttl/common/prepare_placement_certs.sh new file mode 100755 index 000000000..3f45acbe5 --- /dev/null +++ b/tests/kuttl/common/prepare_placement_certs.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# Prepare certificate ConfigMap for kustomize from placement-custom-route secret +# + +set -e + +if [[ -z "${NAMESPACE}" ]]; then + echo "Error: NAMESPACE environment variable must be set" + exit 1 +fi + +# Wait for the secret to be created (retry up to 3 times with 10s wait) +echo "Waiting for placement-custom-route secret to be created..." +MAX_RETRIES=3 +RETRY_COUNT=0 +SECRET_EXISTS=false + +while [[ ${RETRY_COUNT} -lt ${MAX_RETRIES} ]]; do + if oc get secret placement-custom-route -n ${NAMESPACE} &>/dev/null; then + echo "Secret placement-custom-route found in namespace ${NAMESPACE}" + SECRET_EXISTS=true + break + fi + + RETRY_COUNT=$((RETRY_COUNT + 1)) + if [[ ${RETRY_COUNT} -lt ${MAX_RETRIES} ]]; then + echo "Secret not found yet, waiting 10s... (attempt ${RETRY_COUNT}/${MAX_RETRIES})" + sleep 10 + fi +done + +if [[ "${SECRET_EXISTS}" != "true" ]]; then + echo "Error: Secret placement-custom-route not found in namespace ${NAMESPACE} after ${MAX_RETRIES} attempts" + exit 1 +fi + +echo "Fetching certificates from placement-custom-route secret and creating ConfigMap..." + +# Get the directory where the script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Calculate the path to the kustomize directory +KUSTOMIZE_DIR="${SCRIPT_DIR}/../../../config/samples/tls/custom_route_cert" +CONFIGMAP_FILE="${KUSTOMIZE_DIR}/placement-cert-data.yaml" + +echo "Creating ConfigMap file at: ${CONFIGMAP_FILE}" + +cat > "${CONFIGMAP_FILE}" << EOF +apiVersion: v1 +kind: ConfigMap +metadata: + name: placement-cert-data + namespace: ${NAMESPACE} +data: + tls.crt: | +$(oc get secret placement-custom-route -n ${NAMESPACE} -o jsonpath='{.data.tls\.crt}' | base64 -d | sed 's/^/ /') + tls.key: | +$(oc get secret placement-custom-route -n ${NAMESPACE} -o jsonpath='{.data.tls\.key}' | base64 -d | sed 's/^/ /') + ca.crt: | +$(oc get secret placement-custom-route -n ${NAMESPACE} -o jsonpath='{.data.ca\.crt}' | base64 -d | sed 's/^/ /') +EOF + +# Also apply it to the cluster for verification +oc apply -f "${CONFIGMAP_FILE}" + +echo "ConfigMap placement-cert-data created at ${CONFIGMAP_FILE} and applied to namespace ${NAMESPACE}" +echo "This file will be used by kustomize as a resource" diff --git a/tests/kuttl/common/verify_route_override_certs.sh b/tests/kuttl/common/verify_route_override_certs.sh new file mode 100755 index 000000000..e864f6bff --- /dev/null +++ b/tests/kuttl/common/verify_route_override_certs.sh @@ -0,0 +1,71 @@ +#!/bin/bash +# +# Verify that route override certificates in OpenStackControlPlane match the secret +# + +set -e + +SERVICE_NAME=$1 +NAMESPACE=${NAMESPACE:-openstack-kuttl-tests} + +if [[ -z "${SERVICE_NAME}" ]]; then + echo "ERROR: Service name is required" + echo "Usage: $0 " + exit 1 +fi + +echo "Verifying ${SERVICE_NAME} route override certificates..." + +# Get expected certificate values from the secret +EXPECTED_CERT=$(oc get secret ${SERVICE_NAME}-custom-route -n ${NAMESPACE} -o jsonpath='{.data.tls\.crt}' | base64 -d) +EXPECTED_KEY=$(oc get secret ${SERVICE_NAME}-custom-route -n ${NAMESPACE} -o jsonpath='{.data.tls\.key}' | base64 -d) +EXPECTED_CA=$(oc get secret ${SERVICE_NAME}-custom-route -n ${NAMESPACE} -o jsonpath='{.data.ca\.crt}' | base64 -d) + +if [[ -z "${EXPECTED_CERT}" ]] || [[ -z "${EXPECTED_KEY}" ]] || [[ -z "${EXPECTED_CA}" ]]; then + echo "ERROR: Failed to fetch certificate data from ${SERVICE_NAME}-custom-route secret" + exit 1 +fi + +# Get actual certificate values from OpenStackControlPlane +ACTUAL_CERT=$(oc get openstackcontrolplane openstack -n ${NAMESPACE} -o jsonpath="{.spec.${SERVICE_NAME}.apiOverride.route.spec.tls.certificate}") +ACTUAL_KEY=$(oc get openstackcontrolplane openstack -n ${NAMESPACE} -o jsonpath="{.spec.${SERVICE_NAME}.apiOverride.route.spec.tls.key}") +ACTUAL_CA=$(oc get openstackcontrolplane openstack -n ${NAMESPACE} -o jsonpath="{.spec.${SERVICE_NAME}.apiOverride.route.spec.tls.caCertificate}") + +if [[ -z "${ACTUAL_CERT}" ]] || [[ -z "${ACTUAL_KEY}" ]] || [[ -z "${ACTUAL_CA}" ]]; then + echo "ERROR: Failed to fetch certificate data from OpenStackControlPlane ${SERVICE_NAME} override" + exit 1 +fi + +# Trim whitespace for comparison +TRIMMED_EXPECTED_CERT=$(echo "$EXPECTED_CERT" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') +TRIMMED_ACTUAL_CERT=$(echo "$ACTUAL_CERT" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + +TRIMMED_EXPECTED_KEY=$(echo "$EXPECTED_KEY" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') +TRIMMED_ACTUAL_KEY=$(echo "$ACTUAL_KEY" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + +TRIMMED_EXPECTED_CA=$(echo "$EXPECTED_CA" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') +TRIMMED_ACTUAL_CA=$(echo "$ACTUAL_CA" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') + +# Compare certificates +if [[ "$TRIMMED_EXPECTED_CERT" != "$TRIMMED_ACTUAL_CERT" ]]; then + echo "ERROR: Certificate does not match for ${SERVICE_NAME} in OpenStackControlPlane" + echo "Expected cert from secret (first 100 chars): ${TRIMMED_EXPECTED_CERT:0:100}" + echo "Actual cert from controlplane (first 100 chars): ${TRIMMED_ACTUAL_CERT:0:100}" + exit 1 +fi + +# Compare keys +if [[ "$TRIMMED_EXPECTED_KEY" != "$TRIMMED_ACTUAL_KEY" ]]; then + echo "ERROR: Private key does not match for ${SERVICE_NAME} in OpenStackControlPlane" + exit 1 +fi + +# Compare CA certificates +if [[ "$TRIMMED_EXPECTED_CA" != "$TRIMMED_ACTUAL_CA" ]]; then + echo "ERROR: CA certificate does not match for ${SERVICE_NAME} in OpenStackControlPlane" + echo "Expected CA from secret (first 100 chars): ${TRIMMED_EXPECTED_CA:0:100}" + echo "Actual CA from controlplane (first 100 chars): ${TRIMMED_ACTUAL_CA:0:100}" + exit 1 +fi + +echo "✓ All certificates match for ${SERVICE_NAME}" diff --git a/tests/kuttl/tests/ctlplane-tls-custom-route/02-assert-custom-route-secret.yaml b/tests/kuttl/tests/ctlplane-tls-custom-route/02-assert-custom-route-secret.yaml index decb7bfaf..00702b298 100644 --- a/tests/kuttl/tests/ctlplane-tls-custom-route/02-assert-custom-route-secret.yaml +++ b/tests/kuttl/tests/ctlplane-tls-custom-route/02-assert-custom-route-secret.yaml @@ -2,3 +2,13 @@ apiVersion: v1 kind: Secret metadata: name: barbican-custom-route +--- +apiVersion: v1 +kind: Secret +metadata: + name: placement-custom-route +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: placement-cert-data diff --git a/tests/kuttl/tests/ctlplane-tls-custom-route/02-deploy-custom-route-secret.yaml b/tests/kuttl/tests/ctlplane-tls-custom-route/02-deploy-custom-route-secret.yaml index 14981897f..42ca90ad5 100644 --- a/tests/kuttl/tests/ctlplane-tls-custom-route/02-deploy-custom-route-secret.yaml +++ b/tests/kuttl/tests/ctlplane-tls-custom-route/02-deploy-custom-route-secret.yaml @@ -2,4 +2,10 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - script: | - oc apply -n $NAMESPACE -f ../../common/custom-barbican-route.yaml + source ../../common/create_custom_cert.sh + INGRESS_DOMAIN=$(oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}') + create_barbican_placement_routes "${INGRESS_DOMAIN}" "${NAMESPACE}" + + - script: | + # Generate ConfigMap for kustomize from the placement-custom-route secret + bash ../../common/prepare_placement_certs.sh diff --git a/tests/kuttl/tests/ctlplane-tls-custom-route/03-assert-deploy-openstack.yaml b/tests/kuttl/tests/ctlplane-tls-custom-route/03-assert-deploy-openstack.yaml index 12a8436a7..5ce054f6a 100644 --- a/tests/kuttl/tests/ctlplane-tls-custom-route/03-assert-deploy-openstack.yaml +++ b/tests/kuttl/tests/ctlplane-tls-custom-route/03-assert-deploy-openstack.yaml @@ -34,12 +34,6 @@ spec: route: spec: tls: - certificate: | - CERT123 - key: | - KEY123 - caCertificate: | - CACERT123 termination: reencrypt template: databaseInstance: openstack diff --git a/tests/kuttl/tests/ctlplane-tls-custom-route/03-deploy-openstack.yaml b/tests/kuttl/tests/ctlplane-tls-custom-route/03-deploy-openstack.yaml index d5ab11869..0697a3b58 100644 --- a/tests/kuttl/tests/ctlplane-tls-custom-route/03-deploy-openstack.yaml +++ b/tests/kuttl/tests/ctlplane-tls-custom-route/03-deploy-openstack.yaml @@ -2,4 +2,5 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - script: | + # Apply OpenStackControlPlane with kustomize (ConfigMap was created in step 02) oc kustomize ../../../../config/samples/tls/custom_route_cert | oc apply -n $NAMESPACE -f - diff --git a/tests/kuttl/tests/ctlplane-tls-custom-route/04-assert-custom-route-cert.yaml b/tests/kuttl/tests/ctlplane-tls-custom-route/04-assert-custom-route-cert.yaml index c0794fb78..b2c16ec5b 100644 --- a/tests/kuttl/tests/ctlplane-tls-custom-route/04-assert-custom-route-cert.yaml +++ b/tests/kuttl/tests/ctlplane-tls-custom-route/04-assert-custom-route-cert.yaml @@ -9,3 +9,7 @@ commands: - script: | echo "Checking placement custom route certificate..." bash ../../common/osp_check_route_cert.sh "placement" + + - script: | + echo "Verifying placement route override certificates in OpenStackControlPlane..." + bash ../../common/verify_route_override_certs.sh "placement" diff --git a/tests/kuttl/tests/ctlplane-tls-custom-route/README.md b/tests/kuttl/tests/ctlplane-tls-custom-route/README.md new file mode 100644 index 000000000..39fcb1b22 --- /dev/null +++ b/tests/kuttl/tests/ctlplane-tls-custom-route/README.md @@ -0,0 +1,509 @@ +# TLS Custom Route Certificate Test + +## Overview + +This kuttl test validates custom TLS certificate management for OpenStackControlPlane using dynamically generated certificates from cert-manager. The test specifically validates: + +- **Barbican** - Uses secret reference for TLS configuration +- **Placement** - Uses inline certificates injected via Kustomize replacements + +The test uses a hybrid approach combining cert-manager, bash scripts, ConfigMaps, and Kustomize replacements to achieve dynamic certificate injection without modifying source files. + +## Quick Start + +### Running the Test + +```bash +export NAMESPACE="openstack-kuttl-tests" +kubectl kuttl test --test ctlplane-tls-custom-route +``` + +### Manual Certificate Setup + +```bash +# Step 1: Create certificates +source ../../common/create_custom_cert.sh +INGRESS_DOMAIN=$(oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}') +create_barbican_placement_routes "${INGRESS_DOMAIN}" "${NAMESPACE}" + +# Step 2: Create ConfigMap for kustomize +bash ../../common/prepare_placement_certs.sh + +# Step 3: Apply OpenStackControlPlane with kustomize +oc kustomize ../../../../config/samples/tls/custom_route_cert | oc apply -n $NAMESPACE -f - + +# Step 4: Verify certificates +bash ../../common/osp_check_route_cert.sh barbican +bash ../../common/osp_check_route_cert.sh placement +bash ../../common/verify_route_override_certs.sh placement +``` + +## Test Structure + +``` +tests/kuttl/tests/ctlplane-tls-custom-route/ +├── 01-deploy-openstack.yaml # Initial OpenStack deployment +├── 01-assert-deploy-openstack.yaml +├── 02-deploy-custom-route-secret.yaml # Create certificates and ConfigMap +├── 02-assert-custom-route-secret.yaml +├── 03-deploy-openstack.yaml # Apply with custom certs via kustomize +├── 03-assert-deploy-openstack.yaml +├── 04-assert-custom-route-cert.yaml # Verify routes and certificates +├── 04-errors-cleanup.yaml +└── README.md # This file +``` + +## How It Works + +### Architecture Overview + +``` +1. cert-manager generates certificates + └─> barbican-custom-route (Secret) - used by reference + └─> placement-custom-route (Secret) - extracted for inline injection + +2. prepare_placement_certs.sh extracts placement certs + └─> placement-cert-data.yaml (File for kustomize) + └─> placement-cert-data (ConfigMap in cluster for verification) + +3. Kustomize includes ConfigMap file and uses replacements to inject + └─> OpenStackControlPlane with real certificates + +4. OpenStack operators create Routes with certificates + └─> barbican-public (Route) - from secret reference + └─> placement-public (Route) - from inline certificates + +5. Verification scripts validate end-to-end +``` + +### Certificate Hierarchy + +``` +selfsigned-issuer (Self-signed Issuer) + └─> rootca-ingress-custom (CA Certificate) + └─> rootca-ingress-custom (CA Issuer) + ├─> barbican-custom-route-cert (Wildcard Certificate) + │ └─> barbican-custom-route (Secret) + └─> placement-custom-route-cert (Wildcard Certificate) + └─> placement-custom-route (Secret) + └─> placement-cert-data.yaml (File + ConfigMap) +``` + +### Step-by-Step Test Flow + +#### Step 1: Initial Deployment +Deploy OpenStackControlPlane without custom certificates. + +#### Step 2: Create Certificates and ConfigMap +```yaml +commands: + - script: | + source ../../common/create_custom_cert.sh + INGRESS_DOMAIN=$(oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}') + create_barbican_placement_routes "${INGRESS_DOMAIN}" "${NAMESPACE}" +``` + +This creates: +- `rootca-ingress-custom` (CA and Issuer) +- `barbican-custom-route` (Secret with TLS cert) +- `placement-custom-route` (Secret with TLS cert) + +Then creates the ConfigMap: +```yaml + - script: | + bash ../../common/prepare_placement_certs.sh +``` + +This creates: +- `placement-cert-data.yaml` (File for kustomize) +- `placement-cert-data` (ConfigMap in cluster for verification) + +#### Step 3: Apply with Custom Certificates +```yaml +commands: + - script: | + oc kustomize ../../../../config/samples/tls/custom_route_cert | oc apply -n $NAMESPACE -f - +``` + +The kustomize configuration: +1. Includes the ConfigMap file (`placement-cert-data.yaml`) as a resource +2. Patches the OpenStackControlPlane with empty certificate fields +3. Uses `replacements` to inject certificate data from the ConfigMap + +#### Step 4: Verify Certificates +```yaml +commands: + - script: bash ../../common/osp_check_route_cert.sh barbican + - script: bash ../../common/osp_check_route_cert.sh placement + - script: bash ../../common/verify_route_override_certs.sh placement +``` + +## Certificate Creation Utilities + +### Available Functions + +The `../../common/create_custom_cert.sh` script provides: + +| Function | Usage | Description | +|----------|-------|-------------| +| `create_barbican_placement_routes` | ` [namespace]` | One-command setup for barbican and placement | +| `create_service_route_certificate` | ` [namespace]` | Create certificate for any service | +| `create_custom_issuer` | ` [namespace]` | Create root CA and issuer | +| `create_wildcard_certificate` | ` [issuer-name] [namespace]` | Create wildcard certificate | +| `setup_custom_certificate_infrastructure` | `[namespace]` | Setup complete cert infrastructure | +| `cleanup_custom_certificates` | `[namespace]` | Remove all custom certificates | + +### Examples + +#### Create Certificates for Any Service + +```bash +source ../../common/create_custom_cert.sh + +# For keystone +create_service_route_certificate "keystone" "apps-crc.testing" "openstack-kuttl-tests" + +# For glance +create_service_route_certificate "glance" "apps-crc.testing" "openstack-kuttl-tests" +``` + +#### Setup Complete Infrastructure + +```bash +# Creates both ingress and internal issuers +setup_custom_certificate_infrastructure "openstack-kuttl-tests" +``` + +## Kustomize Implementation Details + +### Location + +The kustomize configuration is in: `config/samples/tls/custom_route_cert/` + +### Files + +- **`kustomization.yaml`** - Kustomize configuration with replacements +- **`patch.yaml`** - Patch file with Barbican and Placement TLS configuration +- **`placement-cert-data.yaml`** - Generated ConfigMap file (created by `prepare_placement_certs.sh`) + +### How Replacements Work + +The kustomization includes the ConfigMap file as a resource: + +```yaml +resources: +- placement-cert-data.yaml +``` + +Then uses replacements to inject the certificate data: + +```yaml +replacements: +- source: + kind: ConfigMap + name: placement-cert-data + fieldPath: data.[tls.crt] + targets: + - select: + kind: OpenStackControlPlane + fieldPaths: + - spec.placement.apiOverride.route.spec.tls.certificate +``` + +This replaces the empty `certificate: ""` field in the patch with the actual certificate data from the ConfigMap resource. + +### Key Features + +- ✅ Uses dynamically generated certificates (no hardcoded values) +- ✅ Kustomize-native approach with replacements +- ✅ ConfigMap file is generated (not checked into git) +- ✅ ConfigMap also applied to cluster for inspection/debugging +- ✅ Clean separation: script generates data, kustomize applies + +### OpenStackControlPlane Configuration + +#### Barbican (Secret Reference) + +```yaml +spec: + barbican: + apiOverride: + tls: + secretName: barbican-custom-route +``` + +The operator reads the secret directly and applies it to the route. + +#### Placement (Inline Certificates via Kustomize) + +```yaml +spec: + placement: + apiOverride: + route: + spec: + tls: + certificate: + key: + caCertificate: + termination: reencrypt +``` + +The certificates are injected inline using kustomize replacements from the ConfigMap. + +## Verification and Debugging + +### View Generated ConfigMap File + +```bash +cat config/samples/tls/custom_route_cert/placement-cert-data.yaml +``` + +### View ConfigMap in Cluster + +```bash +oc get configmap placement-cert-data -n $NAMESPACE -o yaml +``` + +### Test Kustomize Output + +```bash +# See the full output +oc kustomize config/samples/tls/custom_route_cert | less + +# Check just the placement TLS config +oc kustomize config/samples/tls/custom_route_cert | yq '.spec.placement.apiOverride.route.spec.tls' +``` + +### Check Certificate Status + +```bash +# View all certificates +oc get certificate -n $NAMESPACE + +# Describe specific certificate +oc describe certificate barbican-custom-route-cert -n $NAMESPACE + +# View secret content +oc get secret barbican-custom-route -n $NAMESPACE -o yaml +``` + +### Decode Certificate + +```bash +oc get secret barbican-custom-route -n $NAMESPACE \ + -o jsonpath='{.data.tls\.crt}' | base64 -d | openssl x509 -text -noout +``` + +### Verify Route TLS + +```bash +# Check route configuration +oc get route barbican-public -n $NAMESPACE -o jsonpath='{.spec.tls}' | jq + +# Verify certificates match (uses dynamic validation) +bash ../../common/osp_check_route_cert.sh barbican +bash ../../common/osp_check_route_cert.sh placement +``` + +### Verify OpenStackControlPlane Overrides + +```bash +# Verify placement certificates in OpenStackControlPlane match the secret +bash ../../common/verify_route_override_certs.sh placement +``` + +## Certificate Specifications + +### Root CA Certificate +- **Algorithm:** ECDSA P-256 +- **Duration:** 87600h (10 years) +- **Usage:** Certificate Sign, CRL Sign +- **IsCA:** true + +### Service Certificates +- **Algorithm:** ECDSA P-256 +- **Duration:** 8760h (1 year) +- **Renewal:** 720h (30 days) before expiration +- **Usage:** Server Auth, Client Auth +- **DNS Names:** `*.domain.com`, `domain.com` + +## Troubleshooting + +### Certificate Not Ready + +**Problem:** Certificate remains in pending state + +**Solution:** +```bash +# Check certificate status +oc describe certificate -n $NAMESPACE + +# Check cert-manager logs +oc logs -n cert-manager -l app=cert-manager --tail=50 +``` + +### ConfigMap File Not Found During Kustomize + +**Error:** Kustomize fails with `placement-cert-data.yaml` not found + +**Solution:** Run `prepare_placement_certs.sh` first to generate the ConfigMap file: +```bash +bash ../../common/prepare_placement_certs.sh +``` + +This creates `config/samples/tls/custom_route_cert/placement-cert-data.yaml` and also applies it to the cluster. + +### Failed to Fetch Certificate Data + +**Error:** `ERROR: Failed to fetch certificate data from placement-custom-route secret` + +**Solution:** Ensure `placement-custom-route` secret exists: +```bash +oc get secret placement-custom-route -n $NAMESPACE + +# If not, create it: +source ../../common/create_custom_cert.sh +INGRESS_DOMAIN=$(oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}') +create_barbican_placement_routes "${INGRESS_DOMAIN}" "${NAMESPACE}" +``` + +### Certificates Not Populated + +**Solution:** +1. Check that prepare_placement_certs.sh ran successfully +2. Verify the ConfigMap file was created: + ```bash + cat config/samples/tls/custom_route_cert/placement-cert-data.yaml + ``` +3. Verify the ConfigMap was applied to the cluster: + ```bash + oc get configmap placement-cert-data -n $NAMESPACE -o yaml + ``` +4. Ensure namespace in ConfigMap matches your target namespace +5. Check that the ConfigMap has the expected data keys: `tls.crt`, `tls.key`, `ca.crt` + +### Route Not Using Custom Certificate + +**Problem:** Route is using wrong certificate + +**Solution:** +```bash +# Verify secret exists +oc get secret -custom-route -n $NAMESPACE + +# Check route configuration +oc get route -public -n $NAMESPACE -o yaml + +# Verify OpenStackControlPlane configuration +oc get openstackcontrolplane -n $NAMESPACE -o yaml | grep -A 10 "secretName:" +``` + +### Certificate Mismatch + +**Error:** `ERROR: Certificate does not match for placement in OpenStackControlPlane` + +**Solution:** +```bash +# Verify secret data +oc get secret placement-custom-route -n $NAMESPACE -o jsonpath='{.data.tls\.crt}' | base64 -d + +# Check OpenStackControlPlane +oc get openstackcontrolplane openstack -n $NAMESPACE -o yaml + +# Reapply with kustomize +oc kustomize config/samples/tls/custom_route_cert | oc apply -n $NAMESPACE -f - +``` + +### Issuer Not Ready + +**Problem:** Issuer shows "Ready" condition as "False" + +**Solution:** +```bash +# Check issuer details +oc describe issuer -n $NAMESPACE + +# Ensure CA secret exists +oc get secret -n $NAMESPACE + +# Verify self-signed issuer +oc get issuer selfsigned-issuer -n $NAMESPACE +``` + +### Script Execution Issues + +**Error:** `Permission denied` + +**Solution:** +```bash +chmod +x ../../common/prepare_placement_certs.sh +chmod +x ../../common/verify_route_override_certs.sh +chmod +x ../../common/osp_check_route_cert.sh +``` + +## Extending for Other Services + +To add dynamic certificate support for another service (e.g., keystone): + +1. **Create the secret:** + ```bash + source ../../common/create_custom_cert.sh + INGRESS_DOMAIN=$(oc get ingresses.config.openshift.io cluster -o jsonpath='{.spec.domain}') + create_service_route_certificate "keystone" "${INGRESS_DOMAIN}" "${NAMESPACE}" + ``` + +2. **Update kustomize patch** to include keystone TLS configuration + +3. **Update prepare_*.sh script** if needed for inline certificates + +4. **Add verification:** + ```yaml + - script: | + bash ../../common/osp_check_route_cert.sh "keystone" + ``` + +## Cleanup + +```bash +# Remove all custom certificates and issuers +source ../../common/create_custom_cert.sh +cleanup_custom_certificates "$NAMESPACE" + +# Remove generated ConfigMap file +rm -f config/samples/tls/custom_route_cert/placement-cert-data.yaml + +# Or manually +oc delete certificate --all -n $NAMESPACE +oc delete issuer --all -n $NAMESPACE +oc delete secret -l cert-manager.io/common-name -n $NAMESPACE +oc delete configmap placement-cert-data -n $NAMESPACE +``` + +## Security Considerations + +⚠️ **Important:** These certificates are for **TESTING PURPOSES ONLY** + +## Files Reference + +### Test Files +- `01-deploy-openstack.yaml` - Initial deployment +- `02-deploy-custom-route-secret.yaml` - Create certificates and ConfigMap +- `03-deploy-openstack.yaml` - Apply with kustomize +- `04-assert-custom-route-cert.yaml` - Verify certificates + +### Common Utilities +- `../../common/create_custom_cert.sh` - Certificate creation functions +- `../../common/prepare_placement_certs.sh` - ConfigMap generation +- `../../common/osp_check_route_cert.sh` - Route certificate validation +- `../../common/verify_route_override_certs.sh` - OpenStackControlPlane validation + +### Kustomize Files +- `../../../../config/samples/tls/custom_route_cert/kustomization.yaml` - Kustomize config +- `../../../../config/samples/tls/custom_route_cert/patch.yaml` - TLS patch + +## Related Documentation + +- [Cert-Manager Documentation](https://cert-manager.io/docs/) +- [Kustomize Documentation](https://kustomize.io/) +- OpenStack Operator TLS Documentation