From 3f1a3e7ddef505615897e2cfc199efca76ea13ec Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Fri, 8 Aug 2025 16:31:06 -0400 Subject: [PATCH 01/16] Deploy NetBox with slurpit_netbox plugin using init container - Add init container to install slurpit_netbox plugin via pip - Enable slurpit_netbox plugin in NetBox configuration - Use shared volume to provide plugins to NetBox container - Update PYTHONPATH to include plugins directory --- fleet/lib/netbox/README.md | 59 +++++ .../base/externalsecret-netbox-keycloak.yaml | 21 ++ .../externalsecret-netbox-postgresql.yaml | 21 ++ .../base/externalsecret-netbox-secrets.yaml | 33 +++ .../base/externalsecret-netbox-valkey.yaml | 17 ++ fleet/lib/netbox/base/kustomization.yaml | 12 + .../netbox/base/netbox-plugin-init-patch.yaml | 51 +++++ fleet/lib/netbox/fleet.yaml | 17 ++ fleet/lib/netbox/values.yaml | 205 ++++++++++++++++++ fleet/s/dev/c/kueyen/netbox | 1 + 10 files changed, 437 insertions(+) create mode 100644 fleet/lib/netbox/README.md create mode 100644 fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml create mode 100644 fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml create mode 100644 fleet/lib/netbox/base/externalsecret-netbox-secrets.yaml create mode 100644 fleet/lib/netbox/base/externalsecret-netbox-valkey.yaml create mode 100644 fleet/lib/netbox/base/kustomization.yaml create mode 100644 fleet/lib/netbox/base/netbox-plugin-init-patch.yaml create mode 100644 fleet/lib/netbox/fleet.yaml create mode 100644 fleet/lib/netbox/values.yaml create mode 120000 fleet/s/dev/c/kueyen/netbox diff --git a/fleet/lib/netbox/README.md b/fleet/lib/netbox/README.md new file mode 100644 index 000000000..e7f56b0d7 --- /dev/null +++ b/fleet/lib/netbox/README.md @@ -0,0 +1,59 @@ +# NetBox Kubernetes Deployment + +## Overview + +NetBox is an Infrastructure Resource Modeling (IRM) application designed to empower network automation. This deployment provides a production-ready NetBox instance on Kubernetes using Helm charts and Fleet configuration management. + +## Architecture + +- **Chart**: netbox v6.1.5 from +- **Namespace**: netbox +- **Components**: Web application, worker processes, PostgreSQL database, Valkey cache +- **Ingress**: NGINX with Let's Encrypt TLS certificates + +## Configuration + +### Core Settings + +- **Timezone**: America/Santiago +- **Superuser**: +- **Change Log Retention**: 90 days +- **Job Retention**: 90 days +- **GraphQL**: Enabled +- **Login Required**: False + +### Security + +- Non-root container execution (UID/GID: 1000) +- Read-only root filesystem +- Dropped capabilities +- Runtime security profile enabled +- External secret management via Kubernetes secrets + +### Storage + +- **Persistence**: Disabled (ephemeral storage) +- **PostgreSQL**: 20Gi persistent storage (rook-ceph-block) +- **Media/Reports/Scripts**: Stored in ephemeral volumes + +### Resources + +| Component | CPU Request | Memory Request | CPU Limit | Memory Limit | +|-----------|-------------|----------------|-----------|--------------| +| NetBox | 500m | 1Gi | 1000m | 2Gi | +| Worker | 500m | 1Gi | 1000m | 2Gi | +| PostgreSQL| 250m | 512Mi | 500m | 1Gi | + +## Access + +NetBox is accessible via dynamically generated hostnames based on cluster configuration: + +```bash +https://netbox.{cluster-name}.{site}.lsst.org +``` + +## Maintenance + +- **Housekeeping**: Daily automated cleanup (00:00 UTC) +- **Job History**: 5 successful/failed jobs retained +- **Monitoring**: Available via cluster monitoring stack diff --git a/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml new file mode 100644 index 000000000..d33fc204e --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml @@ -0,0 +1,21 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-keycloak + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-keycloak + creationPolicy: Owner + data: + - secretKey: SOCIAL_AUTH_KEYCLOAK_KEY + remoteRef: + key: &item netbox-keycloak-sso + property: client-id + - secretKey: SOCIAL_AUTH_KEYCLOAK_SECRET + remoteRef: + key: *item + property: client-secret diff --git a/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml new file mode 100644 index 000000000..6a137bb18 --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml @@ -0,0 +1,21 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-postgresql + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-postgresql + creationPolicy: Owner + data: + - secretKey: password + remoteRef: + key: &item netbox-postgresql + property: password + - secretKey: postgres-password + remoteRef: + key: *item + property: postgres-password diff --git a/fleet/lib/netbox/base/externalsecret-netbox-secrets.yaml b/fleet/lib/netbox/base/externalsecret-netbox-secrets.yaml new file mode 100644 index 000000000..a6e25c74d --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-secrets.yaml @@ -0,0 +1,33 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-secrets + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-secrets + creationPolicy: Owner + data: + - secretKey: username + remoteRef: + key: &item netbox-secrets + property: username + - secretKey: password + remoteRef: + key: *item + property: password + - secretKey: email + remoteRef: + key: *item + property: email + - secretKey: api_token + remoteRef: + key: *item + property: apiToken + - secretKey: secret-key + remoteRef: + key: *item + property: secretKey diff --git a/fleet/lib/netbox/base/externalsecret-netbox-valkey.yaml b/fleet/lib/netbox/base/externalsecret-netbox-valkey.yaml new file mode 100644 index 000000000..e91338163 --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-valkey.yaml @@ -0,0 +1,17 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-valkey + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-valkey + creationPolicy: Owner + data: + - secretKey: valkey-password + remoteRef: + key: &item netbox-valkey + property: password diff --git a/fleet/lib/netbox/base/kustomization.yaml b/fleet/lib/netbox/base/kustomization.yaml new file mode 100644 index 000000000..3aaa5138a --- /dev/null +++ b/fleet/lib/netbox/base/kustomization.yaml @@ -0,0 +1,12 @@ +--- +resources: + - externalsecret-netbox-secrets.yaml + - externalsecret-netbox-valkey.yaml + - externalsecret-netbox-postgresql.yaml + - externalsecret-netbox-keycloak.yaml + +patches: + - path: netbox-plugin-init-patch.yaml + target: + kind: Deployment + name: netbox diff --git a/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml b/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml new file mode 100644 index 000000000..0d1aa42a2 --- /dev/null +++ b/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: netbox +spec: + template: + spec: + # Add shared volume for plugins + volumes: + - name: plugins-volume + emptyDir: {} + # Init container to install slurpit_netbox plugin + initContainers: + - name: install-plugins + image: python:3.11-slim + command: + - /bin/bash + - -c + - | + set -e + echo "Installing slurpit_netbox plugin..." + + # Install pip if not available + python -m ensurepip --upgrade + + # Install the slurpit_netbox plugin to shared volume + pip install --target /opt/plugins slurpit_netbox + + echo "Plugin installation completed" + ls -la /opt/plugins/ + volumeMounts: + - name: plugins-volume + mountPath: /opt/plugins + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + containers: + - name: netbox + # Mount the plugins volume + volumeMounts: + - name: plugins-volume + mountPath: /opt/netbox/plugins + # Update PYTHONPATH to include plugins directory + env: + - name: PYTHONPATH + value: "/opt/netbox/plugins:/opt/netbox" \ No newline at end of file diff --git a/fleet/lib/netbox/fleet.yaml b/fleet/lib/netbox/fleet.yaml new file mode 100644 index 000000000..fc970ed7d --- /dev/null +++ b/fleet/lib/netbox/fleet.yaml @@ -0,0 +1,17 @@ +--- +defaultNamespace: &name netbox +labels: + bundle: *name +namespaceLabels: + lsst.io/discover: "true" +kustomize: + dir: base +helm: + chart: &chart netbox + releaseName: *chart + repo: https://charts.netbox.oss.netboxlabs.com/ + version: 6.1.5 + timeoutSeconds: 600 + waitForJobs: true + valuesFiles: + - values.yaml diff --git a/fleet/lib/netbox/values.yaml b/fleet/lib/netbox/values.yaml new file mode 100644 index 000000000..edae054f6 --- /dev/null +++ b/fleet/lib/netbox/values.yaml @@ -0,0 +1,205 @@ +nameOverride: netbox +clusterDomain: cluster.local + +superuser: + name: admin + email: admin@lsst.org + existingSecret: netbox-secrets + + +allowedHosts: + - netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org + - netbox.kueyen.dev.lsst.org + +allowedHostsIncludesPodIP: false + +admins: + - [Admin User, admin@tudominio.com] + +internalIPs: [127.0.0.1] + +timeZone: America/Santiago + +# Disable persistent volumes to avoid multi-attach issues with ReadWriteOnce storage +# Media files will be stored in ephemeral storage +persistence: + enabled: false + # storageClass: rook-ceph-block + # accessMode: ReadWriteOnce + # size: 10Gi + # annotations: {} + +reportsPersistence: + enabled: false + # storageClass: rook-ceph-block + # accessMode: ReadWriteOnce + # size: 1Gi + # annotations: {} + +scriptsPersistence: + enabled: false + # storageClass: rook-ceph-block + # accessMode: ReadWriteOnce + # size: 1Gi + # annotations: {} + +resourcesPreset: medium +resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + +## @section Traffic Exposure Parameters + +ingress: + enabled: true + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt + nginx.ingress.kubernetes.io/backend-protocol: HTTP + nginx.ingress.kubernetes.io/client-body-buffer-size: 10m + nginx.ingress.kubernetes.io/proxy-read-timeout: "60" + nginx.ingress.kubernetes.io/proxy-send-timeout: "60" + hosts: + - host: netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org + paths: + - / + tls: + - secretName: netbox-tls + hosts: + - netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org + +postgresql: + enabled: true + # Use default PostgreSQL version from chart (bitnami image tags are inconsistent) + auth: + username: netbox + database: netbox + # Dummy passwords - these are overridden by existingSecret + # Required by Bitnami chart to prevent upgrade errors + password: "unused" + postgresPassword: "unused" + existingSecret: netbox-postgresql + secretKeys: + adminPasswordKey: postgres-password + userPasswordKey: password + # Use external secret passwords for consistency + usePasswordFiles: false + # Ensure PostgreSQL is ready before Netbox starts + primary: + persistence: + enabled: true + storageClass: rook-ceph-block + size: 20Gi + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + # Prevent automatic restarts during upgrades + podDisruptionBudget: + create: true + minAvailable: 1 + # Use RollingUpdate strategy (Recreate is not valid for StatefulSets) + updateStrategy: + type: RollingUpdate + # Global PostgreSQL configuration to prevent credential mismatch + global: + postgresql: + auth: + # Dummy passwords - overridden by existingSecret + password: "unused" + postgresPassword: "unused" + existingSecret: netbox-postgresql + secretKeys: + adminPasswordKey: postgres-password + userPasswordKey: password + # Disable automatic version upgrades + upgradeRepmgrExtension: false + +valkey: + enabled: true + auth: + existingSecret: netbox-valkey + existingSecretPasswordKey: valkey-password + +## @section Worker for Netbox parameters + +worker: + enabled: true + replicaCount: 1 + # Fix kubectl image version - 1.33.3 doesn't exist + initContainer: + image: + registry: docker.io + repository: bitnami/kubectl + tag: 1.31-debian-12 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + +## @section Cron housekeeping job parameters + +housekeeping: + enabled: true + schedule: 0 0 * * * + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 + resourcesPreset: medium + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + +# Database configuration using external secrets +extraEnvs: + - name: DB_WAIT_DEBUG + value: "1" + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: netbox-secrets + key: secret-key + - name: SOCIAL_AUTH_KEYCLOAK_KEY + valueFrom: + secretKeyRef: + name: netbox-keycloak + key: SOCIAL_AUTH_KEYCLOAK_KEY + - name: SOCIAL_AUTH_KEYCLOAK_SECRET + valueFrom: + secretKeyRef: + name: netbox-keycloak + key: SOCIAL_AUTH_KEYCLOAK_SECRET + - name: SOCIAL_AUTH_KEYCLOAK_AUTHORIZATION_URL + value: "https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/auth" + - name: SOCIAL_AUTH_KEYCLOAK_ACCESS_TOKEN_URL + value: "https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/token" + - name: SOCIAL_AUTH_KEYCLOAK_ID_KEY + value: "preferred_username" + + +# Plugin configuration +plugins: [] + +## @section Remote Authentication (SSO) Parameters + +remoteAuth: + enabled: true + backends: + - social_core.backends.keycloak.KeycloakOAuth2 + +# All Keycloak configuration is now done via environment variables in extraEnvs +# NetBox will automatically read SOCIAL_AUTH_* environment variables + diff --git a/fleet/s/dev/c/kueyen/netbox b/fleet/s/dev/c/kueyen/netbox new file mode 120000 index 000000000..ccfcf159f --- /dev/null +++ b/fleet/s/dev/c/kueyen/netbox @@ -0,0 +1 @@ +../../../../lib/netbox \ No newline at end of file From 90d88705af7aaa5b8d657f83e02a987ab60c3a34 Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Mon, 24 Nov 2025 10:32:36 -0300 Subject: [PATCH 02/16] Add test files to gitignore --- fleet/lib/netbox/.gitignore | 4 ++ fleet/lib/netbox/base/kustomization.yaml | 6 --- .../netbox/base/netbox-plugin-init-patch.yaml | 51 ------------------- 3 files changed, 4 insertions(+), 57 deletions(-) create mode 100644 fleet/lib/netbox/.gitignore delete mode 100644 fleet/lib/netbox/base/netbox-plugin-init-patch.yaml diff --git a/fleet/lib/netbox/.gitignore b/fleet/lib/netbox/.gitignore new file mode 100644 index 000000000..668037e56 --- /dev/null +++ b/fleet/lib/netbox/.gitignore @@ -0,0 +1,4 @@ +GEMINI.md +values-test.yaml +CLAUDE.md +WARP.md \ No newline at end of file diff --git a/fleet/lib/netbox/base/kustomization.yaml b/fleet/lib/netbox/base/kustomization.yaml index 3aaa5138a..918c851d3 100644 --- a/fleet/lib/netbox/base/kustomization.yaml +++ b/fleet/lib/netbox/base/kustomization.yaml @@ -4,9 +4,3 @@ resources: - externalsecret-netbox-valkey.yaml - externalsecret-netbox-postgresql.yaml - externalsecret-netbox-keycloak.yaml - -patches: - - path: netbox-plugin-init-patch.yaml - target: - kind: Deployment - name: netbox diff --git a/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml b/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml deleted file mode 100644 index 0d1aa42a2..000000000 --- a/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml +++ /dev/null @@ -1,51 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: netbox -spec: - template: - spec: - # Add shared volume for plugins - volumes: - - name: plugins-volume - emptyDir: {} - # Init container to install slurpit_netbox plugin - initContainers: - - name: install-plugins - image: python:3.11-slim - command: - - /bin/bash - - -c - - | - set -e - echo "Installing slurpit_netbox plugin..." - - # Install pip if not available - python -m ensurepip --upgrade - - # Install the slurpit_netbox plugin to shared volume - pip install --target /opt/plugins slurpit_netbox - - echo "Plugin installation completed" - ls -la /opt/plugins/ - volumeMounts: - - name: plugins-volume - mountPath: /opt/plugins - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - containers: - - name: netbox - # Mount the plugins volume - volumeMounts: - - name: plugins-volume - mountPath: /opt/netbox/plugins - # Update PYTHONPATH to include plugins directory - env: - - name: PYTHONPATH - value: "/opt/netbox/plugins:/opt/netbox" \ No newline at end of file From dfa96abb11930c72879660371dc36c2c53a46854 Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Tue, 25 Nov 2025 21:57:16 -0300 Subject: [PATCH 03/16] feat: Enable Keycloak SSO for Netbox This commit finalizes the configuration for Keycloak Single Sign-On (SSO) in Netbox. The flag has been enabled to ensure secure redirection during the authentication process. This was the final step in enabling the existing Keycloak configuration. --- fleet/lib/cnpg-cluster/fleet.yaml | 10 + .../overlays/kueyen/cluster-cnpg-cluster.yaml | 46 +++++ .../overlays/kueyen/cnpg-loadBalancer.yaml | 17 ++ ...externalsecret-cnpg-cluster-superuser.yaml | 21 +++ .../kueyen/pooler-cnpg-cluster-pooler.yaml | 71 +++++++ .../base/configmap-keycloak-backend.yaml | 24 +++ .../base/externalsecret-netbox-keycloak.yaml | 4 + .../externalsecret-netbox-postgresql.yaml | 2 +- fleet/lib/netbox/base/kustomization.yaml | 1 + fleet/lib/netbox/fleet.yaml | 7 + fleet/lib/netbox/values.yaml | 175 +++++++----------- fleet/s/dev/c/kueyen/cnpg-cluster | 1 + fleet/s/dev/c/kueyen/cnpg-system | 1 + 13 files changed, 271 insertions(+), 109 deletions(-) create mode 100644 fleet/lib/cnpg-cluster/overlays/kueyen/cluster-cnpg-cluster.yaml create mode 100644 fleet/lib/cnpg-cluster/overlays/kueyen/cnpg-loadBalancer.yaml create mode 100644 fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml create mode 100644 fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml create mode 100644 fleet/lib/netbox/base/configmap-keycloak-backend.yaml create mode 120000 fleet/s/dev/c/kueyen/cnpg-cluster create mode 120000 fleet/s/dev/c/kueyen/cnpg-system diff --git a/fleet/lib/cnpg-cluster/fleet.yaml b/fleet/lib/cnpg-cluster/fleet.yaml index 9183c9cad..06e81594d 100644 --- a/fleet/lib/cnpg-cluster/fleet.yaml +++ b/fleet/lib/cnpg-cluster/fleet.yaml @@ -61,3 +61,13 @@ targetCustomizations: yaml: overlays: - yagan + - name: kueyen + clusterSelector: + matchExpressions: + - key: management.cattle.io/cluster-display-name + operator: In + values: + - kueyen + yaml: + overlays: + - kueyen diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/cluster-cnpg-cluster.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/cluster-cnpg-cluster.yaml new file mode 100644 index 000000000..171bbebfb --- /dev/null +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/cluster-cnpg-cluster.yaml @@ -0,0 +1,46 @@ +--- +# Cluster Definition +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: cnpg-cluster +spec: + imageName: docker.io/lsstit/cnpgsphere:16.8 + imagePullPolicy: Always + + instances: 3 + + postgresql: + parameters: + max_connections: "1000" + shared_buffers: 256MB + work_mem: 8MB + effective_cache_size: 768MB + idle_session_timeout: 1h + max_slot_wal_keep_size: 1GB + pg_hba: + - host replication postgres all md5 + - host all all 139.229.134.0/23 md5 + - host all all 139.229.136.0/21 md5 + - host all all 139.229.144.0/20 md5 + - host all all 139.229.160.0/19 md5 + - host all all 139.229.192.0/18 md5 + - host all all 140.252.146.0/23 md5 + + enableSuperuserAccess: true + superuserSecret: + name: cnpg-cluster-superuser + + storage: + size: 5Gi + + monitoring: + enablePodMonitor: true + + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: "1" + memory: 1Gi diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/cnpg-loadBalancer.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/cnpg-loadBalancer.yaml new file mode 100644 index 000000000..5cdca9dea --- /dev/null +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/cnpg-loadBalancer.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: cnpg-loadbalancer + labels: + cnpg.io/cluster: cnpg-cluster + annotations: + metallb.universe.tf/loadBalancerIPs: 139.229.134.89 +spec: + ports: + - name: postgresql + port: 5432 + protocol: TCP + selector: + cnpg.io/cluster: cnpg-cluster + role: primary + type: LoadBalancer diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml new file mode 100644 index 000000000..1a3dff27f --- /dev/null +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: cnpg-cluster-superuser +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + template: + type: kubernetes.io/basic-auth + data: + - secretKey: username + remoteRef: + key: cnpg-cluster-superuser-kueyen + property: username + - secretKey: password + remoteRef: + key: cnpg-cluster-superuser-kueyen + property: password diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml new file mode 100644 index 000000000..9d855bd36 --- /dev/null +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Pooler +metadata: + name: cnpg-cluster-pooler + namespace: cloudnativepg +spec: + cluster: + name: cnpg-cluster + instances: 3 + type: rw + + pgbouncer: + poolMode: session + parameters: + max_client_conn: "2000" + default_pool_size: "280" + reserve_pool_size: "20" + server_lifetime: "3600" + log_connections: "1" + log_disconnections: "1" + idle_transaction_timeout: "0" + ignore_startup_parameters: extra_float_digits + + template: + metadata: + labels: + app: pooler + lsst.io/monitor: "true" + spec: + containers: + - name: pgbouncer + resources: + requests: + cpu: "0.1" + memory: 100Mi + limits: + cpu: "0.5" + memory: 500Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: cnpg-cluster-pooler-rw + namespace: cloudnativepg + annotations: + metallb.universe.tf/loadBalancerIPs: 139.229.134.90 +spec: + type: LoadBalancer + internalTrafficPolicy: Cluster + ports: + - name: pgbouncer + port: 5432 + protocol: TCP + targetPort: 5432 + selector: + cnpg.io/poolerName: cnpg-cluster-pooler +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: cnpg-cluster-pooler + namespace: cloudnativepg + labels: + lsst.io/monitor: "true" +spec: + selector: + matchLabels: + cnpg.io/poolerName: cnpg-cluster-pooler + podMetricsEndpoints: + - port: metrics diff --git a/fleet/lib/netbox/base/configmap-keycloak-backend.yaml b/fleet/lib/netbox/base/configmap-keycloak-backend.yaml new file mode 100644 index 000000000..211ef8ad3 --- /dev/null +++ b/fleet/lib/netbox/base/configmap-keycloak-backend.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: netbox-keycloak-backend + namespace: netbox +data: + keycloak_backend.py: | + from social_core.backends.keycloak import KeycloakOAuth2 + + class KeycloakOAuth2NoJWT(KeycloakOAuth2): + """Custom Keycloak backend that skips JWT validation""" + + def public_key(self): + """Override to skip public key validation""" + return None + + def user_data(self, access_token, *args, **kwargs): + """Get user data from userinfo endpoint without JWT validation""" + # Get USERINFO URL from settings + url = self.setting('USERINFO_URL') or self.USERINFO_URL + return self.get_json( + url, + headers={'Authorization': f'Bearer {access_token}'} + ) diff --git a/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml index d33fc204e..3b750f599 100644 --- a/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml +++ b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml @@ -19,3 +19,7 @@ spec: remoteRef: key: *item property: client-secret + - secretKey: url + remoteRef: + key: *item + property: url \ No newline at end of file diff --git a/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml index 6a137bb18..33003f3b6 100644 --- a/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml +++ b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml @@ -13,7 +13,7 @@ spec: data: - secretKey: password remoteRef: - key: &item netbox-postgresql + key: &item cnpg-cluster-superuser-kueyen property: password - secretKey: postgres-password remoteRef: diff --git a/fleet/lib/netbox/base/kustomization.yaml b/fleet/lib/netbox/base/kustomization.yaml index 918c851d3..c9934ff6e 100644 --- a/fleet/lib/netbox/base/kustomization.yaml +++ b/fleet/lib/netbox/base/kustomization.yaml @@ -4,3 +4,4 @@ resources: - externalsecret-netbox-valkey.yaml - externalsecret-netbox-postgresql.yaml - externalsecret-netbox-keycloak.yaml + - configmap-keycloak-backend.yaml diff --git a/fleet/lib/netbox/fleet.yaml b/fleet/lib/netbox/fleet.yaml index fc970ed7d..8f84a8225 100644 --- a/fleet/lib/netbox/fleet.yaml +++ b/fleet/lib/netbox/fleet.yaml @@ -13,5 +13,12 @@ helm: version: 6.1.5 timeoutSeconds: 600 waitForJobs: true +dependsOn: + - selector: + matchLabels: + bundle: external-secrets + - selector: + matchLabels: + bundle: cnpg-cluster valuesFiles: - values.yaml diff --git a/fleet/lib/netbox/values.yaml b/fleet/lib/netbox/values.yaml index edae054f6..7baca85d4 100644 --- a/fleet/lib/netbox/values.yaml +++ b/fleet/lib/netbox/values.yaml @@ -3,18 +3,17 @@ clusterDomain: cluster.local superuser: name: admin - email: admin@lsst.org + email: rubbinobs-it@lsst.org existingSecret: netbox-secrets allowedHosts: - netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org - - netbox.kueyen.dev.lsst.org allowedHostsIncludesPodIP: false admins: - - [Admin User, admin@tudominio.com] + - [Admin User, rubbinobs-it@lsst.org] internalIPs: [127.0.0.1] @@ -24,26 +23,7 @@ timeZone: America/Santiago # Media files will be stored in ephemeral storage persistence: enabled: false - # storageClass: rook-ceph-block - # accessMode: ReadWriteOnce - # size: 10Gi - # annotations: {} -reportsPersistence: - enabled: false - # storageClass: rook-ceph-block - # accessMode: ReadWriteOnce - # size: 1Gi - # annotations: {} - -scriptsPersistence: - enabled: false - # storageClass: rook-ceph-block - # accessMode: ReadWriteOnce - # size: 1Gi - # annotations: {} - -resourcesPreset: medium resources: requests: cpu: 500m @@ -72,55 +52,24 @@ ingress: hosts: - netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org -postgresql: +externalDatabase: enabled: true - # Use default PostgreSQL version from chart (bitnami image tags are inconsistent) + host: cnpg-loadbalancer.cloudnativepg.svc.cluster.local + port: 5432 + user: postgres + database: netbox + existingSecret: netbox-postgresql + existingSecretPasswordKey: postgres-password + +postgresql: + enabled: false + # Explicitly disable all PostgreSQL sub-components auth: - username: netbox - database: netbox - # Dummy passwords - these are overridden by existingSecret - # Required by Bitnami chart to prevent upgrade errors - password: "unused" - postgresPassword: "unused" - existingSecret: netbox-postgresql - secretKeys: - adminPasswordKey: postgres-password - userPasswordKey: password - # Use external secret passwords for consistency - usePasswordFiles: false - # Ensure PostgreSQL is ready before Netbox starts + enabled: false primary: - persistence: - enabled: true - storageClass: rook-ceph-block - size: 20Gi - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: 500m - memory: 1Gi - # Prevent automatic restarts during upgrades - podDisruptionBudget: - create: true - minAvailable: 1 - # Use RollingUpdate strategy (Recreate is not valid for StatefulSets) - updateStrategy: - type: RollingUpdate - # Global PostgreSQL configuration to prevent credential mismatch - global: - postgresql: - auth: - # Dummy passwords - overridden by existingSecret - password: "unused" - postgresPassword: "unused" - existingSecret: netbox-postgresql - secretKeys: - adminPasswordKey: postgres-password - userPasswordKey: password - # Disable automatic version upgrades - upgradeRepmgrExtension: false + enabled: false + metrics: + enabled: false valkey: enabled: true @@ -131,38 +80,11 @@ valkey: ## @section Worker for Netbox parameters worker: - enabled: true - replicaCount: 1 - # Fix kubectl image version - 1.33.3 doesn't exist - initContainer: - image: - registry: docker.io - repository: bitnami/kubectl - tag: 1.31-debian-12 - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 1000m - memory: 2Gi - + enabled: false ## @section Cron housekeeping job parameters housekeeping: - enabled: true - schedule: 0 0 * * * - successfulJobsHistoryLimit: 5 - failedJobsHistoryLimit: 5 - resourcesPreset: medium - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 1000m - memory: 2Gi - + enabled: false # Database configuration using external secrets extraEnvs: - name: DB_WAIT_DEBUG @@ -182,24 +104,61 @@ extraEnvs: secretKeyRef: name: netbox-keycloak key: SOCIAL_AUTH_KEYCLOAK_SECRET - - name: SOCIAL_AUTH_KEYCLOAK_AUTHORIZATION_URL - value: "https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/auth" - - name: SOCIAL_AUTH_KEYCLOAK_ACCESS_TOKEN_URL - value: "https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/token" - - name: SOCIAL_AUTH_KEYCLOAK_ID_KEY - value: "preferred_username" - # Plugin configuration plugins: [] +# Keycloak OAuth2 configuration - using custom backend to avoid JWT validation issues +extraConfig: + - secretName: "" + data: | + import os + import sys + + # Add custom backend path + sys.path.insert(0, '/opt/netbox/custom_backends') + + # Keycloak OAuth2 Configuration + SOCIAL_AUTH_KEYCLOAK_KEY = os.environ.get('SOCIAL_AUTH_KEYCLOAK_KEY') + SOCIAL_AUTH_KEYCLOAK_SECRET = os.environ.get('SOCIAL_AUTH_KEYCLOAK_SECRET') + SOCIAL_AUTH_KEYCLOAK_AUTHORIZATION_URL = '$__file{/etc/secrets/keycloak-credentials/url}/realms/master/protocol/openid-connect/auth' + SOCIAL_AUTH_KEYCLOAK_ACCESS_TOKEN_URL = '$__file{/etc/secrets/keycloak-credentials/url}/realms/master/protocol/openid-connect/token' + SOCIAL_AUTH_KEYCLOAK_USERINFO_URL = '$__file{/etc/secrets/keycloak-credentials/url}/realms/master/protocol/openid-connect/userinfo' + SOCIAL_AUTH_KEYCLOAK_ID_KEY = 'preferred_username' + SOCIAL_AUTH_REDIRECT_IS_HTTPS = True + + # Social Auth user pipeline + SOCIAL_AUTH_USERNAME_IS_FULL_EMAIL = False + SOCIAL_AUTH_USER_FIELDS = ['username', 'email', 'first_name', 'last_name'] + + # Request proper scopes + SOCIAL_AUTH_KEYCLOAK_SCOPE = ['openid', 'profile', 'email'] + + # Redirect URI handling + SOCIAL_AUTH_KEYCLOAK_IGNORE_DEFAULT_SCOPE = True + SOCIAL_AUTH_TRAILING_SLASH = False + + # Extra data to fetch from Keycloak + SOCIAL_AUTH_KEYCLOAK_EXTRA_DATA = ['email', 'preferred_username', 'given_name', 'family_name'] + + # Authentication method for token endpoint + SOCIAL_AUTH_KEYCLOAK_AUTH_EXTRA_ARGUMENTS = {'client_id': os.environ.get('SOCIAL_AUTH_KEYCLOAK_KEY')} + ## @section Remote Authentication (SSO) Parameters remoteAuth: enabled: true backends: - - social_core.backends.keycloak.KeycloakOAuth2 - -# All Keycloak configuration is now done via environment variables in extraEnvs -# NetBox will automatically read SOCIAL_AUTH_* environment variables + - keycloak_backend.KeycloakOAuth2NoJWT + +# Mount custom Keycloak backend to avoid JWT validation issues +extraVolumes: + - name: keycloak-backend + configMap: + name: netbox-keycloak-backend + +extraVolumeMounts: + - name: keycloak-backend + mountPath: /opt/netbox/custom_backends + readOnly: true diff --git a/fleet/s/dev/c/kueyen/cnpg-cluster b/fleet/s/dev/c/kueyen/cnpg-cluster new file mode 120000 index 000000000..f383ebfb0 --- /dev/null +++ b/fleet/s/dev/c/kueyen/cnpg-cluster @@ -0,0 +1 @@ +../../../../lib/cnpg-cluster/ \ No newline at end of file diff --git a/fleet/s/dev/c/kueyen/cnpg-system b/fleet/s/dev/c/kueyen/cnpg-system new file mode 120000 index 000000000..d67b55bb2 --- /dev/null +++ b/fleet/s/dev/c/kueyen/cnpg-system @@ -0,0 +1 @@ +../../../../lib/cnpg-system \ No newline at end of file From 972334c9b5b60a8bcdd744d538efa4e6fac3c47d Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Thu, 18 Dec 2025 09:16:59 -0300 Subject: [PATCH 04/16] (netbox) Install netbox on kueyen --- fleet/lib/netbox/values.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fleet/lib/netbox/values.yaml b/fleet/lib/netbox/values.yaml index 7baca85d4..ae85ac621 100644 --- a/fleet/lib/netbox/values.yaml +++ b/fleet/lib/netbox/values.yaml @@ -63,13 +63,6 @@ externalDatabase: postgresql: enabled: false - # Explicitly disable all PostgreSQL sub-components - auth: - enabled: false - primary: - enabled: false - metrics: - enabled: false valkey: enabled: true From 24a30916d9d20a955478c6d8eabc37c808470d01 Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Fri, 8 Aug 2025 16:31:06 -0400 Subject: [PATCH 05/16] Deploy NetBox with slurpit_netbox plugin using init container - Add init container to install slurpit_netbox plugin via pip - Enable slurpit_netbox plugin in NetBox configuration - Use shared volume to provide plugins to NetBox container - Update PYTHONPATH to include plugins directory --- fleet/lib/netbox/README.md | 59 +++++ .../base/externalsecret-netbox-keycloak.yaml | 21 ++ .../externalsecret-netbox-postgresql.yaml | 21 ++ .../base/externalsecret-netbox-secrets.yaml | 33 +++ .../base/externalsecret-netbox-valkey.yaml | 17 ++ fleet/lib/netbox/base/kustomization.yaml | 12 + .../netbox/base/netbox-plugin-init-patch.yaml | 51 +++++ fleet/lib/netbox/fleet.yaml | 17 ++ fleet/lib/netbox/values.yaml | 205 ++++++++++++++++++ fleet/s/dev/c/kueyen/netbox | 1 + 10 files changed, 437 insertions(+) create mode 100644 fleet/lib/netbox/README.md create mode 100644 fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml create mode 100644 fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml create mode 100644 fleet/lib/netbox/base/externalsecret-netbox-secrets.yaml create mode 100644 fleet/lib/netbox/base/externalsecret-netbox-valkey.yaml create mode 100644 fleet/lib/netbox/base/kustomization.yaml create mode 100644 fleet/lib/netbox/base/netbox-plugin-init-patch.yaml create mode 100644 fleet/lib/netbox/fleet.yaml create mode 100644 fleet/lib/netbox/values.yaml create mode 120000 fleet/s/dev/c/kueyen/netbox diff --git a/fleet/lib/netbox/README.md b/fleet/lib/netbox/README.md new file mode 100644 index 000000000..e7f56b0d7 --- /dev/null +++ b/fleet/lib/netbox/README.md @@ -0,0 +1,59 @@ +# NetBox Kubernetes Deployment + +## Overview + +NetBox is an Infrastructure Resource Modeling (IRM) application designed to empower network automation. This deployment provides a production-ready NetBox instance on Kubernetes using Helm charts and Fleet configuration management. + +## Architecture + +- **Chart**: netbox v6.1.5 from +- **Namespace**: netbox +- **Components**: Web application, worker processes, PostgreSQL database, Valkey cache +- **Ingress**: NGINX with Let's Encrypt TLS certificates + +## Configuration + +### Core Settings + +- **Timezone**: America/Santiago +- **Superuser**: +- **Change Log Retention**: 90 days +- **Job Retention**: 90 days +- **GraphQL**: Enabled +- **Login Required**: False + +### Security + +- Non-root container execution (UID/GID: 1000) +- Read-only root filesystem +- Dropped capabilities +- Runtime security profile enabled +- External secret management via Kubernetes secrets + +### Storage + +- **Persistence**: Disabled (ephemeral storage) +- **PostgreSQL**: 20Gi persistent storage (rook-ceph-block) +- **Media/Reports/Scripts**: Stored in ephemeral volumes + +### Resources + +| Component | CPU Request | Memory Request | CPU Limit | Memory Limit | +|-----------|-------------|----------------|-----------|--------------| +| NetBox | 500m | 1Gi | 1000m | 2Gi | +| Worker | 500m | 1Gi | 1000m | 2Gi | +| PostgreSQL| 250m | 512Mi | 500m | 1Gi | + +## Access + +NetBox is accessible via dynamically generated hostnames based on cluster configuration: + +```bash +https://netbox.{cluster-name}.{site}.lsst.org +``` + +## Maintenance + +- **Housekeeping**: Daily automated cleanup (00:00 UTC) +- **Job History**: 5 successful/failed jobs retained +- **Monitoring**: Available via cluster monitoring stack diff --git a/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml new file mode 100644 index 000000000..d33fc204e --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml @@ -0,0 +1,21 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-keycloak + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-keycloak + creationPolicy: Owner + data: + - secretKey: SOCIAL_AUTH_KEYCLOAK_KEY + remoteRef: + key: &item netbox-keycloak-sso + property: client-id + - secretKey: SOCIAL_AUTH_KEYCLOAK_SECRET + remoteRef: + key: *item + property: client-secret diff --git a/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml new file mode 100644 index 000000000..6a137bb18 --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml @@ -0,0 +1,21 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-postgresql + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-postgresql + creationPolicy: Owner + data: + - secretKey: password + remoteRef: + key: &item netbox-postgresql + property: password + - secretKey: postgres-password + remoteRef: + key: *item + property: postgres-password diff --git a/fleet/lib/netbox/base/externalsecret-netbox-secrets.yaml b/fleet/lib/netbox/base/externalsecret-netbox-secrets.yaml new file mode 100644 index 000000000..a6e25c74d --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-secrets.yaml @@ -0,0 +1,33 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-secrets + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-secrets + creationPolicy: Owner + data: + - secretKey: username + remoteRef: + key: &item netbox-secrets + property: username + - secretKey: password + remoteRef: + key: *item + property: password + - secretKey: email + remoteRef: + key: *item + property: email + - secretKey: api_token + remoteRef: + key: *item + property: apiToken + - secretKey: secret-key + remoteRef: + key: *item + property: secretKey diff --git a/fleet/lib/netbox/base/externalsecret-netbox-valkey.yaml b/fleet/lib/netbox/base/externalsecret-netbox-valkey.yaml new file mode 100644 index 000000000..e91338163 --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-valkey.yaml @@ -0,0 +1,17 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-valkey + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-valkey + creationPolicy: Owner + data: + - secretKey: valkey-password + remoteRef: + key: &item netbox-valkey + property: password diff --git a/fleet/lib/netbox/base/kustomization.yaml b/fleet/lib/netbox/base/kustomization.yaml new file mode 100644 index 000000000..3aaa5138a --- /dev/null +++ b/fleet/lib/netbox/base/kustomization.yaml @@ -0,0 +1,12 @@ +--- +resources: + - externalsecret-netbox-secrets.yaml + - externalsecret-netbox-valkey.yaml + - externalsecret-netbox-postgresql.yaml + - externalsecret-netbox-keycloak.yaml + +patches: + - path: netbox-plugin-init-patch.yaml + target: + kind: Deployment + name: netbox diff --git a/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml b/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml new file mode 100644 index 000000000..0d1aa42a2 --- /dev/null +++ b/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: netbox +spec: + template: + spec: + # Add shared volume for plugins + volumes: + - name: plugins-volume + emptyDir: {} + # Init container to install slurpit_netbox plugin + initContainers: + - name: install-plugins + image: python:3.11-slim + command: + - /bin/bash + - -c + - | + set -e + echo "Installing slurpit_netbox plugin..." + + # Install pip if not available + python -m ensurepip --upgrade + + # Install the slurpit_netbox plugin to shared volume + pip install --target /opt/plugins slurpit_netbox + + echo "Plugin installation completed" + ls -la /opt/plugins/ + volumeMounts: + - name: plugins-volume + mountPath: /opt/plugins + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + containers: + - name: netbox + # Mount the plugins volume + volumeMounts: + - name: plugins-volume + mountPath: /opt/netbox/plugins + # Update PYTHONPATH to include plugins directory + env: + - name: PYTHONPATH + value: "/opt/netbox/plugins:/opt/netbox" \ No newline at end of file diff --git a/fleet/lib/netbox/fleet.yaml b/fleet/lib/netbox/fleet.yaml new file mode 100644 index 000000000..fc970ed7d --- /dev/null +++ b/fleet/lib/netbox/fleet.yaml @@ -0,0 +1,17 @@ +--- +defaultNamespace: &name netbox +labels: + bundle: *name +namespaceLabels: + lsst.io/discover: "true" +kustomize: + dir: base +helm: + chart: &chart netbox + releaseName: *chart + repo: https://charts.netbox.oss.netboxlabs.com/ + version: 6.1.5 + timeoutSeconds: 600 + waitForJobs: true + valuesFiles: + - values.yaml diff --git a/fleet/lib/netbox/values.yaml b/fleet/lib/netbox/values.yaml new file mode 100644 index 000000000..edae054f6 --- /dev/null +++ b/fleet/lib/netbox/values.yaml @@ -0,0 +1,205 @@ +nameOverride: netbox +clusterDomain: cluster.local + +superuser: + name: admin + email: admin@lsst.org + existingSecret: netbox-secrets + + +allowedHosts: + - netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org + - netbox.kueyen.dev.lsst.org + +allowedHostsIncludesPodIP: false + +admins: + - [Admin User, admin@tudominio.com] + +internalIPs: [127.0.0.1] + +timeZone: America/Santiago + +# Disable persistent volumes to avoid multi-attach issues with ReadWriteOnce storage +# Media files will be stored in ephemeral storage +persistence: + enabled: false + # storageClass: rook-ceph-block + # accessMode: ReadWriteOnce + # size: 10Gi + # annotations: {} + +reportsPersistence: + enabled: false + # storageClass: rook-ceph-block + # accessMode: ReadWriteOnce + # size: 1Gi + # annotations: {} + +scriptsPersistence: + enabled: false + # storageClass: rook-ceph-block + # accessMode: ReadWriteOnce + # size: 1Gi + # annotations: {} + +resourcesPreset: medium +resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + +## @section Traffic Exposure Parameters + +ingress: + enabled: true + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt + nginx.ingress.kubernetes.io/backend-protocol: HTTP + nginx.ingress.kubernetes.io/client-body-buffer-size: 10m + nginx.ingress.kubernetes.io/proxy-read-timeout: "60" + nginx.ingress.kubernetes.io/proxy-send-timeout: "60" + hosts: + - host: netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org + paths: + - / + tls: + - secretName: netbox-tls + hosts: + - netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org + +postgresql: + enabled: true + # Use default PostgreSQL version from chart (bitnami image tags are inconsistent) + auth: + username: netbox + database: netbox + # Dummy passwords - these are overridden by existingSecret + # Required by Bitnami chart to prevent upgrade errors + password: "unused" + postgresPassword: "unused" + existingSecret: netbox-postgresql + secretKeys: + adminPasswordKey: postgres-password + userPasswordKey: password + # Use external secret passwords for consistency + usePasswordFiles: false + # Ensure PostgreSQL is ready before Netbox starts + primary: + persistence: + enabled: true + storageClass: rook-ceph-block + size: 20Gi + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 500m + memory: 1Gi + # Prevent automatic restarts during upgrades + podDisruptionBudget: + create: true + minAvailable: 1 + # Use RollingUpdate strategy (Recreate is not valid for StatefulSets) + updateStrategy: + type: RollingUpdate + # Global PostgreSQL configuration to prevent credential mismatch + global: + postgresql: + auth: + # Dummy passwords - overridden by existingSecret + password: "unused" + postgresPassword: "unused" + existingSecret: netbox-postgresql + secretKeys: + adminPasswordKey: postgres-password + userPasswordKey: password + # Disable automatic version upgrades + upgradeRepmgrExtension: false + +valkey: + enabled: true + auth: + existingSecret: netbox-valkey + existingSecretPasswordKey: valkey-password + +## @section Worker for Netbox parameters + +worker: + enabled: true + replicaCount: 1 + # Fix kubectl image version - 1.33.3 doesn't exist + initContainer: + image: + registry: docker.io + repository: bitnami/kubectl + tag: 1.31-debian-12 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + +## @section Cron housekeeping job parameters + +housekeeping: + enabled: true + schedule: 0 0 * * * + successfulJobsHistoryLimit: 5 + failedJobsHistoryLimit: 5 + resourcesPreset: medium + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + +# Database configuration using external secrets +extraEnvs: + - name: DB_WAIT_DEBUG + value: "1" + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: netbox-secrets + key: secret-key + - name: SOCIAL_AUTH_KEYCLOAK_KEY + valueFrom: + secretKeyRef: + name: netbox-keycloak + key: SOCIAL_AUTH_KEYCLOAK_KEY + - name: SOCIAL_AUTH_KEYCLOAK_SECRET + valueFrom: + secretKeyRef: + name: netbox-keycloak + key: SOCIAL_AUTH_KEYCLOAK_SECRET + - name: SOCIAL_AUTH_KEYCLOAK_AUTHORIZATION_URL + value: "https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/auth" + - name: SOCIAL_AUTH_KEYCLOAK_ACCESS_TOKEN_URL + value: "https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/token" + - name: SOCIAL_AUTH_KEYCLOAK_ID_KEY + value: "preferred_username" + + +# Plugin configuration +plugins: [] + +## @section Remote Authentication (SSO) Parameters + +remoteAuth: + enabled: true + backends: + - social_core.backends.keycloak.KeycloakOAuth2 + +# All Keycloak configuration is now done via environment variables in extraEnvs +# NetBox will automatically read SOCIAL_AUTH_* environment variables + diff --git a/fleet/s/dev/c/kueyen/netbox b/fleet/s/dev/c/kueyen/netbox new file mode 120000 index 000000000..ccfcf159f --- /dev/null +++ b/fleet/s/dev/c/kueyen/netbox @@ -0,0 +1 @@ +../../../../lib/netbox \ No newline at end of file From dfa48eb52acd2e7c63290fabf1f9126953be3ac7 Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Mon, 24 Nov 2025 10:32:36 -0300 Subject: [PATCH 06/16] Add test files to gitignore --- fleet/lib/netbox/.gitignore | 4 ++ fleet/lib/netbox/base/kustomization.yaml | 6 --- .../netbox/base/netbox-plugin-init-patch.yaml | 51 ------------------- 3 files changed, 4 insertions(+), 57 deletions(-) create mode 100644 fleet/lib/netbox/.gitignore delete mode 100644 fleet/lib/netbox/base/netbox-plugin-init-patch.yaml diff --git a/fleet/lib/netbox/.gitignore b/fleet/lib/netbox/.gitignore new file mode 100644 index 000000000..668037e56 --- /dev/null +++ b/fleet/lib/netbox/.gitignore @@ -0,0 +1,4 @@ +GEMINI.md +values-test.yaml +CLAUDE.md +WARP.md \ No newline at end of file diff --git a/fleet/lib/netbox/base/kustomization.yaml b/fleet/lib/netbox/base/kustomization.yaml index 3aaa5138a..918c851d3 100644 --- a/fleet/lib/netbox/base/kustomization.yaml +++ b/fleet/lib/netbox/base/kustomization.yaml @@ -4,9 +4,3 @@ resources: - externalsecret-netbox-valkey.yaml - externalsecret-netbox-postgresql.yaml - externalsecret-netbox-keycloak.yaml - -patches: - - path: netbox-plugin-init-patch.yaml - target: - kind: Deployment - name: netbox diff --git a/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml b/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml deleted file mode 100644 index 0d1aa42a2..000000000 --- a/fleet/lib/netbox/base/netbox-plugin-init-patch.yaml +++ /dev/null @@ -1,51 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: netbox -spec: - template: - spec: - # Add shared volume for plugins - volumes: - - name: plugins-volume - emptyDir: {} - # Init container to install slurpit_netbox plugin - initContainers: - - name: install-plugins - image: python:3.11-slim - command: - - /bin/bash - - -c - - | - set -e - echo "Installing slurpit_netbox plugin..." - - # Install pip if not available - python -m ensurepip --upgrade - - # Install the slurpit_netbox plugin to shared volume - pip install --target /opt/plugins slurpit_netbox - - echo "Plugin installation completed" - ls -la /opt/plugins/ - volumeMounts: - - name: plugins-volume - mountPath: /opt/plugins - resources: - requests: - cpu: 100m - memory: 256Mi - limits: - cpu: 500m - memory: 512Mi - containers: - - name: netbox - # Mount the plugins volume - volumeMounts: - - name: plugins-volume - mountPath: /opt/netbox/plugins - # Update PYTHONPATH to include plugins directory - env: - - name: PYTHONPATH - value: "/opt/netbox/plugins:/opt/netbox" \ No newline at end of file From f02a39f28f5b7e29a526079582167ee79c00aca6 Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Tue, 25 Nov 2025 21:57:16 -0300 Subject: [PATCH 07/16] feat: Enable Keycloak SSO for Netbox This commit finalizes the configuration for Keycloak Single Sign-On (SSO) in Netbox. The flag has been enabled to ensure secure redirection during the authentication process. This was the final step in enabling the existing Keycloak configuration. --- fleet/lib/cnpg-cluster/fleet.yaml | 10 + .../overlays/kueyen/cluster-cnpg-cluster.yaml | 46 +++++ .../overlays/kueyen/cnpg-loadBalancer.yaml | 17 ++ ...externalsecret-cnpg-cluster-superuser.yaml | 21 +++ .../kueyen/pooler-cnpg-cluster-pooler.yaml | 71 +++++++ .../base/configmap-keycloak-backend.yaml | 24 +++ .../base/externalsecret-netbox-keycloak.yaml | 4 + .../externalsecret-netbox-postgresql.yaml | 2 +- fleet/lib/netbox/base/kustomization.yaml | 1 + fleet/lib/netbox/fleet.yaml | 7 + fleet/lib/netbox/values.yaml | 175 +++++++----------- fleet/s/dev/c/kueyen/cnpg-cluster | 1 + fleet/s/dev/c/kueyen/cnpg-system | 1 + 13 files changed, 271 insertions(+), 109 deletions(-) create mode 100644 fleet/lib/cnpg-cluster/overlays/kueyen/cluster-cnpg-cluster.yaml create mode 100644 fleet/lib/cnpg-cluster/overlays/kueyen/cnpg-loadBalancer.yaml create mode 100644 fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml create mode 100644 fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml create mode 100644 fleet/lib/netbox/base/configmap-keycloak-backend.yaml create mode 120000 fleet/s/dev/c/kueyen/cnpg-cluster create mode 120000 fleet/s/dev/c/kueyen/cnpg-system diff --git a/fleet/lib/cnpg-cluster/fleet.yaml b/fleet/lib/cnpg-cluster/fleet.yaml index 9183c9cad..06e81594d 100644 --- a/fleet/lib/cnpg-cluster/fleet.yaml +++ b/fleet/lib/cnpg-cluster/fleet.yaml @@ -61,3 +61,13 @@ targetCustomizations: yaml: overlays: - yagan + - name: kueyen + clusterSelector: + matchExpressions: + - key: management.cattle.io/cluster-display-name + operator: In + values: + - kueyen + yaml: + overlays: + - kueyen diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/cluster-cnpg-cluster.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/cluster-cnpg-cluster.yaml new file mode 100644 index 000000000..171bbebfb --- /dev/null +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/cluster-cnpg-cluster.yaml @@ -0,0 +1,46 @@ +--- +# Cluster Definition +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: cnpg-cluster +spec: + imageName: docker.io/lsstit/cnpgsphere:16.8 + imagePullPolicy: Always + + instances: 3 + + postgresql: + parameters: + max_connections: "1000" + shared_buffers: 256MB + work_mem: 8MB + effective_cache_size: 768MB + idle_session_timeout: 1h + max_slot_wal_keep_size: 1GB + pg_hba: + - host replication postgres all md5 + - host all all 139.229.134.0/23 md5 + - host all all 139.229.136.0/21 md5 + - host all all 139.229.144.0/20 md5 + - host all all 139.229.160.0/19 md5 + - host all all 139.229.192.0/18 md5 + - host all all 140.252.146.0/23 md5 + + enableSuperuserAccess: true + superuserSecret: + name: cnpg-cluster-superuser + + storage: + size: 5Gi + + monitoring: + enablePodMonitor: true + + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: "1" + memory: 1Gi diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/cnpg-loadBalancer.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/cnpg-loadBalancer.yaml new file mode 100644 index 000000000..5cdca9dea --- /dev/null +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/cnpg-loadBalancer.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: cnpg-loadbalancer + labels: + cnpg.io/cluster: cnpg-cluster + annotations: + metallb.universe.tf/loadBalancerIPs: 139.229.134.89 +spec: + ports: + - name: postgresql + port: 5432 + protocol: TCP + selector: + cnpg.io/cluster: cnpg-cluster + role: primary + type: LoadBalancer diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml new file mode 100644 index 000000000..1a3dff27f --- /dev/null +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: cnpg-cluster-superuser +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + template: + type: kubernetes.io/basic-auth + data: + - secretKey: username + remoteRef: + key: cnpg-cluster-superuser-kueyen + property: username + - secretKey: password + remoteRef: + key: cnpg-cluster-superuser-kueyen + property: password diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml new file mode 100644 index 000000000..9d855bd36 --- /dev/null +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml @@ -0,0 +1,71 @@ +--- +apiVersion: postgresql.cnpg.io/v1 +kind: Pooler +metadata: + name: cnpg-cluster-pooler + namespace: cloudnativepg +spec: + cluster: + name: cnpg-cluster + instances: 3 + type: rw + + pgbouncer: + poolMode: session + parameters: + max_client_conn: "2000" + default_pool_size: "280" + reserve_pool_size: "20" + server_lifetime: "3600" + log_connections: "1" + log_disconnections: "1" + idle_transaction_timeout: "0" + ignore_startup_parameters: extra_float_digits + + template: + metadata: + labels: + app: pooler + lsst.io/monitor: "true" + spec: + containers: + - name: pgbouncer + resources: + requests: + cpu: "0.1" + memory: 100Mi + limits: + cpu: "0.5" + memory: 500Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: cnpg-cluster-pooler-rw + namespace: cloudnativepg + annotations: + metallb.universe.tf/loadBalancerIPs: 139.229.134.90 +spec: + type: LoadBalancer + internalTrafficPolicy: Cluster + ports: + - name: pgbouncer + port: 5432 + protocol: TCP + targetPort: 5432 + selector: + cnpg.io/poolerName: cnpg-cluster-pooler +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: cnpg-cluster-pooler + namespace: cloudnativepg + labels: + lsst.io/monitor: "true" +spec: + selector: + matchLabels: + cnpg.io/poolerName: cnpg-cluster-pooler + podMetricsEndpoints: + - port: metrics diff --git a/fleet/lib/netbox/base/configmap-keycloak-backend.yaml b/fleet/lib/netbox/base/configmap-keycloak-backend.yaml new file mode 100644 index 000000000..211ef8ad3 --- /dev/null +++ b/fleet/lib/netbox/base/configmap-keycloak-backend.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: netbox-keycloak-backend + namespace: netbox +data: + keycloak_backend.py: | + from social_core.backends.keycloak import KeycloakOAuth2 + + class KeycloakOAuth2NoJWT(KeycloakOAuth2): + """Custom Keycloak backend that skips JWT validation""" + + def public_key(self): + """Override to skip public key validation""" + return None + + def user_data(self, access_token, *args, **kwargs): + """Get user data from userinfo endpoint without JWT validation""" + # Get USERINFO URL from settings + url = self.setting('USERINFO_URL') or self.USERINFO_URL + return self.get_json( + url, + headers={'Authorization': f'Bearer {access_token}'} + ) diff --git a/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml index d33fc204e..3b750f599 100644 --- a/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml +++ b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml @@ -19,3 +19,7 @@ spec: remoteRef: key: *item property: client-secret + - secretKey: url + remoteRef: + key: *item + property: url \ No newline at end of file diff --git a/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml index 6a137bb18..33003f3b6 100644 --- a/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml +++ b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml @@ -13,7 +13,7 @@ spec: data: - secretKey: password remoteRef: - key: &item netbox-postgresql + key: &item cnpg-cluster-superuser-kueyen property: password - secretKey: postgres-password remoteRef: diff --git a/fleet/lib/netbox/base/kustomization.yaml b/fleet/lib/netbox/base/kustomization.yaml index 918c851d3..c9934ff6e 100644 --- a/fleet/lib/netbox/base/kustomization.yaml +++ b/fleet/lib/netbox/base/kustomization.yaml @@ -4,3 +4,4 @@ resources: - externalsecret-netbox-valkey.yaml - externalsecret-netbox-postgresql.yaml - externalsecret-netbox-keycloak.yaml + - configmap-keycloak-backend.yaml diff --git a/fleet/lib/netbox/fleet.yaml b/fleet/lib/netbox/fleet.yaml index fc970ed7d..8f84a8225 100644 --- a/fleet/lib/netbox/fleet.yaml +++ b/fleet/lib/netbox/fleet.yaml @@ -13,5 +13,12 @@ helm: version: 6.1.5 timeoutSeconds: 600 waitForJobs: true +dependsOn: + - selector: + matchLabels: + bundle: external-secrets + - selector: + matchLabels: + bundle: cnpg-cluster valuesFiles: - values.yaml diff --git a/fleet/lib/netbox/values.yaml b/fleet/lib/netbox/values.yaml index edae054f6..7baca85d4 100644 --- a/fleet/lib/netbox/values.yaml +++ b/fleet/lib/netbox/values.yaml @@ -3,18 +3,17 @@ clusterDomain: cluster.local superuser: name: admin - email: admin@lsst.org + email: rubbinobs-it@lsst.org existingSecret: netbox-secrets allowedHosts: - netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org - - netbox.kueyen.dev.lsst.org allowedHostsIncludesPodIP: false admins: - - [Admin User, admin@tudominio.com] + - [Admin User, rubbinobs-it@lsst.org] internalIPs: [127.0.0.1] @@ -24,26 +23,7 @@ timeZone: America/Santiago # Media files will be stored in ephemeral storage persistence: enabled: false - # storageClass: rook-ceph-block - # accessMode: ReadWriteOnce - # size: 10Gi - # annotations: {} -reportsPersistence: - enabled: false - # storageClass: rook-ceph-block - # accessMode: ReadWriteOnce - # size: 1Gi - # annotations: {} - -scriptsPersistence: - enabled: false - # storageClass: rook-ceph-block - # accessMode: ReadWriteOnce - # size: 1Gi - # annotations: {} - -resourcesPreset: medium resources: requests: cpu: 500m @@ -72,55 +52,24 @@ ingress: hosts: - netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org -postgresql: +externalDatabase: enabled: true - # Use default PostgreSQL version from chart (bitnami image tags are inconsistent) + host: cnpg-loadbalancer.cloudnativepg.svc.cluster.local + port: 5432 + user: postgres + database: netbox + existingSecret: netbox-postgresql + existingSecretPasswordKey: postgres-password + +postgresql: + enabled: false + # Explicitly disable all PostgreSQL sub-components auth: - username: netbox - database: netbox - # Dummy passwords - these are overridden by existingSecret - # Required by Bitnami chart to prevent upgrade errors - password: "unused" - postgresPassword: "unused" - existingSecret: netbox-postgresql - secretKeys: - adminPasswordKey: postgres-password - userPasswordKey: password - # Use external secret passwords for consistency - usePasswordFiles: false - # Ensure PostgreSQL is ready before Netbox starts + enabled: false primary: - persistence: - enabled: true - storageClass: rook-ceph-block - size: 20Gi - resources: - requests: - cpu: 250m - memory: 512Mi - limits: - cpu: 500m - memory: 1Gi - # Prevent automatic restarts during upgrades - podDisruptionBudget: - create: true - minAvailable: 1 - # Use RollingUpdate strategy (Recreate is not valid for StatefulSets) - updateStrategy: - type: RollingUpdate - # Global PostgreSQL configuration to prevent credential mismatch - global: - postgresql: - auth: - # Dummy passwords - overridden by existingSecret - password: "unused" - postgresPassword: "unused" - existingSecret: netbox-postgresql - secretKeys: - adminPasswordKey: postgres-password - userPasswordKey: password - # Disable automatic version upgrades - upgradeRepmgrExtension: false + enabled: false + metrics: + enabled: false valkey: enabled: true @@ -131,38 +80,11 @@ valkey: ## @section Worker for Netbox parameters worker: - enabled: true - replicaCount: 1 - # Fix kubectl image version - 1.33.3 doesn't exist - initContainer: - image: - registry: docker.io - repository: bitnami/kubectl - tag: 1.31-debian-12 - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 1000m - memory: 2Gi - + enabled: false ## @section Cron housekeeping job parameters housekeeping: - enabled: true - schedule: 0 0 * * * - successfulJobsHistoryLimit: 5 - failedJobsHistoryLimit: 5 - resourcesPreset: medium - resources: - requests: - cpu: 500m - memory: 1Gi - limits: - cpu: 1000m - memory: 2Gi - + enabled: false # Database configuration using external secrets extraEnvs: - name: DB_WAIT_DEBUG @@ -182,24 +104,61 @@ extraEnvs: secretKeyRef: name: netbox-keycloak key: SOCIAL_AUTH_KEYCLOAK_SECRET - - name: SOCIAL_AUTH_KEYCLOAK_AUTHORIZATION_URL - value: "https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/auth" - - name: SOCIAL_AUTH_KEYCLOAK_ACCESS_TOKEN_URL - value: "https://keycloak.ls.lsst.org/realms/master/protocol/openid-connect/token" - - name: SOCIAL_AUTH_KEYCLOAK_ID_KEY - value: "preferred_username" - # Plugin configuration plugins: [] +# Keycloak OAuth2 configuration - using custom backend to avoid JWT validation issues +extraConfig: + - secretName: "" + data: | + import os + import sys + + # Add custom backend path + sys.path.insert(0, '/opt/netbox/custom_backends') + + # Keycloak OAuth2 Configuration + SOCIAL_AUTH_KEYCLOAK_KEY = os.environ.get('SOCIAL_AUTH_KEYCLOAK_KEY') + SOCIAL_AUTH_KEYCLOAK_SECRET = os.environ.get('SOCIAL_AUTH_KEYCLOAK_SECRET') + SOCIAL_AUTH_KEYCLOAK_AUTHORIZATION_URL = '$__file{/etc/secrets/keycloak-credentials/url}/realms/master/protocol/openid-connect/auth' + SOCIAL_AUTH_KEYCLOAK_ACCESS_TOKEN_URL = '$__file{/etc/secrets/keycloak-credentials/url}/realms/master/protocol/openid-connect/token' + SOCIAL_AUTH_KEYCLOAK_USERINFO_URL = '$__file{/etc/secrets/keycloak-credentials/url}/realms/master/protocol/openid-connect/userinfo' + SOCIAL_AUTH_KEYCLOAK_ID_KEY = 'preferred_username' + SOCIAL_AUTH_REDIRECT_IS_HTTPS = True + + # Social Auth user pipeline + SOCIAL_AUTH_USERNAME_IS_FULL_EMAIL = False + SOCIAL_AUTH_USER_FIELDS = ['username', 'email', 'first_name', 'last_name'] + + # Request proper scopes + SOCIAL_AUTH_KEYCLOAK_SCOPE = ['openid', 'profile', 'email'] + + # Redirect URI handling + SOCIAL_AUTH_KEYCLOAK_IGNORE_DEFAULT_SCOPE = True + SOCIAL_AUTH_TRAILING_SLASH = False + + # Extra data to fetch from Keycloak + SOCIAL_AUTH_KEYCLOAK_EXTRA_DATA = ['email', 'preferred_username', 'given_name', 'family_name'] + + # Authentication method for token endpoint + SOCIAL_AUTH_KEYCLOAK_AUTH_EXTRA_ARGUMENTS = {'client_id': os.environ.get('SOCIAL_AUTH_KEYCLOAK_KEY')} + ## @section Remote Authentication (SSO) Parameters remoteAuth: enabled: true backends: - - social_core.backends.keycloak.KeycloakOAuth2 - -# All Keycloak configuration is now done via environment variables in extraEnvs -# NetBox will automatically read SOCIAL_AUTH_* environment variables + - keycloak_backend.KeycloakOAuth2NoJWT + +# Mount custom Keycloak backend to avoid JWT validation issues +extraVolumes: + - name: keycloak-backend + configMap: + name: netbox-keycloak-backend + +extraVolumeMounts: + - name: keycloak-backend + mountPath: /opt/netbox/custom_backends + readOnly: true diff --git a/fleet/s/dev/c/kueyen/cnpg-cluster b/fleet/s/dev/c/kueyen/cnpg-cluster new file mode 120000 index 000000000..f383ebfb0 --- /dev/null +++ b/fleet/s/dev/c/kueyen/cnpg-cluster @@ -0,0 +1 @@ +../../../../lib/cnpg-cluster/ \ No newline at end of file diff --git a/fleet/s/dev/c/kueyen/cnpg-system b/fleet/s/dev/c/kueyen/cnpg-system new file mode 120000 index 000000000..d67b55bb2 --- /dev/null +++ b/fleet/s/dev/c/kueyen/cnpg-system @@ -0,0 +1 @@ +../../../../lib/cnpg-system \ No newline at end of file From e7aae55840417243a5d43a7a8db5b2f6cc4b0334 Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Thu, 18 Dec 2025 09:16:59 -0300 Subject: [PATCH 08/16] (netbox) Install netbox on kueyen --- fleet/lib/netbox/values.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/fleet/lib/netbox/values.yaml b/fleet/lib/netbox/values.yaml index 7baca85d4..ae85ac621 100644 --- a/fleet/lib/netbox/values.yaml +++ b/fleet/lib/netbox/values.yaml @@ -63,13 +63,6 @@ externalDatabase: postgresql: enabled: false - # Explicitly disable all PostgreSQL sub-components - auth: - enabled: false - primary: - enabled: false - metrics: - enabled: false valkey: enabled: true From 66763a1f2238021fbbc028126fc7cd321efb648e Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Thu, 18 Dec 2025 09:35:28 -0300 Subject: [PATCH 09/16] (kueyen) remove loki, mimir. update externalsecrets from kube-prometheus-stack-pre --- .../externalsecret-grafana-keycloak-credentials.yaml | 2 +- .../lib/kube-prometheus-stack-pre/externalsecret-puppetdb.yaml | 2 +- .../externalsecrets-alertmanager-webhooks.yaml | 2 +- fleet/s/dev/c/kueyen/fluent-bit-kube | 1 - fleet/s/dev/c/kueyen/loki | 1 - fleet/s/dev/c/kueyen/mimir | 1 - fleet/s/dev/c/kueyen/mimir-pre | 1 - 7 files changed, 3 insertions(+), 7 deletions(-) delete mode 120000 fleet/s/dev/c/kueyen/fluent-bit-kube delete mode 120000 fleet/s/dev/c/kueyen/loki delete mode 120000 fleet/s/dev/c/kueyen/mimir delete mode 120000 fleet/s/dev/c/kueyen/mimir-pre diff --git a/fleet/lib/kube-prometheus-stack-pre/externalsecret-grafana-keycloak-credentials.yaml b/fleet/lib/kube-prometheus-stack-pre/externalsecret-grafana-keycloak-credentials.yaml index f39af41a7..1712962b3 100644 --- a/fleet/lib/kube-prometheus-stack-pre/externalsecret-grafana-keycloak-credentials.yaml +++ b/fleet/lib/kube-prometheus-stack-pre/externalsecret-grafana-keycloak-credentials.yaml @@ -1,4 +1,4 @@ -apiVersion: external-secrets.io/v1beta1 +apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: grafana-keycloak-credentials diff --git a/fleet/lib/kube-prometheus-stack-pre/externalsecret-puppetdb.yaml b/fleet/lib/kube-prometheus-stack-pre/externalsecret-puppetdb.yaml index efe760e80..7f0be05ae 100644 --- a/fleet/lib/kube-prometheus-stack-pre/externalsecret-puppetdb.yaml +++ b/fleet/lib/kube-prometheus-stack-pre/externalsecret-puppetdb.yaml @@ -1,5 +1,5 @@ --- -apiVersion: external-secrets.io/v1beta1 +apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: puppetdb diff --git a/fleet/lib/kube-prometheus-stack-pre/externalsecrets-alertmanager-webhooks.yaml b/fleet/lib/kube-prometheus-stack-pre/externalsecrets-alertmanager-webhooks.yaml index 37a12427b..bdf4d9c10 100644 --- a/fleet/lib/kube-prometheus-stack-pre/externalsecrets-alertmanager-webhooks.yaml +++ b/fleet/lib/kube-prometheus-stack-pre/externalsecrets-alertmanager-webhooks.yaml @@ -1,5 +1,5 @@ --- -apiVersion: external-secrets.io/v1beta1 +apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: alertmanager-webhooks diff --git a/fleet/s/dev/c/kueyen/fluent-bit-kube b/fleet/s/dev/c/kueyen/fluent-bit-kube deleted file mode 120000 index 340224dca..000000000 --- a/fleet/s/dev/c/kueyen/fluent-bit-kube +++ /dev/null @@ -1 +0,0 @@ -../../../../lib/fluent-bit-kube \ No newline at end of file diff --git a/fleet/s/dev/c/kueyen/loki b/fleet/s/dev/c/kueyen/loki deleted file mode 120000 index 874625b54..000000000 --- a/fleet/s/dev/c/kueyen/loki +++ /dev/null @@ -1 +0,0 @@ -../../../../lib/loki \ No newline at end of file diff --git a/fleet/s/dev/c/kueyen/mimir b/fleet/s/dev/c/kueyen/mimir deleted file mode 120000 index 30cd3d61a..000000000 --- a/fleet/s/dev/c/kueyen/mimir +++ /dev/null @@ -1 +0,0 @@ -../../../../lib/mimir \ No newline at end of file diff --git a/fleet/s/dev/c/kueyen/mimir-pre b/fleet/s/dev/c/kueyen/mimir-pre deleted file mode 120000 index 5b8d0818c..000000000 --- a/fleet/s/dev/c/kueyen/mimir-pre +++ /dev/null @@ -1 +0,0 @@ -../../../../lib/mimir-pre \ No newline at end of file From 4427e028770bb863a005d4e87d9b065bf4257043 Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Thu, 18 Dec 2025 09:45:51 -0300 Subject: [PATCH 10/16] (netbox) fix yamllint --- .../netbox/base/configmap-keycloak-backend.yaml | 6 +++--- .../base/externalsecret-netbox-keycloak.yaml | 2 +- fleet/lib/netbox/fleet.yaml | 16 ++++++++-------- fleet/lib/netbox/values.yaml | 15 +++++++-------- 4 files changed, 19 insertions(+), 20 deletions(-) diff --git a/fleet/lib/netbox/base/configmap-keycloak-backend.yaml b/fleet/lib/netbox/base/configmap-keycloak-backend.yaml index 211ef8ad3..76d4b7faf 100644 --- a/fleet/lib/netbox/base/configmap-keycloak-backend.yaml +++ b/fleet/lib/netbox/base/configmap-keycloak-backend.yaml @@ -6,14 +6,14 @@ metadata: data: keycloak_backend.py: | from social_core.backends.keycloak import KeycloakOAuth2 - + class KeycloakOAuth2NoJWT(KeycloakOAuth2): """Custom Keycloak backend that skips JWT validation""" - + def public_key(self): """Override to skip public key validation""" return None - + def user_data(self, access_token, *args, **kwargs): """Get user data from userinfo endpoint without JWT validation""" # Get USERINFO URL from settings diff --git a/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml index 3b750f599..e1af722b5 100644 --- a/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml +++ b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml @@ -22,4 +22,4 @@ spec: - secretKey: url remoteRef: key: *item - property: url \ No newline at end of file + property: url diff --git a/fleet/lib/netbox/fleet.yaml b/fleet/lib/netbox/fleet.yaml index 8f84a8225..9526b1244 100644 --- a/fleet/lib/netbox/fleet.yaml +++ b/fleet/lib/netbox/fleet.yaml @@ -13,12 +13,12 @@ helm: version: 6.1.5 timeoutSeconds: 600 waitForJobs: true -dependsOn: - - selector: - matchLabels: - bundle: external-secrets - - selector: - matchLabels: - bundle: cnpg-cluster valuesFiles: - - values.yaml + - values.yaml +dependsOn: +- selector: + matchLabels: + bundle: external-secrets +- selector: + matchLabels: + bundle: cnpg-cluster diff --git a/fleet/lib/netbox/values.yaml b/fleet/lib/netbox/values.yaml index ae85ac621..3191080d4 100644 --- a/fleet/lib/netbox/values.yaml +++ b/fleet/lib/netbox/values.yaml @@ -107,10 +107,10 @@ extraConfig: data: | import os import sys - + # Add custom backend path sys.path.insert(0, '/opt/netbox/custom_backends') - + # Keycloak OAuth2 Configuration SOCIAL_AUTH_KEYCLOAK_KEY = os.environ.get('SOCIAL_AUTH_KEYCLOAK_KEY') SOCIAL_AUTH_KEYCLOAK_SECRET = os.environ.get('SOCIAL_AUTH_KEYCLOAK_SECRET') @@ -119,21 +119,21 @@ extraConfig: SOCIAL_AUTH_KEYCLOAK_USERINFO_URL = '$__file{/etc/secrets/keycloak-credentials/url}/realms/master/protocol/openid-connect/userinfo' SOCIAL_AUTH_KEYCLOAK_ID_KEY = 'preferred_username' SOCIAL_AUTH_REDIRECT_IS_HTTPS = True - + # Social Auth user pipeline SOCIAL_AUTH_USERNAME_IS_FULL_EMAIL = False SOCIAL_AUTH_USER_FIELDS = ['username', 'email', 'first_name', 'last_name'] - + # Request proper scopes SOCIAL_AUTH_KEYCLOAK_SCOPE = ['openid', 'profile', 'email'] - + # Redirect URI handling SOCIAL_AUTH_KEYCLOAK_IGNORE_DEFAULT_SCOPE = True SOCIAL_AUTH_TRAILING_SLASH = False - + # Extra data to fetch from Keycloak SOCIAL_AUTH_KEYCLOAK_EXTRA_DATA = ['email', 'preferred_username', 'given_name', 'family_name'] - + # Authentication method for token endpoint SOCIAL_AUTH_KEYCLOAK_AUTH_EXTRA_ARGUMENTS = {'client_id': os.environ.get('SOCIAL_AUTH_KEYCLOAK_KEY')} @@ -154,4 +154,3 @@ extraVolumeMounts: - name: keycloak-backend mountPath: /opt/netbox/custom_backends readOnly: true - From 359701c4e661de403ac9acdd32f9867be3eab569 Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Thu, 18 Dec 2025 09:55:35 -0300 Subject: [PATCH 11/16] (netbox) updated external secrets api version --- .../overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml index 1a3dff27f..7f321bd0b 100644 --- a/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml @@ -1,5 +1,5 @@ --- -apiVersion: external-secrets.io/v1beta1 +apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: cnpg-cluster-superuser From 105d8c54dbcd23297d7482d53aec9a2add38ab41 Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Thu, 18 Dec 2025 11:23:53 -0300 Subject: [PATCH 12/16] (netbox) remove kueyen mimir obc --- fleet/lib/rook-ceph-conf/fleet.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/fleet/lib/rook-ceph-conf/fleet.yaml b/fleet/lib/rook-ceph-conf/fleet.yaml index f752a6a92..bc7f0ead9 100644 --- a/fleet/lib/rook-ceph-conf/fleet.yaml +++ b/fleet/lib/rook-ceph-conf/fleet.yaml @@ -26,18 +26,6 @@ diff: - /spec/preservePoolsOnDelete - /spec/healthCheck targetCustomizations: - - name: kueyen - clusterSelector: - matchExpressions: - - key: management.cattle.io/cluster-display-name - operator: In - values: - - kueyen - helm: - values: - subchart: - kueyen: - enabled: true - name: ayekan clusterSelector: matchExpressions: From bac6c6b6332aa1e4897fffdb30a1ef1e3f38a342 Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Thu, 18 Dec 2025 11:25:44 -0300 Subject: [PATCH 13/16] (netbox) fix cnpg link --- fleet/s/dev/c/kueyen/cnpg-cluster | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fleet/s/dev/c/kueyen/cnpg-cluster b/fleet/s/dev/c/kueyen/cnpg-cluster index f383ebfb0..702632008 120000 --- a/fleet/s/dev/c/kueyen/cnpg-cluster +++ b/fleet/s/dev/c/kueyen/cnpg-cluster @@ -1 +1 @@ -../../../../lib/cnpg-cluster/ \ No newline at end of file +../../../../lib/cnpg-cluster \ No newline at end of file From 71ea22218b07d3ccbdaf873b7a09f17e37716fbb Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Thu, 18 Dec 2025 15:11:17 -0300 Subject: [PATCH 14/16] (netbox) fix cnpg link --- .../kueyen/pooler-cnpg-cluster-pooler.yaml | 71 ------------------- 1 file changed, 71 deletions(-) delete mode 100644 fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml deleted file mode 100644 index 9d855bd36..000000000 --- a/fleet/lib/cnpg-cluster/overlays/kueyen/pooler-cnpg-cluster-pooler.yaml +++ /dev/null @@ -1,71 +0,0 @@ ---- -apiVersion: postgresql.cnpg.io/v1 -kind: Pooler -metadata: - name: cnpg-cluster-pooler - namespace: cloudnativepg -spec: - cluster: - name: cnpg-cluster - instances: 3 - type: rw - - pgbouncer: - poolMode: session - parameters: - max_client_conn: "2000" - default_pool_size: "280" - reserve_pool_size: "20" - server_lifetime: "3600" - log_connections: "1" - log_disconnections: "1" - idle_transaction_timeout: "0" - ignore_startup_parameters: extra_float_digits - - template: - metadata: - labels: - app: pooler - lsst.io/monitor: "true" - spec: - containers: - - name: pgbouncer - resources: - requests: - cpu: "0.1" - memory: 100Mi - limits: - cpu: "0.5" - memory: 500Mi ---- -apiVersion: v1 -kind: Service -metadata: - name: cnpg-cluster-pooler-rw - namespace: cloudnativepg - annotations: - metallb.universe.tf/loadBalancerIPs: 139.229.134.90 -spec: - type: LoadBalancer - internalTrafficPolicy: Cluster - ports: - - name: pgbouncer - port: 5432 - protocol: TCP - targetPort: 5432 - selector: - cnpg.io/poolerName: cnpg-cluster-pooler ---- -apiVersion: monitoring.coreos.com/v1 -kind: PodMonitor -metadata: - name: cnpg-cluster-pooler - namespace: cloudnativepg - labels: - lsst.io/monitor: "true" -spec: - selector: - matchLabels: - cnpg.io/poolerName: cnpg-cluster-pooler - podMetricsEndpoints: - - port: metrics From 55caaa41eb7f4283c59ddac0aa4bae6fd460cb70 Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Thu, 18 Dec 2025 21:34:51 -0300 Subject: [PATCH 15/16] (netbox) fix password for externalsecret in cnpg --- fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml index 33003f3b6..7609c7d69 100644 --- a/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml +++ b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml @@ -15,7 +15,3 @@ spec: remoteRef: key: &item cnpg-cluster-superuser-kueyen property: password - - secretKey: postgres-password - remoteRef: - key: *item - property: postgres-password From 03be337b82f4ca44b55ff9f71a91df435e28bb9c Mon Sep 17 00:00:00 2001 From: Gonzalo Seriche Date: Thu, 18 Dec 2025 22:36:36 -0300 Subject: [PATCH 16/16] (netbox) fix externalsecret key --- fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml index 7609c7d69..3c0792f41 100644 --- a/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml +++ b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml @@ -11,7 +11,7 @@ spec: name: netbox-postgresql creationPolicy: Owner data: - - secretKey: password + - secretKey: db_password remoteRef: key: &item cnpg-cluster-superuser-kueyen property: password