diff --git a/fleet/lib/cnpg-cluster/fleet.yaml b/fleet/lib/cnpg-cluster/fleet.yaml index 9183c9cad..06e81594d 100644 --- a/fleet/lib/cnpg-cluster/fleet.yaml +++ b/fleet/lib/cnpg-cluster/fleet.yaml @@ -61,3 +61,13 @@ targetCustomizations: yaml: overlays: - yagan + - name: kueyen + clusterSelector: + matchExpressions: + - key: management.cattle.io/cluster-display-name + operator: In + values: + - kueyen + yaml: + overlays: + - kueyen diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/cluster-cnpg-cluster.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/cluster-cnpg-cluster.yaml new file mode 100644 index 000000000..171bbebfb --- /dev/null +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/cluster-cnpg-cluster.yaml @@ -0,0 +1,46 @@ +--- +# Cluster Definition +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: cnpg-cluster +spec: + imageName: docker.io/lsstit/cnpgsphere:16.8 + imagePullPolicy: Always + + instances: 3 + + postgresql: + parameters: + max_connections: "1000" + shared_buffers: 256MB + work_mem: 8MB + effective_cache_size: 768MB + idle_session_timeout: 1h + max_slot_wal_keep_size: 1GB + pg_hba: + - host replication postgres all md5 + - host all all 139.229.134.0/23 md5 + - host all all 139.229.136.0/21 md5 + - host all all 139.229.144.0/20 md5 + - host all all 139.229.160.0/19 md5 + - host all all 139.229.192.0/18 md5 + - host all all 140.252.146.0/23 md5 + + enableSuperuserAccess: true + superuserSecret: + name: cnpg-cluster-superuser + + storage: + size: 5Gi + + monitoring: + enablePodMonitor: true + + resources: + limits: + cpu: "1" + memory: 1Gi + requests: + cpu: "1" + memory: 1Gi diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/cnpg-loadBalancer.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/cnpg-loadBalancer.yaml new file mode 100644 index 000000000..5cdca9dea --- /dev/null +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/cnpg-loadBalancer.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: cnpg-loadbalancer + labels: + cnpg.io/cluster: cnpg-cluster + annotations: + metallb.universe.tf/loadBalancerIPs: 139.229.134.89 +spec: + ports: + - name: postgresql + port: 5432 + protocol: TCP + selector: + cnpg.io/cluster: cnpg-cluster + role: primary + type: LoadBalancer diff --git a/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml b/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml new file mode 100644 index 000000000..7f321bd0b --- /dev/null +++ b/fleet/lib/cnpg-cluster/overlays/kueyen/externalsecret-cnpg-cluster-superuser.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: cnpg-cluster-superuser +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + template: + type: kubernetes.io/basic-auth + data: + - secretKey: username + remoteRef: + key: cnpg-cluster-superuser-kueyen + property: username + - secretKey: password + remoteRef: + key: cnpg-cluster-superuser-kueyen + property: password diff --git a/fleet/lib/netbox/.gitignore b/fleet/lib/netbox/.gitignore new file mode 100644 index 000000000..668037e56 --- /dev/null +++ b/fleet/lib/netbox/.gitignore @@ -0,0 +1,4 @@ +GEMINI.md +values-test.yaml +CLAUDE.md +WARP.md \ No newline at end of file diff --git a/fleet/lib/netbox/README.md b/fleet/lib/netbox/README.md new file mode 100644 index 000000000..e7f56b0d7 --- /dev/null +++ b/fleet/lib/netbox/README.md @@ -0,0 +1,59 @@ +# NetBox Kubernetes Deployment + +## Overview + +NetBox is an Infrastructure Resource Modeling (IRM) application designed to empower network automation. This deployment provides a production-ready NetBox instance on Kubernetes using Helm charts and Fleet configuration management. + +## Architecture + +- **Chart**: netbox v6.1.5 from +- **Namespace**: netbox +- **Components**: Web application, worker processes, PostgreSQL database, Valkey cache +- **Ingress**: NGINX with Let's Encrypt TLS certificates + +## Configuration + +### Core Settings + +- **Timezone**: America/Santiago +- **Superuser**: +- **Change Log Retention**: 90 days +- **Job Retention**: 90 days +- **GraphQL**: Enabled +- **Login Required**: False + +### Security + +- Non-root container execution (UID/GID: 1000) +- Read-only root filesystem +- Dropped capabilities +- Runtime security profile enabled +- External secret management via Kubernetes secrets + +### Storage + +- **Persistence**: Disabled (ephemeral storage) +- **PostgreSQL**: 20Gi persistent storage (rook-ceph-block) +- **Media/Reports/Scripts**: Stored in ephemeral volumes + +### Resources + +| Component | CPU Request | Memory Request | CPU Limit | Memory Limit | +|-----------|-------------|----------------|-----------|--------------| +| NetBox | 500m | 1Gi | 1000m | 2Gi | +| Worker | 500m | 1Gi | 1000m | 2Gi | +| PostgreSQL| 250m | 512Mi | 500m | 1Gi | + +## Access + +NetBox is accessible via dynamically generated hostnames based on cluster configuration: + +```bash +https://netbox.{cluster-name}.{site}.lsst.org +``` + +## Maintenance + +- **Housekeeping**: Daily automated cleanup (00:00 UTC) +- **Job History**: 5 successful/failed jobs retained +- **Monitoring**: Available via cluster monitoring stack diff --git a/fleet/lib/netbox/base/configmap-keycloak-backend.yaml b/fleet/lib/netbox/base/configmap-keycloak-backend.yaml new file mode 100644 index 000000000..76d4b7faf --- /dev/null +++ b/fleet/lib/netbox/base/configmap-keycloak-backend.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: netbox-keycloak-backend + namespace: netbox +data: + keycloak_backend.py: | + from social_core.backends.keycloak import KeycloakOAuth2 + + class KeycloakOAuth2NoJWT(KeycloakOAuth2): + """Custom Keycloak backend that skips JWT validation""" + + def public_key(self): + """Override to skip public key validation""" + return None + + def user_data(self, access_token, *args, **kwargs): + """Get user data from userinfo endpoint without JWT validation""" + # Get USERINFO URL from settings + url = self.setting('USERINFO_URL') or self.USERINFO_URL + return self.get_json( + url, + headers={'Authorization': f'Bearer {access_token}'} + ) diff --git a/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml new file mode 100644 index 000000000..e1af722b5 --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-keycloak.yaml @@ -0,0 +1,25 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-keycloak + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-keycloak + creationPolicy: Owner + data: + - secretKey: SOCIAL_AUTH_KEYCLOAK_KEY + remoteRef: + key: &item netbox-keycloak-sso + property: client-id + - secretKey: SOCIAL_AUTH_KEYCLOAK_SECRET + remoteRef: + key: *item + property: client-secret + - secretKey: url + remoteRef: + key: *item + property: url diff --git a/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml new file mode 100644 index 000000000..3c0792f41 --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-postgresql.yaml @@ -0,0 +1,17 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-postgresql + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-postgresql + creationPolicy: Owner + data: + - secretKey: db_password + remoteRef: + key: &item cnpg-cluster-superuser-kueyen + property: password diff --git a/fleet/lib/netbox/base/externalsecret-netbox-secrets.yaml b/fleet/lib/netbox/base/externalsecret-netbox-secrets.yaml new file mode 100644 index 000000000..a6e25c74d --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-secrets.yaml @@ -0,0 +1,33 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-secrets + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-secrets + creationPolicy: Owner + data: + - secretKey: username + remoteRef: + key: &item netbox-secrets + property: username + - secretKey: password + remoteRef: + key: *item + property: password + - secretKey: email + remoteRef: + key: *item + property: email + - secretKey: api_token + remoteRef: + key: *item + property: apiToken + - secretKey: secret-key + remoteRef: + key: *item + property: secretKey diff --git a/fleet/lib/netbox/base/externalsecret-netbox-valkey.yaml b/fleet/lib/netbox/base/externalsecret-netbox-valkey.yaml new file mode 100644 index 000000000..e91338163 --- /dev/null +++ b/fleet/lib/netbox/base/externalsecret-netbox-valkey.yaml @@ -0,0 +1,17 @@ +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: netbox-valkey + namespace: netbox +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword + target: + name: netbox-valkey + creationPolicy: Owner + data: + - secretKey: valkey-password + remoteRef: + key: &item netbox-valkey + property: password diff --git a/fleet/lib/netbox/base/kustomization.yaml b/fleet/lib/netbox/base/kustomization.yaml new file mode 100644 index 000000000..c9934ff6e --- /dev/null +++ b/fleet/lib/netbox/base/kustomization.yaml @@ -0,0 +1,7 @@ +--- +resources: + - externalsecret-netbox-secrets.yaml + - externalsecret-netbox-valkey.yaml + - externalsecret-netbox-postgresql.yaml + - externalsecret-netbox-keycloak.yaml + - configmap-keycloak-backend.yaml diff --git a/fleet/lib/netbox/fleet.yaml b/fleet/lib/netbox/fleet.yaml new file mode 100644 index 000000000..9526b1244 --- /dev/null +++ b/fleet/lib/netbox/fleet.yaml @@ -0,0 +1,24 @@ +--- +defaultNamespace: &name netbox +labels: + bundle: *name +namespaceLabels: + lsst.io/discover: "true" +kustomize: + dir: base +helm: + chart: &chart netbox + releaseName: *chart + repo: https://charts.netbox.oss.netboxlabs.com/ + version: 6.1.5 + timeoutSeconds: 600 + waitForJobs: true + valuesFiles: + - values.yaml +dependsOn: +- selector: + matchLabels: + bundle: external-secrets +- selector: + matchLabels: + bundle: cnpg-cluster diff --git a/fleet/lib/netbox/values.yaml b/fleet/lib/netbox/values.yaml new file mode 100644 index 000000000..3191080d4 --- /dev/null +++ b/fleet/lib/netbox/values.yaml @@ -0,0 +1,156 @@ +nameOverride: netbox +clusterDomain: cluster.local + +superuser: + name: admin + email: rubbinobs-it@lsst.org + existingSecret: netbox-secrets + + +allowedHosts: + - netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org + +allowedHostsIncludesPodIP: false + +admins: + - [Admin User, rubbinobs-it@lsst.org] + +internalIPs: [127.0.0.1] + +timeZone: America/Santiago + +# Disable persistent volumes to avoid multi-attach issues with ReadWriteOnce storage +# Media files will be stored in ephemeral storage +persistence: + enabled: false + +resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 1000m + memory: 2Gi + +## @section Traffic Exposure Parameters + +ingress: + enabled: true + className: nginx + annotations: + cert-manager.io/cluster-issuer: letsencrypt + nginx.ingress.kubernetes.io/backend-protocol: HTTP + nginx.ingress.kubernetes.io/client-body-buffer-size: 10m + nginx.ingress.kubernetes.io/proxy-read-timeout: "60" + nginx.ingress.kubernetes.io/proxy-send-timeout: "60" + hosts: + - host: netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org + paths: + - / + tls: + - secretName: netbox-tls + hosts: + - netbox.${ get .ClusterLabels "management.cattle.io/cluster-display-name" }.${ .ClusterLabels.site }.lsst.org + +externalDatabase: + enabled: true + host: cnpg-loadbalancer.cloudnativepg.svc.cluster.local + port: 5432 + user: postgres + database: netbox + existingSecret: netbox-postgresql + existingSecretPasswordKey: postgres-password + +postgresql: + enabled: false + +valkey: + enabled: true + auth: + existingSecret: netbox-valkey + existingSecretPasswordKey: valkey-password + +## @section Worker for Netbox parameters + +worker: + enabled: false +## @section Cron housekeeping job parameters + +housekeeping: + enabled: false +# Database configuration using external secrets +extraEnvs: + - name: DB_WAIT_DEBUG + value: "1" + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: netbox-secrets + key: secret-key + - name: SOCIAL_AUTH_KEYCLOAK_KEY + valueFrom: + secretKeyRef: + name: netbox-keycloak + key: SOCIAL_AUTH_KEYCLOAK_KEY + - name: SOCIAL_AUTH_KEYCLOAK_SECRET + valueFrom: + secretKeyRef: + name: netbox-keycloak + key: SOCIAL_AUTH_KEYCLOAK_SECRET + +# Plugin configuration +plugins: [] + +# Keycloak OAuth2 configuration - using custom backend to avoid JWT validation issues +extraConfig: + - secretName: "" + data: | + import os + import sys + + # Add custom backend path + sys.path.insert(0, '/opt/netbox/custom_backends') + + # Keycloak OAuth2 Configuration + SOCIAL_AUTH_KEYCLOAK_KEY = os.environ.get('SOCIAL_AUTH_KEYCLOAK_KEY') + SOCIAL_AUTH_KEYCLOAK_SECRET = os.environ.get('SOCIAL_AUTH_KEYCLOAK_SECRET') + SOCIAL_AUTH_KEYCLOAK_AUTHORIZATION_URL = '$__file{/etc/secrets/keycloak-credentials/url}/realms/master/protocol/openid-connect/auth' + SOCIAL_AUTH_KEYCLOAK_ACCESS_TOKEN_URL = '$__file{/etc/secrets/keycloak-credentials/url}/realms/master/protocol/openid-connect/token' + SOCIAL_AUTH_KEYCLOAK_USERINFO_URL = '$__file{/etc/secrets/keycloak-credentials/url}/realms/master/protocol/openid-connect/userinfo' + SOCIAL_AUTH_KEYCLOAK_ID_KEY = 'preferred_username' + SOCIAL_AUTH_REDIRECT_IS_HTTPS = True + + # Social Auth user pipeline + SOCIAL_AUTH_USERNAME_IS_FULL_EMAIL = False + SOCIAL_AUTH_USER_FIELDS = ['username', 'email', 'first_name', 'last_name'] + + # Request proper scopes + SOCIAL_AUTH_KEYCLOAK_SCOPE = ['openid', 'profile', 'email'] + + # Redirect URI handling + SOCIAL_AUTH_KEYCLOAK_IGNORE_DEFAULT_SCOPE = True + SOCIAL_AUTH_TRAILING_SLASH = False + + # Extra data to fetch from Keycloak + SOCIAL_AUTH_KEYCLOAK_EXTRA_DATA = ['email', 'preferred_username', 'given_name', 'family_name'] + + # Authentication method for token endpoint + SOCIAL_AUTH_KEYCLOAK_AUTH_EXTRA_ARGUMENTS = {'client_id': os.environ.get('SOCIAL_AUTH_KEYCLOAK_KEY')} + +## @section Remote Authentication (SSO) Parameters + +remoteAuth: + enabled: true + backends: + - keycloak_backend.KeycloakOAuth2NoJWT + +# Mount custom Keycloak backend to avoid JWT validation issues +extraVolumes: + - name: keycloak-backend + configMap: + name: netbox-keycloak-backend + +extraVolumeMounts: + - name: keycloak-backend + mountPath: /opt/netbox/custom_backends + readOnly: true diff --git a/fleet/lib/rook-ceph-conf/fleet.yaml b/fleet/lib/rook-ceph-conf/fleet.yaml index f752a6a92..bc7f0ead9 100644 --- a/fleet/lib/rook-ceph-conf/fleet.yaml +++ b/fleet/lib/rook-ceph-conf/fleet.yaml @@ -26,18 +26,6 @@ diff: - /spec/preservePoolsOnDelete - /spec/healthCheck targetCustomizations: - - name: kueyen - clusterSelector: - matchExpressions: - - key: management.cattle.io/cluster-display-name - operator: In - values: - - kueyen - helm: - values: - subchart: - kueyen: - enabled: true - name: ayekan clusterSelector: matchExpressions: diff --git a/fleet/s/dev/c/kueyen/cnpg-cluster b/fleet/s/dev/c/kueyen/cnpg-cluster new file mode 120000 index 000000000..702632008 --- /dev/null +++ b/fleet/s/dev/c/kueyen/cnpg-cluster @@ -0,0 +1 @@ +../../../../lib/cnpg-cluster \ No newline at end of file diff --git a/fleet/s/dev/c/kueyen/cnpg-system b/fleet/s/dev/c/kueyen/cnpg-system new file mode 120000 index 000000000..d67b55bb2 --- /dev/null +++ b/fleet/s/dev/c/kueyen/cnpg-system @@ -0,0 +1 @@ +../../../../lib/cnpg-system \ No newline at end of file diff --git a/fleet/s/dev/c/kueyen/fluent-bit-kube b/fleet/s/dev/c/kueyen/fluent-bit-kube deleted file mode 120000 index 340224dca..000000000 --- a/fleet/s/dev/c/kueyen/fluent-bit-kube +++ /dev/null @@ -1 +0,0 @@ -../../../../lib/fluent-bit-kube \ No newline at end of file diff --git a/fleet/s/dev/c/kueyen/loki b/fleet/s/dev/c/kueyen/loki deleted file mode 120000 index 874625b54..000000000 --- a/fleet/s/dev/c/kueyen/loki +++ /dev/null @@ -1 +0,0 @@ -../../../../lib/loki \ No newline at end of file diff --git a/fleet/s/dev/c/kueyen/mimir b/fleet/s/dev/c/kueyen/mimir deleted file mode 120000 index 30cd3d61a..000000000 --- a/fleet/s/dev/c/kueyen/mimir +++ /dev/null @@ -1 +0,0 @@ -../../../../lib/mimir \ No newline at end of file diff --git a/fleet/s/dev/c/kueyen/netbox b/fleet/s/dev/c/kueyen/netbox new file mode 120000 index 000000000..ccfcf159f --- /dev/null +++ b/fleet/s/dev/c/kueyen/netbox @@ -0,0 +1 @@ +../../../../lib/netbox \ No newline at end of file