From d5efa755ddea7772687788a0da2fec077eb664d1 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Tue, 13 Jan 2026 01:46:31 -0300
Subject: [PATCH 01/16] #971 - add Argo Workflows manifests

---
 .../environments/eks/argo-workflow/README.md  | 35 ++++++++
 .../argo-basic-auth-externalsecret.yaml       | 18 ++++
 .../argo-workflow/argo-server-ingress.yaml    | 45 ++++++++++
 .../eks/argo-workflow/argo-server.yaml        | 83 +++++++++++++++++
 .../eks/argo-workflow/configmap.yaml          | 29 ++++++
 .../eks/argo-workflow/externalsecret.yaml     | 38 ++++++++
 .../eks/argo-workflow/namespace.yaml          |  7 ++
 .../environments/eks/argo-workflow/rbac.yaml  | 85 ++++++++++++++++++
 .../workflow-controller-deployment.yaml       | 90 +++++++++++++++++++
 9 files changed, 430 insertions(+)
 create mode 100644 terraform/environments/eks/argo-workflow/README.md
 create mode 100644 terraform/environments/eks/argo-workflow/argo-basic-auth-externalsecret.yaml
 create mode 100644 terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
 create mode 100644 terraform/environments/eks/argo-workflow/argo-server.yaml
 create mode 100644 terraform/environments/eks/argo-workflow/configmap.yaml
 create mode 100644 terraform/environments/eks/argo-workflow/externalsecret.yaml
 create mode 100644 terraform/environments/eks/argo-workflow/namespace.yaml
 create mode 100644 terraform/environments/eks/argo-workflow/rbac.yaml
 create mode 100644 terraform/environments/eks/argo-workflow/workflow-controller-deployment.yaml

diff --git a/terraform/environments/eks/argo-workflow/README.md b/terraform/environments/eks/argo-workflow/README.md
new file mode 100644
index 00000000..8228d7bb
--- /dev/null
+++ b/terraform/environments/eks/argo-workflow/README.md
@@ -0,0 +1,35 @@
+# Argo Workflows
+
+These manifests install a minimal Argo Workflows control plane into the `argo` namespace. The controller and server components rely on a shared PostgreSQL database (for example, the RDS modules under `terraform/environments/eks`) for workflow persistence.
+
+## Components
+- `namespace.yaml` – declares the `argo` namespace.
+- `externalsecret.yaml` – syncs the AWS Secrets Manager entry `credreg-argo-workflows` into a Kubernetes Secret named `argo-postgres`.
+- `configmap.yaml` – controller configuration that enables Postgres-based persistence; set the host/database here, while credentials come from the synced secret.
+- `rbac.yaml` – service accounts plus the RBAC needed by the workflow controller and Argo server.
+- `workflow-controller-deployment.yaml` – runs `workflow-controller` with the standard `argoexec` image.
+- `argo-server.yaml` – exposes the Argo UI/API inside the cluster on port `2746`.
+- `argo-basic-auth-externalsecret.yaml` – syncs the AWS Secrets Manager entry `credreg-argo-basic-auth` (or similar) to supply the base64-encoded `user:password` string for ingress auth.
+- `argo-server-ingress.yaml` – optional HTTPS ingress + certificate (via cert-manager + Let’s Encrypt) and basic auth for external access to the Argo UI.
+
+## Before applying
+1. **Provision or reference a PostgreSQL instance.** Ensure the desired environment has a reachable database endpoint.
+2. **Create the Secrets Manager entry.** Create `credreg-argo-workflows` (or adjust the `remoteRef.key` value) with JSON keys `host`, `port`, `database`, `username`, `password`, `sslmode`. The External Secrets Operator will sync it into the cluster and the controller/server pick them up via env vars.
+3. **Update `configmap.yaml`.** Set `persistence.postgresql.host` (and database/table names if they differ) for the target environment. Even though credentials are secret-backed, Argo still requires the host in this config.
+4. **Install Argo CRDs.** Apply the upstream CRDs from https://github.com/argoproj/argo-workflows/releases (required only once per cluster) before rolling out these manifests.
+5. **Configure DNS if using the ingress.** Update `argo-server-ingress.yaml` with the desired hostname(s) and point the DNS record at the ingress controller’s load balancer.
+
+## Apply order
+```bash
+kubectl apply -f terraform/environments/eks/argo-workflow/namespace.yaml
+kubectl apply -f terraform/environments/eks/argo-workflow/externalsecret.yaml
+kubectl apply -f terraform/environments/eks/argo-workflow/rbac.yaml
+kubectl apply -f terraform/environments/eks/argo-workflow/configmap.yaml
+kubectl apply -f terraform/environments/eks/argo-workflow/workflow-controller-deployment.yaml
+kubectl apply -f terraform/environments/eks/argo-workflow/argo-server.yaml
+# Optional ingress / certificate
+kubectl apply -f terraform/environments/eks/argo-workflow/argo-basic-auth-externalsecret.yaml
+kubectl apply -f terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
+```
+
+Once the `argo-postgres` secret is synced and the controller connects to Postgres successfully, `kubectl get wf -n argo` should show persisted workflows even after pod restarts.
diff --git a/terraform/environments/eks/argo-workflow/argo-basic-auth-externalsecret.yaml b/terraform/environments/eks/argo-workflow/argo-basic-auth-externalsecret.yaml
new file mode 100644
index 00000000..72299c59
--- /dev/null
+++ b/terraform/environments/eks/argo-workflow/argo-basic-auth-externalsecret.yaml
@@ -0,0 +1,18 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: argo-basic-auth
+  namespace: argo
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: aws-secret-manager
+    kind: ClusterSecretStore
+  target:
+    name: argo-basic-auth
+    creationPolicy: Owner
+  data:
+    - secretKey: auth
+      remoteRef:
+        key: credreg-argo-basic-auth
+        property: auth
diff --git a/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml b/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
new file mode 100644
index 00000000..e47cf359
--- /dev/null
+++ b/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
@@ -0,0 +1,45 @@
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: argo-server-cert
+  namespace: argo
+spec:
+  secretName: argo-server-tls
+  issuerRef:
+    kind: ClusterIssuer
+    name: letsencrypt-prod
+  dnsNames:
+    - argo.credentialengineregistry.org
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: argo-server
+  namespace: argo
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt-prod
+    nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
+    nginx.ingress.kubernetes.io/ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/auth-type: "basic"
+    nginx.ingress.kubernetes.io/auth-secret: "argo-basic-auth"
+    nginx.ingress.kubernetes.io/auth-realm: "Authentication Required"
+    nginx.ingress.kubernetes.io/proxy-body-size: "10m"
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
+spec:
+  ingressClassName: nginx
+  tls:
+    - hosts:
+        - argo.credentialengineregistry.org
+      secretName: argo-server-tls
+  rules:
+    - host: argo.credentialengineregistry.org
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: argo-server
+                port:
+                  number: 2746
diff --git a/terraform/environments/eks/argo-workflow/argo-server.yaml b/terraform/environments/eks/argo-workflow/argo-server.yaml
new file mode 100644
index 00000000..11c827ca
--- /dev/null
+++ b/terraform/environments/eks/argo-workflow/argo-server.yaml
@@ -0,0 +1,83 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: argo-server
+  namespace: argo
+  labels:
+    app.kubernetes.io/name: argo-server
+    app.kubernetes.io/part-of: argo-workflows
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: argo-server
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: argo-server
+        app.kubernetes.io/part-of: argo-workflows
+    spec:
+      serviceAccountName: argo-server
+      containers:
+        - name: argo-server
+          image: quay.io/argoproj/argocli:v3.7.7
+          imagePullPolicy: IfNotPresent
+          args:
+            - server
+            - --auth-mode
+            - server
+            - --namespaced
+            - --namespace
+            - argo
+            - --configmap
+            - workflow-controller-configmap
+          envFrom:
+            - secretRef:
+                name: argo-postgres
+          ports:
+            - containerPort: 2746
+              name: web
+          livenessProbe:
+            httpGet:
+              scheme: HTTPS
+              path: /healthz
+              port: web
+              httpHeaders:
+                - name: Host
+                  value: localhost
+            initialDelaySeconds: 10
+            periodSeconds: 30
+          readinessProbe:
+            httpGet:
+              scheme: HTTPS
+              path: /healthz
+              port: web
+              httpHeaders:
+                - name: Host
+                  value: localhost
+            initialDelaySeconds: 10
+            periodSeconds: 15
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: argo-server
+  namespace: argo
+  labels:
+    app.kubernetes.io/name: argo-server
+    app.kubernetes.io/part-of: argo-workflows
+spec:
+  selector:
+    app.kubernetes.io/name: argo-server
+  type: ClusterIP
+  ports:
+    - name: web
+      port: 2746
+      targetPort: web
diff --git a/terraform/environments/eks/argo-workflow/configmap.yaml b/terraform/environments/eks/argo-workflow/configmap.yaml
new file mode 100644
index 00000000..4fa48ea1
--- /dev/null
+++ b/terraform/environments/eks/argo-workflow/configmap.yaml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: workflow-controller-configmap
+  namespace: argo
+data:
+  config: |
+    metricsConfig:
+      enabled: false
+      secure: true
+    telemetryConfig:
+      enabled: false
+      secure: true
+    namespace: argo
+    persistence:
+      archive: true
+      nodeStatusOffload: true
+      postgresql:
+        host: argo-workflows.cwdkv5tua6nq.us-east-1.rds.amazonaws.com
+        port: 5432
+        database: argo_workflows
+        tableName: argo_workflows
+        sslMode: require
+        userNameSecret:
+          name: argo-postgres
+          key: username
+        passwordSecret:
+          name: argo-postgres
+          key: password
diff --git a/terraform/environments/eks/argo-workflow/externalsecret.yaml b/terraform/environments/eks/argo-workflow/externalsecret.yaml
new file mode 100644
index 00000000..dd4e1085
--- /dev/null
+++ b/terraform/environments/eks/argo-workflow/externalsecret.yaml
@@ -0,0 +1,38 @@
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: argo-postgres
+  namespace: argo
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    name: aws-secret-manager
+    kind: ClusterSecretStore
+  target:
+    name: argo-postgres
+    creationPolicy: Owner
+  data:
+    - secretKey: host
+      remoteRef:
+        key: credreg-argo-workflows
+        property: host
+    - secretKey: port
+      remoteRef:
+        key: credreg-argo-workflows
+        property: port
+    - secretKey: database
+      remoteRef:
+        key: credreg-argo-workflows
+        property: database
+    - secretKey: username
+      remoteRef:
+        key: credreg-argo-workflows
+        property: username
+    - secretKey: password
+      remoteRef:
+        key: credreg-argo-workflows
+        property: password
+    - secretKey: sslmode
+      remoteRef:
+        key: credreg-argo-workflows
+        property: sslmode
diff --git a/terraform/environments/eks/argo-workflow/namespace.yaml b/terraform/environments/eks/argo-workflow/namespace.yaml
new file mode 100644
index 00000000..93edba60
--- /dev/null
+++ b/terraform/environments/eks/argo-workflow/namespace.yaml
@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: argo
+  labels:
+    app.kubernetes.io/name: argo
+    app.kubernetes.io/part-of: argo-workflows
diff --git a/terraform/environments/eks/argo-workflow/rbac.yaml b/terraform/environments/eks/argo-workflow/rbac.yaml
new file mode 100644
index 00000000..324cd70c
--- /dev/null
+++ b/terraform/environments/eks/argo-workflow/rbac.yaml
@@ -0,0 +1,85 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: argo-workflow-controller
+  namespace: argo
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/part-of: argo-workflows
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: argo-server
+  namespace: argo
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/part-of: argo-workflows
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: argo-workflow-controller
+  labels:
+    app.kubernetes.io/part-of: argo-workflows
+rules:
+  - apiGroups: ["argoproj.io"]
+    resources: ["workflowtasksets", "workflowtasksets/status", "workflowartifactgctasks", "workflows", "workflows/finalizers", "workflows/status", "workflowtemplates", "cronworkflows", "clusterworkflowtemplates", "clusterworkflowtemplates/finalizers", "workflowtaskresults"]
+    verbs: ["*"]
+  - apiGroups: [""]
+    resources: ["configmaps", "persistentvolumeclaims", "pods", "pods/log", "pods/exec", "secrets", "serviceaccounts", "services", "events"]
+    verbs: ["*"]
+  - apiGroups: ["apps"]
+    resources: ["deployments", "replicasets", "statefulsets"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "create", "update", "patch"]
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["create", "delete", "get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: argo-workflow-controller
+  labels:
+    app.kubernetes.io/part-of: argo-workflows
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: argo-workflow-controller
+subjects:
+  - kind: ServiceAccount
+    name: argo-workflow-controller
+    namespace: argo
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: argo-server
+  namespace: argo
+rules:
+  - apiGroups: ["argoproj.io"]
+    resources: ["workflows", "workflowtemplates", "cronworkflows"]
+    verbs: ["*"]
+  - apiGroups: [""]
+    resources: ["configmaps", "secrets", "pods", "pods/log", "services"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "patch", "update"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: argo-server
+  namespace: argo
+subjects:
+  - kind: ServiceAccount
+    name: argo-server
+    namespace: argo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: argo-server
diff --git a/terraform/environments/eks/argo-workflow/workflow-controller-deployment.yaml b/terraform/environments/eks/argo-workflow/workflow-controller-deployment.yaml
new file mode 100644
index 00000000..2a135176
--- /dev/null
+++ b/terraform/environments/eks/argo-workflow/workflow-controller-deployment.yaml
@@ -0,0 +1,90 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: workflow-controller
+  namespace: argo
+  labels:
+    app.kubernetes.io/name: workflow-controller
+    app.kubernetes.io/part-of: argo-workflows
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: workflow-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: workflow-controller
+        app.kubernetes.io/part-of: argo-workflows
+    spec:
+      serviceAccountName: argo-workflow-controller
+      containers:
+        - name: workflow-controller
+          image: quay.io/argoproj/workflow-controller:v3.7.7
+          imagePullPolicy: IfNotPresent
+          args:
+            - --configmap
+            - workflow-controller-configmap
+            - --executor-image
+            - quay.io/argoproj/argoexec:v3.7.7
+          env:
+            - name: LEADER_ELECTION_IDENTITY
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: ARGO_POSTGRES_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: argo-postgres
+                  key: host
+            - name: ARGO_POSTGRES_PORT
+              valueFrom:
+                secretKeyRef:
+                  name: argo-postgres
+                  key: port
+            - name: ARGO_POSTGRES_DB
+              valueFrom:
+                secretKeyRef:
+                  name: argo-postgres
+                  key: database
+            - name: ARGO_POSTGRES_USERNAME
+              valueFrom:
+                secretKeyRef:
+                  name: argo-postgres
+                  key: username
+            - name: ARGO_POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: argo-postgres
+                  key: password
+            - name: ARGO_POSTGRES_SSLMODE
+              valueFrom:
+                secretKeyRef:
+                  name: argo-postgres
+                  key: sslmode
+          ports:
+            - containerPort: 9090
+              name: metrics
+          livenessProbe:
+            httpGet:
+              port: 6060
+              path: /healthz
+            failureThreshold: 3
+            initialDelaySeconds: 90
+            periodSeconds: 60
+            timeoutSeconds: 30
+          readinessProbe:
+            httpGet:
+              port: 6060
+              path: /healthz
+            failureThreshold: 3
+            initialDelaySeconds: 90
+            periodSeconds: 60
+            timeoutSeconds: 30
+          resources:
+            requests:
+              cpu: 100m
+              memory: 256Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi

From c37c9fb88ad76ecc44793c22bdafc772980ec4f7 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Tue, 13 Jan 2026 18:15:34 -0300
Subject: [PATCH 02/16] add IP restrictions

---
 .../environments/eks/argo-workflow/argo-server-ingress.yaml      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml b/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
index e47cf359..746ad754 100644
--- a/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
+++ b/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
@@ -26,6 +26,7 @@ metadata:
     nginx.ingress.kubernetes.io/proxy-body-size: "10m"
     nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
     nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
+    nginx.ingress.kubernetes.io/whitelist-source-range: "67.40.27.250/32,98.13.197.1/32,129.224.215.205/32"
 spec:
   ingressClassName: nginx
   tls:

From 72177343b52cc3bcd2011bf128d163b5ab98c15f Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Tue, 13 Jan 2026 18:17:50 -0300
Subject: [PATCH 03/16] add CE ip add

---
 .../environments/eks/argo-workflow/argo-server-ingress.yaml     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml b/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
index 746ad754..8068c0b9 100644
--- a/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
+++ b/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
@@ -26,7 +26,7 @@ metadata:
     nginx.ingress.kubernetes.io/proxy-body-size: "10m"
     nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
     nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
-    nginx.ingress.kubernetes.io/whitelist-source-range: "67.40.27.250/32,98.13.197.1/32,129.224.215.205/32"
+    nginx.ingress.kubernetes.io/whitelist-source-range: "71.212.64.155/32,129.224.215.205/32"
 spec:
   ingressClassName: nginx
   tls:

From 3342f77286b1bfc2d2d35a47d749ac49589ca504 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Tue, 13 Jan 2026 19:22:13 -0300
Subject: [PATCH 04/16] adapting for staging namespace/environment

---
 .../eks/argo-workflow/namespace.yaml          |  7 -------
 .../argo-workflow/README.md                   | 20 +++++++++----------
 .../argo-basic-auth-externalsecret.yaml       |  2 +-
 .../argo-workflow/argo-server-ingress.yaml    | 10 +++++-----
 .../argo-workflow/argo-server.yaml            |  6 +++---
 .../argo-workflow/configmap.yaml              |  6 +++---
 .../argo-workflow/externalsecret.yaml         | 14 ++++++-------
 .../argo-workflow/rbac.yaml                   | 12 +++++------
 .../argo-workflow/wf.json                     | 20 +++++++++++++++++++
 .../workflow-controller-deployment.yaml       |  2 +-
 10 files changed, 55 insertions(+), 44 deletions(-)
 delete mode 100644 terraform/environments/eks/argo-workflow/namespace.yaml
 rename terraform/environments/eks/{ => k8s-manifests-staging}/argo-workflow/README.md (67%)
 rename terraform/environments/eks/{ => k8s-manifests-staging}/argo-workflow/argo-basic-auth-externalsecret.yaml (92%)
 rename terraform/environments/eks/{ => k8s-manifests-staging}/argo-workflow/argo-server-ingress.yaml (85%)
 rename terraform/environments/eks/{ => k8s-manifests-staging}/argo-workflow/argo-server.yaml (95%)
 rename terraform/environments/eks/{ => k8s-manifests-staging}/argo-workflow/configmap.yaml (80%)
 rename terraform/environments/eks/{ => k8s-manifests-staging}/argo-workflow/externalsecret.yaml (69%)
 rename terraform/environments/eks/{ => k8s-manifests-staging}/argo-workflow/rbac.yaml (92%)
 create mode 100644 terraform/environments/eks/k8s-manifests-staging/argo-workflow/wf.json
 rename terraform/environments/eks/{ => k8s-manifests-staging}/argo-workflow/workflow-controller-deployment.yaml (98%)

diff --git a/terraform/environments/eks/argo-workflow/namespace.yaml b/terraform/environments/eks/argo-workflow/namespace.yaml
deleted file mode 100644
index 93edba60..00000000
--- a/terraform/environments/eks/argo-workflow/namespace.yaml
+++ /dev/null
@@ -1,7 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: argo
-  labels:
-    app.kubernetes.io/name: argo
-    app.kubernetes.io/part-of: argo-workflows
diff --git a/terraform/environments/eks/argo-workflow/README.md b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/README.md
similarity index 67%
rename from terraform/environments/eks/argo-workflow/README.md
rename to terraform/environments/eks/k8s-manifests-staging/argo-workflow/README.md
index 8228d7bb..7549e71c 100644
--- a/terraform/environments/eks/argo-workflow/README.md
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/README.md
@@ -1,9 +1,8 @@
 # Argo Workflows
 
-These manifests install a minimal Argo Workflows control plane into the `argo` namespace. The controller and server components rely on a shared PostgreSQL database (for example, the RDS modules under `terraform/environments/eks`) for workflow persistence.
+These manifests install a minimal Argo Workflows control plane into the shared `credreg-staging` namespace. The controller and server components rely on a shared PostgreSQL database (for example, the RDS modules under `terraform/environments/eks`) for workflow persistence.
 
 ## Components
-- `namespace.yaml` – declares the `argo` namespace.
 - `externalsecret.yaml` – syncs the AWS Secrets Manager entry `credreg-argo-workflows` into a Kubernetes Secret named `argo-postgres`.
 - `configmap.yaml` – controller configuration that enables Postgres-based persistence; set the host/database here, while credentials come from the synced secret.
 - `rbac.yaml` – service accounts plus the RBAC needed by the workflow controller and Argo server.
@@ -21,15 +20,14 @@ These manifests install a minimal Argo Workflows control plane into the `argo` n
 
 ## Apply order
 ```bash
-kubectl apply -f terraform/environments/eks/argo-workflow/namespace.yaml
-kubectl apply -f terraform/environments/eks/argo-workflow/externalsecret.yaml
-kubectl apply -f terraform/environments/eks/argo-workflow/rbac.yaml
-kubectl apply -f terraform/environments/eks/argo-workflow/configmap.yaml
-kubectl apply -f terraform/environments/eks/argo-workflow/workflow-controller-deployment.yaml
-kubectl apply -f terraform/environments/eks/argo-workflow/argo-server.yaml
+kubectl apply -f terraform/environments/eks/k8s-manifests-staging/argo-workflow/externalsecret.yaml
+kubectl apply -f terraform/environments/eks/k8s-manifests-staging/argo-workflow/rbac.yaml
+kubectl apply -f terraform/environments/eks/k8s-manifests-staging/argo-workflow/configmap.yaml
+kubectl apply -f terraform/environments/eks/k8s-manifests-staging/argo-workflow/workflow-controller-deployment.yaml
+kubectl apply -f terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server.yaml
 # Optional ingress / certificate
-kubectl apply -f terraform/environments/eks/argo-workflow/argo-basic-auth-externalsecret.yaml
-kubectl apply -f terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
+kubectl apply -f terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-basic-auth-externalsecret.yaml
+kubectl apply -f terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server-ingress.yaml
 ```
 
-Once the `argo-postgres` secret is synced and the controller connects to Postgres successfully, `kubectl get wf -n argo` should show persisted workflows even after pod restarts.
+Once the `argo-postgres` secret is synced and the controller connects to Postgres successfully, `kubectl get wf -n credreg-staging` should show persisted workflows even after pod restarts.
diff --git a/terraform/environments/eks/argo-workflow/argo-basic-auth-externalsecret.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-basic-auth-externalsecret.yaml
similarity index 92%
rename from terraform/environments/eks/argo-workflow/argo-basic-auth-externalsecret.yaml
rename to terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-basic-auth-externalsecret.yaml
index 72299c59..bd415a38 100644
--- a/terraform/environments/eks/argo-workflow/argo-basic-auth-externalsecret.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-basic-auth-externalsecret.yaml
@@ -2,7 +2,7 @@ apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
   name: argo-basic-auth
-  namespace: argo
+  namespace: credreg-staging
 spec:
   refreshInterval: 1h
   secretStoreRef:
diff --git a/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server-ingress.yaml
similarity index 85%
rename from terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
rename to terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server-ingress.yaml
index 8068c0b9..caa32520 100644
--- a/terraform/environments/eks/argo-workflow/argo-server-ingress.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server-ingress.yaml
@@ -2,20 +2,20 @@ apiVersion: cert-manager.io/v1
 kind: Certificate
 metadata:
   name: argo-server-cert
-  namespace: argo
+  namespace: credreg-staging
 spec:
   secretName: argo-server-tls
   issuerRef:
     kind: ClusterIssuer
     name: letsencrypt-prod
   dnsNames:
-    - argo.credentialengineregistry.org
+    - argo-staging.credentialengineregistry.org
 ---
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: argo-server
-  namespace: argo
+  namespace: credreg-staging
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt-prod
     nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
@@ -31,10 +31,10 @@ spec:
   ingressClassName: nginx
   tls:
     - hosts:
-        - argo.credentialengineregistry.org
+        - argo-staging.credentialengineregistry.org
       secretName: argo-server-tls
   rules:
-    - host: argo.credentialengineregistry.org
+    - host: argo-staging.credentialengineregistry.org
       http:
         paths:
           - path: /
diff --git a/terraform/environments/eks/argo-workflow/argo-server.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server.yaml
similarity index 95%
rename from terraform/environments/eks/argo-workflow/argo-server.yaml
rename to terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server.yaml
index 11c827ca..e1ff3150 100644
--- a/terraform/environments/eks/argo-workflow/argo-server.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server.yaml
@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: argo-server
-  namespace: argo
+  namespace: credreg-staging
   labels:
     app.kubernetes.io/name: argo-server
     app.kubernetes.io/part-of: argo-workflows
@@ -28,7 +28,7 @@ spec:
             - server
             - --namespaced
             - --namespace
-            - argo
+            - credreg-staging
             - --configmap
             - workflow-controller-configmap
           envFrom:
@@ -69,7 +69,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: argo-server
-  namespace: argo
+  namespace: credreg-staging
   labels:
     app.kubernetes.io/name: argo-server
     app.kubernetes.io/part-of: argo-workflows
diff --git a/terraform/environments/eks/argo-workflow/configmap.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/configmap.yaml
similarity index 80%
rename from terraform/environments/eks/argo-workflow/configmap.yaml
rename to terraform/environments/eks/k8s-manifests-staging/argo-workflow/configmap.yaml
index 4fa48ea1..4878162c 100644
--- a/terraform/environments/eks/argo-workflow/configmap.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/configmap.yaml
@@ -2,7 +2,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: workflow-controller-configmap
-  namespace: argo
+  namespace: credreg-staging
 data:
   config: |
     metricsConfig:
@@ -11,12 +11,12 @@ data:
     telemetryConfig:
       enabled: false
       secure: true
-    namespace: argo
+    namespace: credreg-staging
     persistence:
       archive: true
       nodeStatusOffload: true
       postgresql:
-        host: argo-workflows.cwdkv5tua6nq.us-east-1.rds.amazonaws.com
+        host: argo-workflows-staging.cwdkv5tua6nq.us-east-1.rds.amazonaws.com
         port: 5432
         database: argo_workflows
         tableName: argo_workflows
diff --git a/terraform/environments/eks/argo-workflow/externalsecret.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/externalsecret.yaml
similarity index 69%
rename from terraform/environments/eks/argo-workflow/externalsecret.yaml
rename to terraform/environments/eks/k8s-manifests-staging/argo-workflow/externalsecret.yaml
index dd4e1085..5e782e72 100644
--- a/terraform/environments/eks/argo-workflow/externalsecret.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/externalsecret.yaml
@@ -2,7 +2,7 @@ apiVersion: external-secrets.io/v1
 kind: ExternalSecret
 metadata:
   name: argo-postgres
-  namespace: argo
+  namespace: credreg-staging
 spec:
   refreshInterval: 1h
   secretStoreRef:
@@ -14,25 +14,25 @@ spec:
   data:
     - secretKey: host
       remoteRef:
-        key: credreg-argo-workflows
+        key: credreg-argo-workflows-staging
         property: host
     - secretKey: port
       remoteRef:
-        key: credreg-argo-workflows
+        key: credreg-argo-workflows-staging
         property: port
     - secretKey: database
       remoteRef:
-        key: credreg-argo-workflows
+        key: credreg-argo-workflows-staging
         property: database
     - secretKey: username
       remoteRef:
-        key: credreg-argo-workflows
+        key: credreg-argo-workflows-staging
         property: username
     - secretKey: password
       remoteRef:
-        key: credreg-argo-workflows
+        key: credreg-argo-workflows-staging
         property: password
     - secretKey: sslmode
       remoteRef:
-        key: credreg-argo-workflows
+        key: credreg-argo-workflows-staging
         property: sslmode
diff --git a/terraform/environments/eks/argo-workflow/rbac.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/rbac.yaml
similarity index 92%
rename from terraform/environments/eks/argo-workflow/rbac.yaml
rename to terraform/environments/eks/k8s-manifests-staging/argo-workflow/rbac.yaml
index 324cd70c..e7260839 100644
--- a/terraform/environments/eks/argo-workflow/rbac.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/rbac.yaml
@@ -2,7 +2,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: argo-workflow-controller
-  namespace: argo
+  namespace: credreg-staging
   labels:
     app.kubernetes.io/component: controller
     app.kubernetes.io/part-of: argo-workflows
@@ -11,7 +11,7 @@ apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: argo-server
-  namespace: argo
+  namespace: credreg-staging
   labels:
     app.kubernetes.io/component: server
     app.kubernetes.io/part-of: argo-workflows
@@ -52,13 +52,13 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: argo-workflow-controller
-    namespace: argo
+    namespace: credreg-staging
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: argo-server
-  namespace: argo
+  namespace: credreg-staging
 rules:
   - apiGroups: ["argoproj.io"]
     resources: ["workflows", "workflowtemplates", "cronworkflows"]
@@ -74,11 +74,11 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: argo-server
-  namespace: argo
+  namespace: credreg-staging
 subjects:
   - kind: ServiceAccount
     name: argo-server
-    namespace: argo
+    namespace: credreg-staging
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/wf.json b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/wf.json
new file mode 100644
index 00000000..b3605599
--- /dev/null
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/wf.json
@@ -0,0 +1,20 @@
+{
+       "workflow": {
+         "apiVersion": "argoproj.io/v1alpha1",
+         "kind": "Workflow",
+         "metadata": { "generateName": "rest-test-" },
+         "spec": {
+           "entrypoint": "hello",
+           "templates": [
+             {
+               "name": "hello",
+               "container": {
+                 "image": "docker/whalesay:latest",
+                 "command": ["cowsay"],
+                 "args": ["hello from REST"]
+               }
+             }
+           ]
+         }
+       }
+     }
diff --git a/terraform/environments/eks/argo-workflow/workflow-controller-deployment.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/workflow-controller-deployment.yaml
similarity index 98%
rename from terraform/environments/eks/argo-workflow/workflow-controller-deployment.yaml
rename to terraform/environments/eks/k8s-manifests-staging/argo-workflow/workflow-controller-deployment.yaml
index 2a135176..127fb2e3 100644
--- a/terraform/environments/eks/argo-workflow/workflow-controller-deployment.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/workflow-controller-deployment.yaml
@@ -2,7 +2,7 @@ apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: workflow-controller
-  namespace: argo
+  namespace: credreg-staging
   labels:
     app.kubernetes.io/name: workflow-controller
     app.kubernetes.io/part-of: argo-workflows

From a642f1c034e0e0aa996b38eca8b0c424991d9358 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Tue, 13 Jan 2026 19:29:31 -0300
Subject: [PATCH 05/16] add documentation for triggering a dummy workflow using
 rest call

---
 .../argo-workflow/TRIGGER-DUMMY-WORKFLOW.md   | 62 +++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md

diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md
new file mode 100644
index 00000000..972b36ae
--- /dev/null
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md
@@ -0,0 +1,62 @@
+# Triggering Argo Workflows via port-forward + curl
+
+Use this guide when you need to submit a workflow from your workstation without going through the ingress (no basic auth). The flow is:
+
+1. **Port-forward the Argo server service**
+   ```bash
+   kubectl port-forward -n credreg-staging svc/argo-server 2746:2746
+   ```
+   Leave this running in a separate terminal; it exposes `https://localhost:2746`.
+
+2. **Mint a service-account token**
+   ```bash
+   BEARER=$(kubectl create token argo-server -n credreg-staging)
+   ```
+   Any SA with workflow submit/list permissions works (`argo-server` or `argo-workflow-controller`).
+
+3. **Create the workflow payload**
+   ```bash
+   cat > wf.json <<'EOF'
+   {
+     "workflow": {
+       "apiVersion": "argoproj.io/v1alpha1",
+       "kind": "Workflow",
+       "metadata": { "generateName": "rest-test-" },
+       "spec": {
+         "entrypoint": "hello",
+         "templates": [
+           {
+             "name": "hello",
+             "container": {
+               "image": "docker/whalesay:latest",
+               "command": ["cowsay"],
+               "args": ["hello from REST"]
+             }
+           }
+         ]
+       }
+     }
+   }
+   EOF
+   ```
+
+4. **Submit the workflow**
+   ```bash
+   curl -sk https://localhost:2746/api/v1/workflows/credreg-staging \
+     -H "Authorization: Bearer $BEARER" \
+     -H 'Content-Type: application/json' \
+     -d @wf.json
+   ```
+   A successful response echoes the workflow metadata (UID, status, etc.).
+
+5. **Verify status**
+   ```bash
+   kubectl get wf -n credreg-staging
+   kubectl logs -n credreg-staging wf/<workflow-name>
+   ```
+
+6. **Clean up**
+   - `kubectl delete wf <workflow-name> -n credreg-staging` (optional)
+   - Stop the `kubectl port-forward` process.
+
+> Tip: For ad-hoc tests, this approach avoids ingress auth entirely. When you’re ready to call the public endpoint, add the ingress basic-auth header and keep using the Bearer token in parallel.

From 59e323cba8fc133e3a4d7a351d7d40510fd861be Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Tue, 13 Jan 2026 19:31:05 -0300
Subject: [PATCH 06/16] adding Postman instructions to triggering a dummy argo
 workflow

---
 .../argo-workflow/TRIGGER-DUMMY-WORKFLOW.md      | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md
index 972b36ae..4b8c639b 100644
--- a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md
@@ -40,7 +40,7 @@ Use this guide when you need to submit a workflow from your workstation without
    EOF
    ```
 
-4. **Submit the workflow**
+4. **Submit the workflow (cURL)**
    ```bash
    curl -sk https://localhost:2746/api/v1/workflows/credreg-staging \
      -H "Authorization: Bearer $BEARER" \
@@ -49,6 +49,20 @@ Use this guide when you need to submit a workflow from your workstation without
    ```
    A successful response echoes the workflow metadata (UID, status, etc.).
 
+## Trigger via Postman
+
+1. Keep the port-forward running: `kubectl port-forward -n credreg-staging svc/argo-server 2746:2746`.
+2. Generate a Bearer token: `kubectl create token argo-server -n credreg-staging` (copy the value).
+3. In Postman:
+   - **Method:** `POST`
+   - **URL:** `https://localhost:2746/api/v1/workflows/credreg-staging`
+   - **Headers:**
+     - `Authorization: Bearer <token>`
+     - `Content-Type: application/json`
+   - **Body:** raw JSON from `wf.json` (same payload as above).
+4. Disable SSL verification in Postman (Settings → General → “SSL certificate verification” off) or import the Argo server cert so the self-signed TLS passes.
+5. Send the request; you should see the workflow metadata returned. Use the same token for subsequent requests until it expires.
+
 5. **Verify status**
    ```bash
    kubectl get wf -n credreg-staging

From 9879c6039528b76f30ed02384b510c6e1d8f06f2 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Tue, 13 Jan 2026 20:11:39 -0300
Subject: [PATCH 07/16] fix wf example

---
 .../argo-workflow/TRIGGER-DUMMY-WORKFLOW.md   | 23 ++++++----
 .../argo-workflow/wf.json                     | 46 +++++++++++--------
 2 files changed, 40 insertions(+), 29 deletions(-)

diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md
index 4b8c639b..c2f32a6f 100644
--- a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/TRIGGER-DUMMY-WORKFLOW.md
@@ -23,18 +23,21 @@ Use this guide when you need to submit a workflow from your workstation without
        "kind": "Workflow",
        "metadata": { "generateName": "rest-test-" },
        "spec": {
-         "entrypoint": "hello",
+      "serviceAccountName": "argo-workflow-controller",
+      "entrypoint": "hello",
          "templates": [
            {
-             "name": "hello",
-             "container": {
-               "image": "docker/whalesay:latest",
-               "command": ["cowsay"],
-               "args": ["hello from REST"]
-             }
-           }
-         ]
-       }
+          "name": "hello",
+          "container": {
+            "image": "public.ecr.aws/docker/library/debian:stable-slim",
+            "command": ["bash", "-c"],
+            "args": [
+              "apt-get update >/dev/null && DEBIAN_FRONTEND=noninteractive apt-get install -y cowsay >/dev/null && /usr/games/cowsay \"hello from REST\""
+            ]
+          }
+        }
+      ]
+    }
      }
    }
    EOF
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/wf.json b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/wf.json
index b3605599..11fc609d 100644
--- a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/wf.json
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/wf.json
@@ -1,20 +1,28 @@
 {
-       "workflow": {
-         "apiVersion": "argoproj.io/v1alpha1",
-         "kind": "Workflow",
-         "metadata": { "generateName": "rest-test-" },
-         "spec": {
-           "entrypoint": "hello",
-           "templates": [
-             {
-               "name": "hello",
-               "container": {
-                 "image": "docker/whalesay:latest",
-                 "command": ["cowsay"],
-                 "args": ["hello from REST"]
-               }
-             }
-           ]
-         }
-       }
-     }
+  "workflow": {
+    "apiVersion": "argoproj.io/v1alpha1",
+    "kind": "Workflow",
+    "metadata": {
+      "generateName": "rest-test-"
+    },
+    "spec": {
+      "serviceAccountName": "argo-workflow-controller",
+      "entrypoint": "hello",
+      "templates": [
+        {
+          "name": "hello",
+          "container": {
+            "image": "public.ecr.aws/docker/library/debian:stable-slim",
+            "command": [
+              "bash",
+              "-c"
+            ],
+            "args": [
+              "apt-get update >/dev/null && DEBIAN_FRONTEND=noninteractive apt-get install -y cowsay >/dev/null && /usr/games/cowsay \"hello from REST\""
+            ]
+          }
+        }
+      ]
+    }
+  }
+}

From 0172f8222f4f7b01399cff448fc50da7cf3fb024 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Fri, 23 Jan 2026 12:08:03 -0300
Subject: [PATCH 08/16] Add S3 to Elasticsearch indexing workflow

- Add API endpoint POST /workflows/index-all-s3-to-es
- Add IndexS3GraphToEs service to index S3 objects to ES
- Add WorkflowPolicy for admin authorization
- Add Argo WorkflowTemplate with Keycloak auth
- Add rake task s3:index_all_to_es for manual runs
- Update README with workflow documentation
---
 app/api/v1/base.rb                            |   3 +
 app/api/v1/workflows.rb                       |  61 ++++++++++
 app/policies/workflow_policy.rb               |  12 ++
 app/services/index_s3_graph_to_es.rb          |  85 +++++++++++++
 lib/tasks/s3.rake                             |  77 ++++++++++++
 .../argo-workflow/README.md                   | 112 +++++++++++++++++-
 .../argo-keycloak-credentials-secret.yaml     |  25 ++++
 .../index-s3-to-es-workflow-template.yaml     | 109 +++++++++++++++++
 8 files changed, 482 insertions(+), 2 deletions(-)
 create mode 100644 app/api/v1/workflows.rb
 create mode 100644 app/policies/workflow_policy.rb
 create mode 100644 app/services/index_s3_graph_to_es.rb
 create mode 100644 lib/tasks/s3.rake
 create mode 100644 terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-keycloak-credentials-secret.yaml
 create mode 100644 terraform/environments/eks/k8s-manifests-staging/argo-workflow/index-s3-to-es-workflow-template.yaml

diff --git a/app/api/v1/base.rb b/app/api/v1/base.rb
index a7146caa..b8821fc4 100644
--- a/app/api/v1/base.rb
+++ b/app/api/v1/base.rb
@@ -17,6 +17,7 @@
 require 'v1/indexed_resources'
 require 'v1/indexer'
 require 'v1/envelope_communities'
+require 'v1/workflows'
 
 module API
   module V1
@@ -64,6 +65,8 @@ class Base < Grape::API
         mount API::V1::Organizations
         mount API::V1::Publishers
       end
+
+      mount API::V1::Workflows
     end
   end
 end
diff --git a/app/api/v1/workflows.rb b/app/api/v1/workflows.rb
new file mode 100644
index 00000000..cbafc9f0
--- /dev/null
+++ b/app/api/v1/workflows.rb
@@ -0,0 +1,61 @@
+require 'mountable_api'
+require 'helpers/shared_helpers'
+
+module API
+  module V1
+    # Endpoints for operational workflows (called by Argo Workflows)
+    class Workflows < MountableAPI
+      mounted do
+        helpers SharedHelpers
+
+        before do
+          authenticate!
+        end
+
+        resource :workflows do
+          desc 'Indexes all S3 JSON-LD graphs to Elasticsearch. ' \
+               'S3 is treated as the source of truth. ' \
+               'Called by Argo Workflows for orchestration.'
+          post 'index-all-s3-to-es' do
+            authorize :workflow, :trigger?
+
+            bucket_name = ENV['ENVELOPE_GRAPHS_BUCKET']
+            error!({ error: 'ENVELOPE_GRAPHS_BUCKET not configured' }, 500) unless bucket_name
+
+            es_address = ENV['ELASTICSEARCH_ADDRESS']
+            error!({ error: 'ELASTICSEARCH_ADDRESS not configured' }, 500) unless es_address
+
+            s3 = Aws::S3::Resource.new(region: ENV['AWS_REGION'].presence)
+            bucket = s3.bucket(bucket_name)
+
+            errors = {}
+            processed = 0
+            skipped = 0
+
+            bucket.objects.each do |object|
+              next unless object.key.end_with?('.json')
+
+              processed += 1
+
+              begin
+                IndexS3GraphToEs.call(object.key)
+              rescue StandardError => e
+                errors[object.key] = "#{e.class}: #{e.message}"
+              end
+            end
+
+            status_code = errors.empty? ? 200 : 207
+
+            status status_code
+            {
+              message: errors.empty? ? 'Indexing completed successfully' : 'Indexing completed with errors',
+              processed: processed,
+              errors_count: errors.size,
+              errors: errors.first(100).to_h
+            }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/app/policies/workflow_policy.rb b/app/policies/workflow_policy.rb
new file mode 100644
index 00000000..380810d6
--- /dev/null
+++ b/app/policies/workflow_policy.rb
@@ -0,0 +1,12 @@
+require_relative 'application_policy'
+
+# Specifies policies for workflow operations
+class WorkflowPolicy < ApplicationPolicy
+  def trigger?
+    user.admin?
+  end
+
+  def show?
+    user.admin?
+  end
+end
diff --git a/app/services/index_s3_graph_to_es.rb b/app/services/index_s3_graph_to_es.rb
new file mode 100644
index 00000000..0dd0f912
--- /dev/null
+++ b/app/services/index_s3_graph_to_es.rb
@@ -0,0 +1,85 @@
+# Indexes a JSON-LD graph from S3 directly to Elasticsearch
+# Does not require database access - S3 is the source of truth
+class IndexS3GraphToEs
+  attr_reader :s3_key, :community_name, :ctid
+
+  def initialize(s3_key)
+    @s3_key = s3_key
+    parse_s3_key
+  end
+
+  class << self
+    def call(s3_key)
+      new(s3_key).call
+    end
+  end
+
+  def call
+    return unless elasticsearch_address
+
+    client.index(
+      body: graph_json,
+      id: ctid,
+      index: community_name
+    )
+  rescue Elastic::Transport::Transport::Errors::BadRequest => e
+    raise e unless e.message.include?('Limit of total fields')
+
+    increase_total_fields_limit
+    retry
+  end
+
+  private
+
+  def parse_s3_key
+    # S3 key format: {community_name}/{ctid}.json
+    parts = s3_key.split('/')
+    @community_name = parts[0..-2].join('/')
+    @ctid = parts.last.sub(/\.json\z/i, '')
+  end
+
+  def graph_content
+    @graph_content ||= s3_object.get.body.read
+  end
+
+  def graph_json
+    @graph_json ||= JSON.parse(graph_content).to_json
+  end
+
+  def client
+    @client ||= Elasticsearch::Client.new(host: elasticsearch_address)
+  end
+
+  def elasticsearch_address
+    ENV['ELASTICSEARCH_ADDRESS'].presence
+  end
+
+  def s3_bucket
+    @s3_bucket ||= s3_resource.bucket(s3_bucket_name)
+  end
+
+  def s3_bucket_name
+    ENV['ENVELOPE_GRAPHS_BUCKET'].presence
+  end
+
+  def s3_object
+    @s3_object ||= s3_bucket.object(s3_key)
+  end
+
+  def s3_resource
+    @s3_resource ||= Aws::S3::Resource.new(region: ENV['AWS_REGION'].presence)
+  end
+
+  def increase_total_fields_limit
+    settings = client.indices.get_settings(index: community_name)
+
+    current_limit = settings
+                    .dig(community_name, 'settings', 'index', 'mapping', 'total_fields', 'limit')
+                    .to_i
+
+    client.indices.put_settings(
+      body: { 'index.mapping.total_fields.limit' => current_limit * 2 },
+      index: community_name
+    )
+  end
+end
diff --git a/lib/tasks/s3.rake b/lib/tasks/s3.rake
new file mode 100644
index 00000000..6ae3f5c5
--- /dev/null
+++ b/lib/tasks/s3.rake
@@ -0,0 +1,77 @@
+namespace :s3 do
+  desc 'Index all S3 JSON-LD graphs to Elasticsearch (S3 as source of truth)'
+  task index_all_to_es: :environment do
+    require 'benchmark'
+    require 'json'
+
+    bucket_name = ENV['ENVELOPE_GRAPHS_BUCKET']
+    abort 'ENVELOPE_GRAPHS_BUCKET environment variable is not set' unless bucket_name
+
+    es_address = ENV['ELASTICSEARCH_ADDRESS']
+    abort 'ELASTICSEARCH_ADDRESS environment variable is not set' unless es_address
+
+    $stdout.sync = true
+
+    s3 = Aws::S3::Resource.new(region: ENV['AWS_REGION'].presence)
+    bucket = s3.bucket(bucket_name)
+
+    errors = {}
+    processed = 0
+    skipped = 0
+
+    puts "Starting S3 to ES indexing from bucket: #{bucket_name}"
+    puts "Elasticsearch address: #{es_address}"
+    puts "Counting objects..."
+
+    # Count total objects for progress reporting
+    total = bucket.objects.count { |obj| obj.key.end_with?('.json') }
+    puts "Found #{total} JSON files to index"
+    puts "Started at #{Time.now.utc}"
+
+    time = Benchmark.measure do
+      bucket.objects.each do |object|
+        next unless object.key.end_with?('.json')
+
+        processed += 1
+
+        begin
+          IndexS3GraphToEs.call(object.key)
+        rescue StandardError => e
+          errors[object.key] = "#{e.class}: #{e.message}"
+        end
+
+        # Progress every 100 records
+        if (processed % 100).zero?
+          puts "Progress: processed=#{processed}/#{total} errors=#{errors.size} skipped=#{skipped}"
+        end
+      end
+    end
+
+    puts time
+    puts "Finished at #{Time.now.utc} - processed=#{processed}, errors=#{errors.size}"
+
+    # Write errors to file
+    if errors.any?
+      File.write('/tmp/s3_index_errors.json', JSON.pretty_generate(errors))
+      puts "Wrote /tmp/s3_index_errors.json (#{errors.size} entries)"
+
+      # Upload errors to S3
+      begin
+        error_bucket = ENV['S3_ERRORS_BUCKET'] || bucket_name
+        error_key = "errors/s3-index-errors-#{Time.now.utc.strftime('%Y%m%dT%H%M%SZ')}.json"
+        s3_client = Aws::S3::Client.new(region: ENV['AWS_REGION'].presence)
+        s3_client.put_object(
+          bucket: error_bucket,
+          key: error_key,
+          body: File.open('/tmp/s3_index_errors.json', 'rb')
+        )
+        puts "Uploaded errors to s3://#{error_bucket}/#{error_key}"
+      rescue StandardError => e
+        warn "Failed to upload errors to S3: #{e.class}: #{e.message}"
+      end
+
+      warn "Encountered #{errors.size} errors. Sample: #{errors.to_a.first(5).to_h.inspect}"
+      exit 1
+    end
+  end
+end
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/README.md b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/README.md
index 7549e71c..a90e0549 100644
--- a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/README.md
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/README.md
@@ -9,14 +9,14 @@ These manifests install a minimal Argo Workflows control plane into the shared `
 - `workflow-controller-deployment.yaml` – runs `workflow-controller` with the standard `argoexec` image.
 - `argo-server.yaml` – exposes the Argo UI/API inside the cluster on port `2746`.
 - `argo-basic-auth-externalsecret.yaml` – syncs the AWS Secrets Manager entry `credreg-argo-basic-auth` (or similar) to supply the base64-encoded `user:password` string for ingress auth.
-- `argo-server-ingress.yaml` – optional HTTPS ingress + certificate (via cert-manager + Let’s Encrypt) and basic auth for external access to the Argo UI.
+- `argo-server-ingress.yaml` – optional HTTPS ingress + certificate (via cert-manager + Let's Encrypt) and basic auth for external access to the Argo UI.
 
 ## Before applying
 1. **Provision or reference a PostgreSQL instance.** Ensure the desired environment has a reachable database endpoint.
 2. **Create the Secrets Manager entry.** Create `credreg-argo-workflows` (or adjust the `remoteRef.key` value) with JSON keys `host`, `port`, `database`, `username`, `password`, `sslmode`. The External Secrets Operator will sync it into the cluster and the controller/server pick them up via env vars.
 3. **Update `configmap.yaml`.** Set `persistence.postgresql.host` (and database/table names if they differ) for the target environment. Even though credentials are secret-backed, Argo still requires the host in this config.
 4. **Install Argo CRDs.** Apply the upstream CRDs from https://github.com/argoproj/argo-workflows/releases (required only once per cluster) before rolling out these manifests.
-5. **Configure DNS if using the ingress.** Update `argo-server-ingress.yaml` with the desired hostname(s) and point the DNS record at the ingress controller’s load balancer.
+5. **Configure DNS if using the ingress.** Update `argo-server-ingress.yaml` with the desired hostname(s) and point the DNS record at the ingress controller's load balancer.
 
 ## Apply order
 ```bash
@@ -31,3 +31,111 @@ kubectl apply -f terraform/environments/eks/k8s-manifests-staging/argo-workflow/
 ```
 
 Once the `argo-postgres` secret is synced and the controller connects to Postgres successfully, `kubectl get wf -n credreg-staging` should show persisted workflows even after pod restarts.
+
+## Workflow Templates
+
+### index-s3-to-es
+
+Indexes all JSON-LD graphs from S3 directly to Elasticsearch. S3 is treated as the source of truth.
+
+**Architecture:**
+```
+Argo Workflow (curl container)
+    │
+    ├──1. POST to Keycloak /token (client credentials grant)
+    │      → Obtain fresh JWT
+    │
+    └──2. POST /workflows/index-all-s3-to-es
+           │
+           ▼
+      Registry API
+           │
+           ├──▶ List S3 bucket objects
+           │
+           └──▶ For each .json file:
+                   └──▶ Index to Elasticsearch
+```
+
+**Prerequisites - Keycloak Service Account:**
+
+1. Create a Keycloak client in the `CE-Test` realm:
+   - **Client ID**: e.g., `argo-workflows`
+   - **Client authentication**: ON (confidential client)
+   - **Service accounts roles**: ON
+   - **Authentication flow**: Only "Service accounts roles" enabled
+
+2. Assign the admin role to the service account:
+   - Go to the client → Service Account Roles
+   - Assign `ROLE_ADMINISTRATOR` from the `RegistryAPI` client
+
+3. Get the client secret:
+   - Go to the client → Credentials
+   - Update the Client Secret
+
+
+**Required configuration:**
+
+1. **Keycloak Credentials Secret** (`argo-keycloak-credentials`):
+   - `client_id` – Keycloak client ID
+   - `client_secret` – Keycloak client secret
+
+2. **Registry API environment variables** (already in app-configmap):
+   - `ENVELOPE_GRAPHS_BUCKET` – S3 bucket containing JSON-LD graphs
+   - `ELASTICSEARCH_ADDRESS` – Elasticsearch endpoint
+   - `AWS_REGION` – AWS region for S3 access
+
+**Trigger the workflow:**
+
+Via Argo CLI:
+```bash
+argo submit --from workflowtemplate/index-s3-to-es -n credreg-staging
+```
+
+Via Argo REST API:
+```bash
+kubectl port-forward -n credreg-staging svc/argo-server 2746:2746
+BEARER=$(kubectl create token argo-server -n credreg-staging)
+
+curl -sk https://localhost:2746/api/v1/workflows/credreg-staging \
+  -H "Authorization: Bearer $BEARER" \
+  -H 'Content-Type: application/json' \
+  -d '{
+    "workflow": {
+      "metadata": { "generateName": "index-s3-to-es-" },
+      "spec": { "workflowTemplateRef": { "name": "index-s3-to-es" } }
+    }
+  }'
+```
+
+Via Argo UI:
+1. Navigate to the Argo UI
+2. Go to Workflow Templates
+3. Select `index-s3-to-es`
+4. Click "Submit"
+
+**Monitor workflow:**
+```bash
+# List workflows
+kubectl get wf -n credreg-staging
+
+# Watch workflow status
+argo watch -n credreg-staging <workflow-name>
+
+# View logs
+argo logs -n credreg-staging <workflow-name>
+```
+
+**Workflow parameters:**
+
+| Parameter | Default | Description |
+|-----------|---------|-------------|
+| `api-base-url` | `http://main-app.credreg-staging.svc.cluster.local:9292` | Registry API base URL |
+| `keycloak-url` | `https://test-ce-kc-002.credentialengine.org/realms/CE-Test/protocol/openid-connect/token` | Keycloak token endpoint |
+
+Override parameters when submitting:
+```bash
+argo submit --from workflowtemplate/index-s3-to-es \
+  -p api-base-url=http://custom-api:9292 \
+  -p keycloak-url=https://other-keycloak/realms/X/protocol/openid-connect/token \
+  -n credreg-staging
+```
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-keycloak-credentials-secret.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-keycloak-credentials-secret.yaml
new file mode 100644
index 00000000..3933afb5
--- /dev/null
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-keycloak-credentials-secret.yaml
@@ -0,0 +1,25 @@
+# Keycloak service account credentials for Argo workflows
+# Used to obtain JWT tokens via Client Credentials Grant
+#
+# Prerequisites:
+# 1. Create a Keycloak client with:
+#    - Client authentication: ON (confidential)
+#    - Service accounts roles: ON
+#    - Assign ROLE_ADMINISTRATOR to the service account
+#
+# 2. Update the values below with your client credentials
+# 3. Apply: kubectl apply -f argo-keycloak-credentials-secret.yaml
+#
+# Alternatively, use External Secrets Operator to sync from AWS Secrets Manager
+apiVersion: v1
+kind: Secret
+metadata:
+  name: argo-keycloak-credentials
+  namespace: credreg-staging
+  labels:
+    app: credential-registry
+    component: argo-workflow
+type: Opaque
+stringData:
+  client_id: "[KEYCLOAK_CLIENT_ID]"
+  client_secret: "[KEYCLOAK_CLIENT_SECRET]"
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/index-s3-to-es-workflow-template.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/index-s3-to-es-workflow-template.yaml
new file mode 100644
index 00000000..539f4af2
--- /dev/null
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/index-s3-to-es-workflow-template.yaml
@@ -0,0 +1,109 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: index-s3-to-es
+  namespace: credreg-staging
+  labels:
+    app: credential-registry
+spec:
+  serviceAccountName: argo-workflow-controller
+  entrypoint: index-s3-to-es
+  arguments:
+    parameters:
+      - name: api-base-url
+        value: "http://main-app.credreg-staging.svc.cluster.local:9292"
+      - name: keycloak-url
+        value: "https://test-ce-kc-002.credentialengine.org/realms/CE-Test/protocol/openid-connect/token"
+  templates:
+    - name: index-s3-to-es
+      metadata:
+        labels:
+          app: credential-registry
+          workflow: index-s3-to-es
+      inputs:
+        parameters:
+          - name: api-base-url
+          - name: keycloak-url
+      container:
+        image: curlimages/curl:latest
+        command:
+          - /bin/sh
+          - -c
+          - |
+            set -e
+            echo "=== S3 to Elasticsearch Indexing Workflow ==="
+            echo "Started at: $(date -u)"
+            echo ""
+
+            # Step 1: Get access token from Keycloak
+            echo "Step 1: Authenticating with Keycloak..."
+            TOKEN_RESPONSE=$(curl -sf -X POST "{{inputs.parameters.keycloak-url}}" \
+              -H "Content-Type: application/x-www-form-urlencoded" \
+              -d "grant_type=client_credentials" \
+              -d "client_id=${KEYCLOAK_CLIENT_ID}" \
+              -d "client_secret=${KEYCLOAK_CLIENT_SECRET}")
+
+            ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | sed -n 's/.*"access_token":"\([^"]*\)".*/\1/p')
+
+            if [ -z "$ACCESS_TOKEN" ]; then
+              echo "ERROR: Failed to obtain access token from Keycloak"
+              echo "Response: $TOKEN_RESPONSE"
+              exit 1
+            fi
+            echo "Successfully obtained access token"
+            echo ""
+
+            # Step 2: Call Registry API
+            echo "Step 2: Calling Registry API at {{inputs.parameters.api-base-url}}"
+            response=$(curl -sf -X POST \
+              "{{inputs.parameters.api-base-url}}/workflows/index-all-s3-to-es" \
+              -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+              -H "Content-Type: application/json" \
+              -w "\n%{http_code}" \
+              --max-time 43200)
+
+            # Extract HTTP status code (last line)
+            http_code=$(echo "$response" | tail -n1)
+            body=$(echo "$response" | sed '$d')
+
+            echo ""
+            echo "=== API Response ==="
+            echo "HTTP Status: $http_code"
+            echo "Body:"
+            echo "$body"
+            echo ""
+
+            # Exit with error if not success (200 or 207)
+            if [ "$http_code" != "200" ] && [ "$http_code" != "207" ]; then
+              echo "ERROR: API call failed with status $http_code"
+              exit 1
+            fi
+
+            echo "=== Workflow Completed Successfully ==="
+            echo "Finished at: $(date -u)"
+        env:
+          - name: KEYCLOAK_CLIENT_ID
+            valueFrom:
+              secretKeyRef:
+                name: argo-keycloak-credentials
+                key: client_id
+          - name: KEYCLOAK_CLIENT_SECRET
+            valueFrom:
+              secretKeyRef:
+                name: argo-keycloak-credentials
+                key: client_secret
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "128Mi"
+          limits:
+            cpu: "200m"
+            memory: "256Mi"
+      retryStrategy:
+        limit: 2
+        retryPolicy: OnFailure
+        backoff:
+          duration: "60s"
+          factor: 2
+          maxDuration: "10m"
+      activeDeadlineSeconds: 43200

From d7f7a0175a56796f1db18a88b2790f3ebfbe2de4 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Fri, 6 Feb 2026 11:19:14 -0300
Subject: [PATCH 09/16] add config files

---
 .../elasticsearch-statefulset.yaml            |   6 +-
 .../jobs/index-s3-to-es-direct.yaml           | 205 ++++++++++++++++++
 .../elasticsearch-statefulset.yaml            |   6 +-
 3 files changed, 211 insertions(+), 6 deletions(-)
 create mode 100644 terraform/environments/eks/k8s-manifests-staging/argo-workflow/jobs/index-s3-to-es-direct.yaml

diff --git a/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-statefulset.yaml b/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-statefulset.yaml
index 71203faf..dfebac0c 100644
--- a/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-statefulset.yaml
+++ b/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-statefulset.yaml
@@ -7,7 +7,7 @@ metadata:
     app: elasticsearch
 spec:
   serviceName: elasticsearch-discovery
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: elasticsearch
@@ -51,8 +51,8 @@ spec:
               value: "false"
             - name: network.host
               value: "0.0.0.0"
-            - name: discovery.seed_hosts
-              value: "elasticsearch-discovery"
+            - name: discovery.type
+              value: "single-node"
           volumeMounts:
             - name: elasticsearch-data
               mountPath: /usr/share/elasticsearch/data
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/jobs/index-s3-to-es-direct.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/jobs/index-s3-to-es-direct.yaml
new file mode 100644
index 00000000..04f9902f
--- /dev/null
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/jobs/index-s3-to-es-direct.yaml
@@ -0,0 +1,205 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: index-s3-to-es-direct
+  namespace: credreg-staging
+  labels:
+    app: credential-registry
+spec:
+  serviceAccountName: main-app-service-account
+  entrypoint: index-s3-to-es
+  arguments:
+    parameters:
+      - name: s3-bucket
+        value: "cer-envelope-graphs-staging"
+      - name: es-endpoint
+        value: "http://elasticsearch.credreg-staging.svc.cluster.local:9200"
+      - name: batch-size
+        value: "500"
+  templates:
+    - name: index-s3-to-es
+      metadata:
+        labels:
+          app: credential-registry
+          workflow: index-s3-to-es-direct
+      inputs:
+        parameters:
+          - name: s3-bucket
+          - name: es-endpoint
+          - name: batch-size
+      container:
+        image: public.ecr.aws/aws-cli/aws-cli:latest
+        command:
+          - /bin/bash
+          - -c
+          - |
+            set -e
+
+            echo "=== S3 to Elasticsearch Direct Indexing ==="
+            echo "Started at: $(date -u)"
+            echo ""
+
+            # Install curl
+            echo "Installing curl..."
+            yum install -y curl --quiet
+
+            S3_BUCKET="{{inputs.parameters.s3-bucket}}"
+            ES_ENDPOINT="{{inputs.parameters.es-endpoint}}"
+            BATCH_SIZE="{{inputs.parameters.batch-size}}"
+
+            # Generate index name with ce- prefix and ISO8601 UTC timestamp
+            INDEX_NAME="ce-$(date -u +%Y-%m-%dT%H:%M:%SZ)"
+            echo "Target index: ${INDEX_NAME}"
+            echo "S3 bucket: ${S3_BUCKET}"
+            echo "ES endpoint: ${ES_ENDPOINT}"
+            echo "Batch size: ${BATCH_SIZE}"
+            echo ""
+
+            # Delete index if exists
+            echo "Step 1: Checking if index exists..."
+            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "${ES_ENDPOINT}/${INDEX_NAME}")
+            if [ "$HTTP_CODE" = "200" ]; then
+              echo "Index exists, deleting..."
+              curl -s -X DELETE "${ES_ENDPOINT}/${INDEX_NAME}"
+              echo "Index deleted."
+            else
+              echo "Index does not exist."
+            fi
+            echo ""
+
+            # Create new index
+            echo "Step 2: Creating index ${INDEX_NAME}..."
+            curl -s -X PUT "${ES_ENDPOINT}/${INDEX_NAME}" \
+              -H "Content-Type: application/json" \
+              -d '{
+                "settings": {
+                  "number_of_shards": 1,
+                  "number_of_replicas": 0,
+                  "refresh_interval": "-1"
+                }
+              }'
+            echo ""
+            echo "Index created."
+            echo ""
+
+            # List all JSON files from S3
+            echo "Step 3: Listing files from S3..."
+            TMPDIR="/tmp/s3-to-es"
+            mkdir -p "${TMPDIR}"
+
+            aws s3 ls "s3://${S3_BUCKET}/" --recursive | grep '\.json$' | awk '{print $4}' > "${TMPDIR}/files.txt"
+            TOTAL_FILES=$(wc -l < "${TMPDIR}/files.txt")
+            echo "Found ${TOTAL_FILES} JSON files."
+            echo ""
+
+            # Process files in batches using ES _bulk API
+            echo "Step 4: Indexing documents in batches of ${BATCH_SIZE}..."
+            BATCH_NUM=0
+            INDEXED=0
+            ERRORS=0
+
+            while true; do
+              # Get next batch of files
+              OFFSET=$((BATCH_NUM * BATCH_SIZE))
+              BATCH_FILES=$(tail -n +$((OFFSET + 1)) "${TMPDIR}/files.txt" | head -n "${BATCH_SIZE}")
+
+              if [ -z "${BATCH_FILES}" ]; then
+                break
+              fi
+
+              BATCH_NUM=$((BATCH_NUM + 1))
+              BULK_FILE="${TMPDIR}/bulk_${BATCH_NUM}.ndjson"
+              rm -f "${BULK_FILE}"
+
+              # Download files and build bulk request
+              for S3_KEY in ${BATCH_FILES}; do
+                # Extract CTID from filename (without .json extension)
+                FILENAME=$(basename "${S3_KEY}" .json)
+
+                # Download file
+                if aws s3 cp "s3://${S3_BUCKET}/${S3_KEY}" "${TMPDIR}/doc.json" --quiet 2>/dev/null; then
+                  # Append action line and document to bulk file
+                  echo "{\"index\":{\"_index\":\"${INDEX_NAME}\",\"_id\":\"${FILENAME}\"}}" >> "${BULK_FILE}"
+                  # Document must be on single line for bulk API
+                  cat "${TMPDIR}/doc.json" | tr -d '\n' >> "${BULK_FILE}"
+                  echo "" >> "${BULK_FILE}"
+                else
+                  echo "Warning: Failed to download ${S3_KEY}"
+                  ERRORS=$((ERRORS + 1))
+                fi
+              done
+
+              # Send bulk request
+              if [ -f "${BULK_FILE}" ] && [ -s "${BULK_FILE}" ]; then
+                RESPONSE=$(curl -s -X POST "${ES_ENDPOINT}/_bulk" \
+                  -H "Content-Type: application/x-ndjson" \
+                  --data-binary @"${BULK_FILE}")
+
+                # Check for errors in response
+                HAS_ERRORS=$(echo "${RESPONSE}" | grep -o '"errors":\s*true' || true)
+                if [ -n "${HAS_ERRORS}" ]; then
+                  echo "Warning: Batch ${BATCH_NUM} had some errors"
+                  ERRORS=$((ERRORS + 1))
+                fi
+
+                BATCH_COUNT=$(echo "${BATCH_FILES}" | wc -w)
+                INDEXED=$((INDEXED + BATCH_COUNT))
+                echo "Batch ${BATCH_NUM}: indexed ${BATCH_COUNT} documents (total: ${INDEXED}/${TOTAL_FILES})"
+              fi
+
+              # Cleanup batch file
+              rm -f "${BULK_FILE}"
+            done
+
+            echo ""
+
+            # Refresh index to make documents searchable
+            echo "Step 5: Refreshing index..."
+            curl -s -X POST "${ES_ENDPOINT}/${INDEX_NAME}/_refresh"
+            echo ""
+
+            # Update index settings for production
+            echo "Step 6: Updating index settings..."
+            curl -s -X PUT "${ES_ENDPOINT}/${INDEX_NAME}/_settings" \
+              -H "Content-Type: application/json" \
+              -d '{
+                "index": {
+                  "refresh_interval": "1s",
+                  "number_of_replicas": 1
+                }
+              }'
+            echo ""
+
+            # Get final count
+            DOC_COUNT=$(curl -s "${ES_ENDPOINT}/${INDEX_NAME}/_count" | grep -o '"count":[0-9]*' | cut -d: -f2)
+            echo ""
+            echo "=== Indexing Complete ==="
+            echo "Index name: ${INDEX_NAME}"
+            echo "Documents indexed: ${DOC_COUNT}"
+            echo "Errors: ${ERRORS}"
+            echo "Finished at: $(date -u)"
+
+            if [ "${ERRORS}" -gt 0 ]; then
+              echo "Warning: Completed with ${ERRORS} errors"
+              exit 0
+            fi
+        env:
+          - name: AWS_REGION
+            value: us-east-1
+          - name: AWS_DEFAULT_REGION
+            value: us-east-1
+        resources:
+          requests:
+            cpu: "500m"
+            memory: "512Mi"
+          limits:
+            cpu: "1000m"
+            memory: "1Gi"
+      retryStrategy:
+        limit: 2
+        retryPolicy: OnFailure
+        backoff:
+          duration: "60s"
+          factor: 2
+          maxDuration: "10m"
+      activeDeadlineSeconds: 86400  # 24 hours max
diff --git a/terraform/environments/eks/k8s-manifests-staging/elasticsearch-statefulset.yaml b/terraform/environments/eks/k8s-manifests-staging/elasticsearch-statefulset.yaml
index b845c13d..ed9b75ee 100644
--- a/terraform/environments/eks/k8s-manifests-staging/elasticsearch-statefulset.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/elasticsearch-statefulset.yaml
@@ -7,7 +7,7 @@ metadata:
     app: elasticsearch
 spec:
   serviceName: elasticsearch-discovery
-  replicas: 2
+  replicas: 1
   selector:
     matchLabels:
       app: elasticsearch
@@ -51,8 +51,8 @@ spec:
               value: "false"
             - name: network.host
               value: "0.0.0.0"
-            - name: discovery.seed_hosts
-              value: "elasticsearch-discovery"
+            - name: discovery.type
+              value: "single-node"
           volumeMounts:
             - name: elasticsearch-data
               mountPath: /usr/share/elasticsearch/data

From 71f14cd86497a8bbd919c6eb3ad386beb3683b49 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Thu, 19 Feb 2026 18:51:19 -0300
Subject: [PATCH 10/16] implement argo workflows to zip graphs and extract
 resources

---
 app/services/sync_envelope_graph_with_s3.rb   |  30 +++
 .../k8s-manifests-staging/app-deployment.yaml |   5 +
 .../argo-workflow/argo-server-ingress.yaml    |   2 +-
 .../bundle-ce-registry-workflow-template.yaml | 217 ++++++++++++++++++
 .../argo-workflow/externalsecret.yaml         |   4 +
 .../argo-workflow/rbac.yaml                   |  26 +++
 ...ate-graph-resources-workflow-template.yaml | 179 +++++++++++++++
 .../modules/eks/irsa-iam-policy-and-role.tf   |  10 +-
 8 files changed, 470 insertions(+), 3 deletions(-)
 create mode 100644 terraform/environments/eks/k8s-manifests-staging/argo-workflow/bundle-ce-registry-workflow-template.yaml
 create mode 100644 terraform/environments/eks/k8s-manifests-staging/argo-workflow/validate-graph-resources-workflow-template.yaml

diff --git a/app/services/sync_envelope_graph_with_s3.rb b/app/services/sync_envelope_graph_with_s3.rb
index 6a9b39e1..bea23277 100644
--- a/app/services/sync_envelope_graph_with_s3.rb
+++ b/app/services/sync_envelope_graph_with_s3.rb
@@ -27,6 +27,7 @@ def upload
     )
 
     envelope.update_column(:s3_url, s3_object.public_url)
+    trigger_validate_graph_workflow
   end
 
   def remove
@@ -54,4 +55,33 @@ def s3_object
   def s3_resource
     @s3_resource ||= Aws::S3::Resource.new(region: ENV['AWS_REGION'].presence)
   end
+
+  def trigger_validate_graph_workflow
+    argo_token = ENV['ARGO_TOKEN'].presence
+    argo_namespace = ENV['ARGO_NAMESPACE'].presence || 'credreg-staging'
+    dest_bucket = ENV['ARGO_RESOURCE_BUCKET'].presence || 'cer-resources-prod'
+    return unless argo_token
+
+    graph_s3_path = "s3://#{s3_bucket_name}/#{s3_key}"
+    argo_url = "https://argo-server.#{argo_namespace}.svc.cluster.local:2746"
+
+    HTTP.auth("Bearer #{argo_token}")
+        .ssl_context(OpenSSL::SSL::SSLContext.new.tap { |ctx| ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE })
+        .post(
+          "#{argo_url}/api/v1/workflows/#{argo_namespace}/submit",
+          json: {
+            namespace: argo_namespace,
+            resourceKind: 'WorkflowTemplate',
+            resourceName: 'validate-graph-resources',
+            submitOptions: {
+              parameters: [
+                "graph-s3-path=#{graph_s3_path}",
+                "dest-bucket=#{dest_bucket}"
+              ]
+            }
+          }
+        )
+  rescue StandardError => e
+    MR.logger.error("Failed to trigger validate-graph-resources workflow: #{e.message}")
+  end
 end
diff --git a/terraform/environments/eks/k8s-manifests-staging/app-deployment.yaml b/terraform/environments/eks/k8s-manifests-staging/app-deployment.yaml
index 1a048426..f02251ea 100644
--- a/terraform/environments/eks/k8s-manifests-staging/app-deployment.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/app-deployment.yaml
@@ -38,6 +38,11 @@ spec:
           env:
             - name: NEW_RELIC_APP_NAME
               value: "Credential Engine Staging"
+            - name: ARGO_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: argo-postgres
+                  key: argo_token
           ports:
             - containerPort: 9292
           envFrom:
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server-ingress.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server-ingress.yaml
index caa32520..895c8e6f 100644
--- a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server-ingress.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/argo-server-ingress.yaml
@@ -26,7 +26,7 @@ metadata:
     nginx.ingress.kubernetes.io/proxy-body-size: "10m"
     nginx.ingress.kubernetes.io/proxy-read-timeout: "300"
     nginx.ingress.kubernetes.io/proxy-send-timeout: "300"
-    nginx.ingress.kubernetes.io/whitelist-source-range: "71.212.64.155/32,129.224.215.205/32"
+    nginx.ingress.kubernetes.io/whitelist-source-range: "71.212.64.155/32,129.224.215.205/32,148.222.194.113/32"
 spec:
   ingressClassName: nginx
   tls:
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/bundle-ce-registry-workflow-template.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/bundle-ce-registry-workflow-template.yaml
new file mode 100644
index 00000000..49dee0ef
--- /dev/null
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/bundle-ce-registry-workflow-template.yaml
@@ -0,0 +1,217 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: bundle-ce-registry-to-zip
+  namespace: credreg-staging
+  labels:
+    app: credential-registry
+spec:
+  serviceAccountName: main-app-service-account
+  entrypoint: bundle-ce-registry-to-zip
+  arguments:
+    parameters:
+      - name: dest-bucket
+        value: "cer-envelope-downloads"
+      - name: slack-webhook
+        value: ""
+  templates:
+    - name: bundle-ce-registry-to-zip
+      inputs:
+        parameters:
+          - name: dest-bucket
+          - name: slack-webhook
+      metadata:
+        labels:
+          app: credential-registry
+          workflow: bundle-ce-registry-to-zip
+      container:
+        image: python:3.11-slim
+        command:
+          - /bin/sh
+          - -c
+          - |
+            set -e
+            echo "=== Bundle CE Registry JSON files to ZIP ==="
+            echo "Started at: $(date -u)"
+            echo ""
+
+            echo "[$(date -u +%H:%M:%S)] Upgrading pip..."
+            pip install --quiet --upgrade pip --root-user-action=ignore
+            echo "[$(date -u +%H:%M:%S)] Installing boto3..."
+            pip install --quiet boto3 --root-user-action=ignore
+            echo "[$(date -u +%H:%M:%S)] Dependencies ready."
+            echo ""
+
+            python3 - << 'PYEOF'
+            import boto3
+            import zipfile
+            import zlib
+            import os
+            import json
+            import queue
+            import secrets
+            import threading
+            import time
+            import urllib.request
+            from concurrent.futures import ThreadPoolExecutor, as_completed
+            from datetime import datetime, timezone
+
+            SOURCE_BUCKET = "cer-envelope-graphs"
+            SOURCE_PREFIX = "ce_registry/"
+            DEST_BUCKET = os.environ["DEST_BUCKET"]
+            SLACK_WEBHOOK = os.environ["SLACK_WEBHOOK"]
+            COMMUNITY_NAME = "ce_registry"
+            WORKERS = 64
+            ZIP_PATH = "/tmp/ce_registry.zip"
+
+            zip_key = f"{COMMUNITY_NAME}_{int(time.time())}_{secrets.token_hex(16)}.zip"
+
+            def notify_slack(text):
+                if not SLACK_WEBHOOK:
+                    return
+                try:
+                    payload = json.dumps({"text": text}).encode()
+                    req = urllib.request.Request(
+                        SLACK_WEBHOOK, data=payload,
+                        headers={"Content-Type": "application/json"}
+                    )
+                    urllib.request.urlopen(req, timeout=10)
+                except Exception as e:
+                    print(f"Warning: Slack notification failed: {e}")
+
+            s3 = boto3.client("s3")
+            print(f"Source:      s3://{SOURCE_BUCKET}/{SOURCE_PREFIX}")
+            print(f"Destination: s3://{DEST_BUCKET}/{zip_key}")
+            print()
+
+            job_start = datetime.now(timezone.utc)
+            zip_size_mb = None
+            error_msg = None
+
+            try:
+                # List all *.json objects
+                paginator = s3.get_paginator("list_objects_v2")
+                json_keys = []
+                for page in paginator.paginate(Bucket=SOURCE_BUCKET, Prefix=SOURCE_PREFIX):
+                    for obj in page.get("Contents", []):
+                        if obj["Key"].endswith(".json"):
+                            json_keys.append(obj["Key"])
+
+                if not json_keys:
+                    raise RuntimeError("No .json files found at source location")
+
+                total = len(json_keys)
+                print(f"[{datetime.now(timezone.utc).strftime('%H:%M:%S')}] Found {total} JSON file(s). Downloading+compressing with {WORKERS} workers...")
+                start_time = datetime.now(timezone.utc)
+
+                result_queue = queue.Queue(maxsize=WORKERS * 2)
+                counter = {"done": 0}
+                lock = threading.Lock()
+
+                def download_and_compress(key):
+                    """Download from S3 and compress with raw DEFLATE in the worker thread."""
+                    obj = s3.get_object(Bucket=SOURCE_BUCKET, Key=key)
+                    data = obj["Body"].read()
+                    crc = zlib.crc32(data) & 0xFFFFFFFF
+                    compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, zlib.DEFLATED, -15)
+                    compressed = compressor.compress(data) + compressor.flush()
+                    return os.path.basename(key), compressed, crc, len(data)
+
+                def write_precompressed(zf, filename, compressed_data, crc32, original_size):
+                    """Write pre-compressed raw DEFLATE data directly into the ZIP (no re-compression)."""
+                    zinfo = zipfile.ZipInfo(filename=filename)
+                    zinfo.file_size = original_size
+                    zinfo.compress_size = len(compressed_data)
+                    zinfo.CRC = crc32
+                    zinfo.compress_type = zipfile.ZIP_DEFLATED
+                    zinfo.flag_bits = 0
+                    with zf._lock:
+                        zinfo.header_offset = zf.fp.tell()
+                        zf.fp.write(zinfo.FileHeader())
+                        zf.fp.write(compressed_data)
+                        zf.filelist.append(zinfo)
+                        zf.NameToInfo[filename] = zinfo
+                        zf._didModify = True
+
+                def producer():
+                    with ThreadPoolExecutor(max_workers=WORKERS) as pool:
+                        futures = {pool.submit(download_and_compress, k): k for k in sorted(json_keys)}
+                        for future in as_completed(futures):
+                            result_queue.put(future.result())
+                    result_queue.put(None)  # sentinel
+
+                producer_thread = threading.Thread(target=producer, daemon=True)
+                producer_thread.start()
+
+                # Main thread writes pre-compressed entries to ZIP (pure I/O, no CPU compression)
+                with zipfile.ZipFile(ZIP_PATH, "w") as zf:
+                    while True:
+                        item = result_queue.get()
+                        if item is None:
+                            break
+                        filename, compressed_data, crc32, original_size = item
+                        write_precompressed(zf, filename, compressed_data, crc32, original_size)
+                        with lock:
+                            counter["done"] += 1
+                            done = counter["done"]
+                        if done % 2000 == 0:
+                            elapsed = (datetime.now(timezone.utc) - start_time).seconds
+                            rate = done / elapsed if elapsed > 0 else 0
+                            eta = int((total - done) / rate) if rate > 0 else 0
+                            print(f"  [{datetime.now(timezone.utc).strftime('%H:%M:%S')}] {done}/{total} files ({done*100//total}%) — {rate:.0f} files/s — ETA: {eta//60}m{eta%60:02d}s")
+
+                producer_thread.join()
+
+                zip_size_mb = os.path.getsize(ZIP_PATH) / 1024 / 1024
+                print(f"\nZIP size: {zip_size_mb:.2f} MB")
+
+                print(f"Uploading to s3://{DEST_BUCKET}/{zip_key} ...")
+                s3.upload_file(ZIP_PATH, DEST_BUCKET, zip_key,
+                               ExtraArgs={"ContentType": "application/zip"})
+
+                print(f"\n=== Done! Uploaded s3://{DEST_BUCKET}/{zip_key} ===")
+
+            except Exception as e:
+                error_msg = str(e)
+                print(f"\nERROR: {error_msg}")
+                raise
+
+            finally:
+                duration = int((datetime.now(timezone.utc) - job_start).total_seconds())
+                dur_str = f"{duration // 60}m{duration % 60:02d}s"
+                if error_msg:
+                    msg = (
+                        f":x: *CE Registry ZIP bundle failed* (staging)\n"
+                        f">*Duration:* {dur_str}\n"
+                        f">*Error:* {error_msg}"
+                    )
+                else:
+                    msg = (
+                        f":white_check_mark: *CE Registry ZIP bundle succeeded* (staging)\n"
+                        f">*Files:* {total:,}\n"
+                        f">*ZIP size:* {zip_size_mb:.2f} MB\n"
+                        f">*Uploaded:* `s3://{DEST_BUCKET}/{zip_key}`\n"
+                        f">*Duration:* {dur_str}"
+                    )
+                notify_slack(msg)
+            PYEOF
+        env:
+          - name: DEST_BUCKET
+            value: "{{inputs.parameters.dest-bucket}}"
+          - name: SLACK_WEBHOOK
+            value: "{{inputs.parameters.slack-webhook}}"
+        resources:
+          requests:
+            cpu: "1000m"
+            memory: "2Gi"
+          limits:
+            cpu: "2000m"
+            memory: "4Gi"
+      activeDeadlineSeconds: 10800
+      retryStrategy:
+        limit: 2
+        retryPolicy: OnFailure
+        backoff:
+          duration: "60s"
+          factor: 2
+          maxDuration: "3h"
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/externalsecret.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/externalsecret.yaml
index 5e782e72..4c274281 100644
--- a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/externalsecret.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/externalsecret.yaml
@@ -36,3 +36,7 @@ spec:
       remoteRef:
         key: credreg-argo-workflows-staging
         property: sslmode
+    - secretKey: argo_token
+      remoteRef:
+        key: credreg-argo-workflows-staging
+        property: argo_token
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/rbac.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/rbac.yaml
index e7260839..f6d64339 100644
--- a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/rbac.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/rbac.yaml
@@ -3,6 +3,8 @@ kind: ServiceAccount
 metadata:
   name: argo-workflow-controller
   namespace: credreg-staging
+  annotations:
+    eks.amazonaws.com/role-arn: arn:aws:iam::996810415034:role/ce-registry-eks-application-irsa-role
   labels:
     app.kubernetes.io/component: controller
     app.kubernetes.io/part-of: argo-workflows
@@ -83,3 +85,27 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: argo-server
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: argo-workflow-executor
+  namespace: credreg-staging
+rules:
+  - apiGroups: ["argoproj.io"]
+    resources: ["workflowtaskresults"]
+    verbs: ["create", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: argo-workflow-executor-main-app
+  namespace: credreg-staging
+subjects:
+  - kind: ServiceAccount
+    name: main-app-service-account
+    namespace: credreg-staging
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: argo-workflow-executor
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/validate-graph-resources-workflow-template.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/validate-graph-resources-workflow-template.yaml
new file mode 100644
index 00000000..13ff9fb0
--- /dev/null
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/validate-graph-resources-workflow-template.yaml
@@ -0,0 +1,179 @@
+apiVersion: argoproj.io/v1alpha1
+kind: WorkflowTemplate
+metadata:
+  name: validate-graph-resources
+  namespace: credreg-staging
+  labels:
+    app: credential-registry
+spec:
+  serviceAccountName: main-app-service-account
+  entrypoint: validate-graph-resources
+  arguments:
+    parameters:
+      - name: graph-s3-path
+      - name: dest-bucket
+        value: "cer-resources-prod"
+      - name: slack-webhook
+        value: ""
+  templates:
+    - name: validate-graph-resources
+      inputs:
+        parameters:
+          - name: graph-s3-path
+          - name: dest-bucket
+          - name: slack-webhook
+      metadata:
+        labels:
+          app: credential-registry
+          workflow: validate-graph-resources
+      container:
+        image: python:3.11-slim
+        command:
+          - /bin/sh
+          - -c
+          - |
+            set -e
+            echo "=== Validate Graph Resources ==="
+            echo "Started at: $(date -u)"
+            echo ""
+
+            echo "[$(date -u +%H:%M:%S)] Upgrading pip..."
+            pip install --quiet --upgrade pip --root-user-action=ignore
+            echo "[$(date -u +%H:%M:%S)] Installing boto3..."
+            pip install --quiet boto3 --root-user-action=ignore
+            echo "[$(date -u +%H:%M:%S)] Dependencies ready."
+            echo ""
+
+            python3 - << 'PYEOF'
+            import boto3
+            import json
+            import os
+            import urllib.request
+            from concurrent.futures import ThreadPoolExecutor, as_completed
+            from datetime import datetime, timezone
+            from urllib.parse import urlparse
+
+            GRAPH_S3_PATH = os.environ["GRAPH_S3_PATH"]
+            DEST_BUCKET = os.environ["DEST_BUCKET"]
+            SLACK_WEBHOOK = os.environ["SLACK_WEBHOOK"]
+            WORKERS = 32
+
+            def notify_slack(text):
+                if not SLACK_WEBHOOK:
+                    return
+                try:
+                    payload = json.dumps({"text": text}).encode()
+                    req = urllib.request.Request(
+                        SLACK_WEBHOOK, data=payload,
+                        headers={"Content-Type": "application/json"}
+                    )
+                    urllib.request.urlopen(req, timeout=10)
+                except Exception as e:
+                    print(f"Warning: Slack notification failed: {e}")
+
+            def parse_s3_path(s3_path):
+                parsed = urlparse(s3_path)
+                return parsed.netloc, parsed.path.lstrip("/")
+
+            s3 = boto3.client("s3")
+            source_bucket, source_key = parse_s3_path(GRAPH_S3_PATH)
+
+            print(f"Graph:       {GRAPH_S3_PATH}")
+            print(f"Destination: s3://{DEST_BUCKET}/{{ctid}}.json")
+            print()
+
+            job_start = datetime.now(timezone.utc)
+            uploaded_count = 0
+            error_msg = None
+
+            try:
+                print(f"[{datetime.now(timezone.utc).strftime('%H:%M:%S')}] Downloading graph...")
+                obj = s3.get_object(Bucket=source_bucket, Key=source_key)
+                graph_json = obj["Body"].read().decode("utf-8")
+
+                data = json.loads(graph_json)
+                if "@graph" not in data or not isinstance(data["@graph"], list):
+                    raise ValueError("Graph JSON does not contain a valid '@graph' array.")
+
+                resources = []
+                for element in data["@graph"]:
+                    if "ceterms:ctid" not in element:
+                        raise ValueError(
+                            f"Resource is missing 'ceterms:ctid': {json.dumps(element)[:100]}..."
+                        )
+                    ctid = element["ceterms:ctid"]
+                    if not ctid:
+                        raise ValueError("'ceterms:ctid' is null.")
+                    resources.append((ctid, json.dumps(element)))
+
+                total = len(resources)
+                print(f"[{datetime.now(timezone.utc).strftime('%H:%M:%S')}] Validated {total} resource(s). Uploading with {WORKERS} workers...")
+                start_time = datetime.now(timezone.utc)
+
+                def upload(args):
+                    ctid, resource_json = args
+                    s3.put_object(
+                        Bucket=DEST_BUCKET,
+                        Key=f"{ctid}.json",
+                        Body=resource_json.encode("utf-8"),
+                        ContentType="application/json"
+                    )
+                    return ctid
+
+                with ThreadPoolExecutor(max_workers=WORKERS) as pool:
+                    futures = {pool.submit(upload, r): r[0] for r in resources}
+                    for i, future in enumerate(as_completed(futures), 1):
+                        ctid = future.result()
+                        print(f"  [{i}/{total}] Uploaded {ctid}.json")
+
+                uploaded_count = total
+                duration = int((datetime.now(timezone.utc) - start_time).total_seconds())
+                print(f"\n=== Done! {total} resource(s) uploaded to s3://{DEST_BUCKET}/ in {duration}s ===")
+
+            except Exception as e:
+                error_msg = str(e)
+                print(f"\nERROR: {error_msg}")
+                raise
+
+            finally:
+                duration = int((datetime.now(timezone.utc) - job_start).total_seconds())
+                dur_str = f"{duration // 60}m{duration % 60:02d}s"
+                if error_msg:
+                    msg = (
+                        f":x: *Validate Graph Resources failed* (staging)\n"
+                        f">*Graph:* `{GRAPH_S3_PATH}`\n"
+                        f">*Duration:* {dur_str}\n"
+                        f">*Error:* {error_msg}"
+                    )
+                else:
+                    msg = (
+                        f":white_check_mark: *Validate Graph Resources succeeded* (staging)\n"
+                        f">*Graph:* `{GRAPH_S3_PATH}`\n"
+                        f">*Resources uploaded:* {uploaded_count:,}\n"
+                        f">*Destination:* `s3://{DEST_BUCKET}/`\n"
+                        f">*Duration:* {dur_str}"
+                    )
+                notify_slack(msg)
+            PYEOF
+        env:
+          - name: GRAPH_S3_PATH
+            value: "{{inputs.parameters.graph-s3-path}}"
+          - name: DEST_BUCKET
+            value: "{{inputs.parameters.dest-bucket}}"
+          - name: SLACK_WEBHOOK
+            value: "{{inputs.parameters.slack-webhook}}"
+        resources:
+          requests:
+            cpu: "200m"
+            memory: "256Mi"
+          limits:
+            cpu: "500m"
+            memory: "512Mi"
+      activeDeadlineSeconds: 3600
+      retryStrategy:
+        limit: 2
+        retryPolicy: OnFailure
+        backoff:
+          duration: "30s"
+          factor: 2
+          maxDuration: "1h"
diff --git a/terraform/modules/eks/irsa-iam-policy-and-role.tf b/terraform/modules/eks/irsa-iam-policy-and-role.tf
index df33dcba..c7236b43 100644
--- a/terraform/modules/eks/irsa-iam-policy-and-role.tf
+++ b/terraform/modules/eks/irsa-iam-policy-and-role.tf
@@ -123,7 +123,10 @@ resource "aws_iam_policy" "application_policy" {
           "arn:aws:s3:::cer-envelope-graphs-sandbox/*",
           "arn:aws:s3:::cer-envelope-graphs-sandb/*",
           "arn:aws:s3:::cer-envelope-graphs-prod/*",
-          "arn:aws:s3:::cer-envelope-downloads/*"
+          "arn:aws:s3:::cer-envelope-downloads/*",
+          "arn:aws:s3:::cer-envelope-graphs/*",
+          "arn:aws:s3:::ocn-exports/*",
+          "arn:aws:s3:::cer-resources*/*"
         ]
       },
       {
@@ -139,7 +142,10 @@ resource "aws_iam_policy" "application_policy" {
           "arn:aws:s3:::cer-envelope-graphs-sandbox",
           "arn:aws:s3:::cer-envelope-graphs-sandb",
           "arn:aws:s3:::cer-envelope-graphs-prod",
-          "arn:aws:s3:::cer-envelope-downloads"
+          "arn:aws:s3:::cer-envelope-downloads",
+          "arn:aws:s3:::cer-envelope-graphs",
+          "arn:aws:s3:::ocn-exports",
+          "arn:aws:s3:::cer-resources*"
         ]
       }
     ]

From af25e88c84b8a2aa28a97ccc59d863f84038ff59 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Thu, 19 Feb 2026 19:01:57 -0300
Subject: [PATCH 11/16] Remove S3-to-Elasticsearch indexing workflow

Drops the ES indexing API endpoint, service, policy, rake task, and
Argo WorkflowTemplate as this functionality is no longer needed.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
---
 app/api/v1/base.rb                            |   2 -
 app/api/v1/workflows.rb                       |  61 ------
 app/policies/workflow_policy.rb               |  12 -
 app/services/index_s3_graph_to_es.rb          |  85 --------
 lib/tasks/s3.rake                             |  77 -------
 .../index-s3-to-es-workflow-template.yaml     | 109 ----------
 .../jobs/index-s3-to-es-direct.yaml           | 205 ------------------
 7 files changed, 551 deletions(-)
 delete mode 100644 app/api/v1/workflows.rb
 delete mode 100644 app/policies/workflow_policy.rb
 delete mode 100644 app/services/index_s3_graph_to_es.rb
 delete mode 100644 lib/tasks/s3.rake
 delete mode 100644 terraform/environments/eks/k8s-manifests-staging/argo-workflow/index-s3-to-es-workflow-template.yaml
 delete mode 100644 terraform/environments/eks/k8s-manifests-staging/argo-workflow/jobs/index-s3-to-es-direct.yaml

diff --git a/app/api/v1/base.rb b/app/api/v1/base.rb
index b8821fc4..e63275b1 100644
--- a/app/api/v1/base.rb
+++ b/app/api/v1/base.rb
@@ -17,7 +17,6 @@
 require 'v1/indexed_resources'
 require 'v1/indexer'
 require 'v1/envelope_communities'
-require 'v1/workflows'
 
 module API
   module V1
@@ -66,7 +65,6 @@ class Base < Grape::API
         mount API::V1::Publishers
       end
 
-      mount API::V1::Workflows
     end
   end
 end
diff --git a/app/api/v1/workflows.rb b/app/api/v1/workflows.rb
deleted file mode 100644
index cbafc9f0..00000000
--- a/app/api/v1/workflows.rb
+++ /dev/null
@@ -1,61 +0,0 @@
-require 'mountable_api'
-require 'helpers/shared_helpers'
-
-module API
-  module V1
-    # Endpoints for operational workflows (called by Argo Workflows)
-    class Workflows < MountableAPI
-      mounted do
-        helpers SharedHelpers
-
-        before do
-          authenticate!
-        end
-
-        resource :workflows do
-          desc 'Indexes all S3 JSON-LD graphs to Elasticsearch. ' \
-               'S3 is treated as the source of truth. ' \
-               'Called by Argo Workflows for orchestration.'
-          post 'index-all-s3-to-es' do
-            authorize :workflow, :trigger?
-
-            bucket_name = ENV['ENVELOPE_GRAPHS_BUCKET']
-            error!({ error: 'ENVELOPE_GRAPHS_BUCKET not configured' }, 500) unless bucket_name
-
-            es_address = ENV['ELASTICSEARCH_ADDRESS']
-            error!({ error: 'ELASTICSEARCH_ADDRESS not configured' }, 500) unless es_address
-
-            s3 = Aws::S3::Resource.new(region: ENV['AWS_REGION'].presence)
-            bucket = s3.bucket(bucket_name)
-
-            errors = {}
-            processed = 0
-            skipped = 0
-
-            bucket.objects.each do |object|
-              next unless object.key.end_with?('.json')
-
-              processed += 1
-
-              begin
-                IndexS3GraphToEs.call(object.key)
-              rescue StandardError => e
-                errors[object.key] = "#{e.class}: #{e.message}"
-              end
-            end
-
-            status_code = errors.empty? ? 200 : 207
-
-            status status_code
-            {
-              message: errors.empty? ? 'Indexing completed successfully' : 'Indexing completed with errors',
-              processed: processed,
-              errors_count: errors.size,
-              errors: errors.first(100).to_h
-            }
-          end
-        end
-      end
-    end
-  end
-end
diff --git a/app/policies/workflow_policy.rb b/app/policies/workflow_policy.rb
deleted file mode 100644
index 380810d6..00000000
--- a/app/policies/workflow_policy.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-require_relative 'application_policy'
-
-# Specifies policies for workflow operations
-class WorkflowPolicy < ApplicationPolicy
-  def trigger?
-    user.admin?
-  end
-
-  def show?
-    user.admin?
-  end
-end
diff --git a/app/services/index_s3_graph_to_es.rb b/app/services/index_s3_graph_to_es.rb
deleted file mode 100644
index 0dd0f912..00000000
--- a/app/services/index_s3_graph_to_es.rb
+++ /dev/null
@@ -1,85 +0,0 @@
-# Indexes a JSON-LD graph from S3 directly to Elasticsearch
-# Does not require database access - S3 is the source of truth
-class IndexS3GraphToEs
-  attr_reader :s3_key, :community_name, :ctid
-
-  def initialize(s3_key)
-    @s3_key = s3_key
-    parse_s3_key
-  end
-
-  class << self
-    def call(s3_key)
-      new(s3_key).call
-    end
-  end
-
-  def call
-    return unless elasticsearch_address
-
-    client.index(
-      body: graph_json,
-      id: ctid,
-      index: community_name
-    )
-  rescue Elastic::Transport::Transport::Errors::BadRequest => e
-    raise e unless e.message.include?('Limit of total fields')
-
-    increase_total_fields_limit
-    retry
-  end
-
-  private
-
-  def parse_s3_key
-    # S3 key format: {community_name}/{ctid}.json
-    parts = s3_key.split('/')
-    @community_name = parts[0..-2].join('/')
-    @ctid = parts.last.sub(/\.json\z/i, '')
-  end
-
-  def graph_content
-    @graph_content ||= s3_object.get.body.read
-  end
-
-  def graph_json
-    @graph_json ||= JSON.parse(graph_content).to_json
-  end
-
-  def client
-    @client ||= Elasticsearch::Client.new(host: elasticsearch_address)
-  end
-
-  def elasticsearch_address
-    ENV['ELASTICSEARCH_ADDRESS'].presence
-  end
-
-  def s3_bucket
-    @s3_bucket ||= s3_resource.bucket(s3_bucket_name)
-  end
-
-  def s3_bucket_name
-    ENV['ENVELOPE_GRAPHS_BUCKET'].presence
-  end
-
-  def s3_object
-    @s3_object ||= s3_bucket.object(s3_key)
-  end
-
-  def s3_resource
-    @s3_resource ||= Aws::S3::Resource.new(region: ENV['AWS_REGION'].presence)
-  end
-
-  def increase_total_fields_limit
-    settings = client.indices.get_settings(index: community_name)
-
-    current_limit = settings
-                    .dig(community_name, 'settings', 'index', 'mapping', 'total_fields', 'limit')
-                    .to_i
-
-    client.indices.put_settings(
-      body: { 'index.mapping.total_fields.limit' => current_limit * 2 },
-      index: community_name
-    )
-  end
-end
diff --git a/lib/tasks/s3.rake b/lib/tasks/s3.rake
deleted file mode 100644
index 6ae3f5c5..00000000
--- a/lib/tasks/s3.rake
+++ /dev/null
@@ -1,77 +0,0 @@
-namespace :s3 do
-  desc 'Index all S3 JSON-LD graphs to Elasticsearch (S3 as source of truth)'
-  task index_all_to_es: :environment do
-    require 'benchmark'
-    require 'json'
-
-    bucket_name = ENV['ENVELOPE_GRAPHS_BUCKET']
-    abort 'ENVELOPE_GRAPHS_BUCKET environment variable is not set' unless bucket_name
-
-    es_address = ENV['ELASTICSEARCH_ADDRESS']
-    abort 'ELASTICSEARCH_ADDRESS environment variable is not set' unless es_address
-
-    $stdout.sync = true
-
-    s3 = Aws::S3::Resource.new(region: ENV['AWS_REGION'].presence)
-    bucket = s3.bucket(bucket_name)
-
-    errors = {}
-    processed = 0
-    skipped = 0
-
-    puts "Starting S3 to ES indexing from bucket: #{bucket_name}"
-    puts "Elasticsearch address: #{es_address}"
-    puts "Counting objects..."
-
-    # Count total objects for progress reporting
-    total = bucket.objects.count { |obj| obj.key.end_with?('.json') }
-    puts "Found #{total} JSON files to index"
-    puts "Started at #{Time.now.utc}"
-
-    time = Benchmark.measure do
-      bucket.objects.each do |object|
-        next unless object.key.end_with?('.json')
-
-        processed += 1
-
-        begin
-          IndexS3GraphToEs.call(object.key)
-        rescue StandardError => e
-          errors[object.key] = "#{e.class}: #{e.message}"
-        end
-
-        # Progress every 100 records
-        if (processed % 100).zero?
-          puts "Progress: processed=#{processed}/#{total} errors=#{errors.size} skipped=#{skipped}"
-        end
-      end
-    end
-
-    puts time
-    puts "Finished at #{Time.now.utc} - processed=#{processed}, errors=#{errors.size}"
-
-    # Write errors to file
-    if errors.any?
-      File.write('/tmp/s3_index_errors.json', JSON.pretty_generate(errors))
-      puts "Wrote /tmp/s3_index_errors.json (#{errors.size} entries)"
-
-      # Upload errors to S3
-      begin
-        error_bucket = ENV['S3_ERRORS_BUCKET'] || bucket_name
-        error_key = "errors/s3-index-errors-#{Time.now.utc.strftime('%Y%m%dT%H%M%SZ')}.json"
-        s3_client = Aws::S3::Client.new(region: ENV['AWS_REGION'].presence)
-        s3_client.put_object(
-          bucket: error_bucket,
-          key: error_key,
-          body: File.open('/tmp/s3_index_errors.json', 'rb')
-        )
-        puts "Uploaded errors to s3://#{error_bucket}/#{error_key}"
-      rescue StandardError => e
-        warn "Failed to upload errors to S3: #{e.class}: #{e.message}"
-      end
-
-      warn "Encountered #{errors.size} errors. Sample: #{errors.to_a.first(5).to_h.inspect}"
-      exit 1
-    end
-  end
-end
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/index-s3-to-es-workflow-template.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/index-s3-to-es-workflow-template.yaml
deleted file mode 100644
index 539f4af2..00000000
--- a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/index-s3-to-es-workflow-template.yaml
+++ /dev/null
@@ -1,109 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: WorkflowTemplate
-metadata:
-  name: index-s3-to-es
-  namespace: credreg-staging
-  labels:
-    app: credential-registry
-spec:
-  serviceAccountName: argo-workflow-controller
-  entrypoint: index-s3-to-es
-  arguments:
-    parameters:
-      - name: api-base-url
-        value: "http://main-app.credreg-staging.svc.cluster.local:9292"
-      - name: keycloak-url
-        value: "https://test-ce-kc-002.credentialengine.org/realms/CE-Test/protocol/openid-connect/token"
-  templates:
-    - name: index-s3-to-es
-      metadata:
-        labels:
-          app: credential-registry
-          workflow: index-s3-to-es
-      inputs:
-        parameters:
-          - name: api-base-url
-          - name: keycloak-url
-      container:
-        image: curlimages/curl:latest
-        command:
-          - /bin/sh
-          - -c
-          - |
-            set -e
-            echo "=== S3 to Elasticsearch Indexing Workflow ==="
-            echo "Started at: $(date -u)"
-            echo ""
-
-            # Step 1: Get access token from Keycloak
-            echo "Step 1: Authenticating with Keycloak..."
-            TOKEN_RESPONSE=$(curl -sf -X POST "{{inputs.parameters.keycloak-url}}" \
-              -H "Content-Type: application/x-www-form-urlencoded" \
-              -d "grant_type=client_credentials" \
-              -d "client_id=${KEYCLOAK_CLIENT_ID}" \
-              -d "client_secret=${KEYCLOAK_CLIENT_SECRET}")
-
-            ACCESS_TOKEN=$(echo "$TOKEN_RESPONSE" | sed -n 's/.*"access_token":"\([^"]*\)".*/\1/p')
-
-            if [ -z "$ACCESS_TOKEN" ]; then
-              echo "ERROR: Failed to obtain access token from Keycloak"
-              echo "Response: $TOKEN_RESPONSE"
-              exit 1
-            fi
-            echo "Successfully obtained access token"
-            echo ""
-
-            # Step 2: Call Registry API
-            echo "Step 2: Calling Registry API at {{inputs.parameters.api-base-url}}"
-            response=$(curl -sf -X POST \
-              "{{inputs.parameters.api-base-url}}/workflows/index-all-s3-to-es" \
-              -H "Authorization: Bearer ${ACCESS_TOKEN}" \
-              -H "Content-Type: application/json" \
-              -w "\n%{http_code}" \
-              --max-time 43200)
-
-            # Extract HTTP status code (last line)
-            http_code=$(echo "$response" | tail -n1)
-            body=$(echo "$response" | sed '$d')
-
-            echo ""
-            echo "=== API Response ==="
-            echo "HTTP Status: $http_code"
-            echo "Body:"
-            echo "$body"
-            echo ""
-
-            # Exit with error if not success (200 or 207)
-            if [ "$http_code" != "200" ] && [ "$http_code" != "207" ]; then
-              echo "ERROR: API call failed with status $http_code"
-              exit 1
-            fi
-
-            echo "=== Workflow Completed Successfully ==="
-            echo "Finished at: $(date -u)"
-        env:
-          - name: KEYCLOAK_CLIENT_ID
-            valueFrom:
-              secretKeyRef:
-                name: argo-keycloak-credentials
-                key: client_id
-          - name: KEYCLOAK_CLIENT_SECRET
-            valueFrom:
-              secretKeyRef:
-                name: argo-keycloak-credentials
-                key: client_secret
-        resources:
-          requests:
-            cpu: "100m"
-            memory: "128Mi"
-          limits:
-            cpu: "200m"
-            memory: "256Mi"
-      retryStrategy:
-        limit: 2
-        retryPolicy: OnFailure
-        backoff:
-          duration: "60s"
-          factor: 2
-          maxDuration: "10m"
-      activeDeadlineSeconds: 43200
diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/jobs/index-s3-to-es-direct.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/jobs/index-s3-to-es-direct.yaml
deleted file mode 100644
index 04f9902f..00000000
--- a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/jobs/index-s3-to-es-direct.yaml
+++ /dev/null
@@ -1,205 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: WorkflowTemplate
-metadata:
-  name: index-s3-to-es-direct
-  namespace: credreg-staging
-  labels:
-    app: credential-registry
-spec:
-  serviceAccountName: main-app-service-account
-  entrypoint: index-s3-to-es
-  arguments:
-    parameters:
-      - name: s3-bucket
-        value: "cer-envelope-graphs-staging"
-      - name: es-endpoint
-        value: "http://elasticsearch.credreg-staging.svc.cluster.local:9200"
-      - name: batch-size
-        value: "500"
-  templates:
-    - name: index-s3-to-es
-      metadata:
-        labels:
-          app: credential-registry
-          workflow: index-s3-to-es-direct
-      inputs:
-        parameters:
-          - name: s3-bucket
-          - name: es-endpoint
-          - name: batch-size
-      container:
-        image: public.ecr.aws/aws-cli/aws-cli:latest
-        command:
-          - /bin/bash
-          - -c
-          - |
-            set -e
-
-            echo "=== S3 to Elasticsearch Direct Indexing ==="
-            echo "Started at: $(date -u)"
-            echo ""
-
-            # Install curl
-            echo "Installing curl..."
-            yum install -y curl --quiet
-
-            S3_BUCKET="{{inputs.parameters.s3-bucket}}"
-            ES_ENDPOINT="{{inputs.parameters.es-endpoint}}"
-            BATCH_SIZE="{{inputs.parameters.batch-size}}"
-
-            # Generate index name with ce- prefix and ISO8601 UTC timestamp
-            INDEX_NAME="ce-$(date -u +%Y-%m-%dT%H:%M:%SZ)"
-            echo "Target index: ${INDEX_NAME}"
-            echo "S3 bucket: ${S3_BUCKET}"
-            echo "ES endpoint: ${ES_ENDPOINT}"
-            echo "Batch size: ${BATCH_SIZE}"
-            echo ""
-
-            # Delete index if exists
-            echo "Step 1: Checking if index exists..."
-            HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "${ES_ENDPOINT}/${INDEX_NAME}")
-            if [ "$HTTP_CODE" = "200" ]; then
-              echo "Index exists, deleting..."
-              curl -s -X DELETE "${ES_ENDPOINT}/${INDEX_NAME}"
-              echo "Index deleted."
-            else
-              echo "Index does not exist."
-            fi
-            echo ""
-
-            # Create new index
-            echo "Step 2: Creating index ${INDEX_NAME}..."
-            curl -s -X PUT "${ES_ENDPOINT}/${INDEX_NAME}" \
-              -H "Content-Type: application/json" \
-              -d '{
-                "settings": {
-                  "number_of_shards": 1,
-                  "number_of_replicas": 0,
-                  "refresh_interval": "-1"
-                }
-              }'
-            echo ""
-            echo "Index created."
-            echo ""
-
-            # List all JSON files from S3
-            echo "Step 3: Listing files from S3..."
-            TMPDIR="/tmp/s3-to-es"
-            mkdir -p "${TMPDIR}"
-
-            aws s3 ls "s3://${S3_BUCKET}/" --recursive | grep '\.json$' | awk '{print $4}' > "${TMPDIR}/files.txt"
-            TOTAL_FILES=$(wc -l < "${TMPDIR}/files.txt")
-            echo "Found ${TOTAL_FILES} JSON files."
-            echo ""
-
-            # Process files in batches using ES _bulk API
-            echo "Step 4: Indexing documents in batches of ${BATCH_SIZE}..."
-            BATCH_NUM=0
-            INDEXED=0
-            ERRORS=0
-
-            while true; do
-              # Get next batch of files
-              OFFSET=$((BATCH_NUM * BATCH_SIZE))
-              BATCH_FILES=$(tail -n +$((OFFSET + 1)) "${TMPDIR}/files.txt" | head -n "${BATCH_SIZE}")
-
-              if [ -z "${BATCH_FILES}" ]; then
-                break
-              fi
-
-              BATCH_NUM=$((BATCH_NUM + 1))
-              BULK_FILE="${TMPDIR}/bulk_${BATCH_NUM}.ndjson"
-              rm -f "${BULK_FILE}"
-
-              # Download files and build bulk request
-              for S3_KEY in ${BATCH_FILES}; do
-                # Extract CTID from filename (without .json extension)
-                FILENAME=$(basename "${S3_KEY}" .json)
-
-                # Download file
-                if aws s3 cp "s3://${S3_BUCKET}/${S3_KEY}" "${TMPDIR}/doc.json" --quiet 2>/dev/null; then
-                  # Append action line and document to bulk file
-                  echo "{\"index\":{\"_index\":\"${INDEX_NAME}\",\"_id\":\"${FILENAME}\"}}" >> "${BULK_FILE}"
-                  # Document must be on single line for bulk API
-                  cat "${TMPDIR}/doc.json" | tr -d '\n' >> "${BULK_FILE}"
-                  echo "" >> "${BULK_FILE}"
-                else
-                  echo "Warning: Failed to download ${S3_KEY}"
-                  ERRORS=$((ERRORS + 1))
-                fi
-              done
-
-              # Send bulk request
-              if [ -f "${BULK_FILE}" ] && [ -s "${BULK_FILE}" ]; then
-                RESPONSE=$(curl -s -X POST "${ES_ENDPOINT}/_bulk" \
-                  -H "Content-Type: application/x-ndjson" \
-                  --data-binary @"${BULK_FILE}")
-
-                # Check for errors in response
-                HAS_ERRORS=$(echo "${RESPONSE}" | grep -o '"errors":\s*true' || true)
-                if [ -n "${HAS_ERRORS}" ]; then
-                  echo "Warning: Batch ${BATCH_NUM} had some errors"
-                  ERRORS=$((ERRORS + 1))
-                fi
-
-                BATCH_COUNT=$(echo "${BATCH_FILES}" | wc -w)
-                INDEXED=$((INDEXED + BATCH_COUNT))
-                echo "Batch ${BATCH_NUM}: indexed ${BATCH_COUNT} documents (total: ${INDEXED}/${TOTAL_FILES})"
-              fi
-
-              # Cleanup batch file
-              rm -f "${BULK_FILE}"
-            done
-
-            echo ""
-
-            # Refresh index to make documents searchable
-            echo "Step 5: Refreshing index..."
-            curl -s -X POST "${ES_ENDPOINT}/${INDEX_NAME}/_refresh"
-            echo ""
-
-            # Update index settings for production
-            echo "Step 6: Updating index settings..."
-            curl -s -X PUT "${ES_ENDPOINT}/${INDEX_NAME}/_settings" \
-              -H "Content-Type: application/json" \
-              -d '{
-                "index": {
-                  "refresh_interval": "1s",
-                  "number_of_replicas": 1
-                }
-              }'
-            echo ""
-
-            # Get final count
-            DOC_COUNT=$(curl -s "${ES_ENDPOINT}/${INDEX_NAME}/_count" | grep -o '"count":[0-9]*' | cut -d: -f2)
-            echo ""
-            echo "=== Indexing Complete ==="
-            echo "Index name: ${INDEX_NAME}"
-            echo "Documents indexed: ${DOC_COUNT}"
-            echo "Errors: ${ERRORS}"
-            echo "Finished at: $(date -u)"
-
-            if [ "${ERRORS}" -gt 0 ]; then
-              echo "Warning: Completed with ${ERRORS} errors"
-              exit 0
-            fi
-        env:
-          - name: AWS_REGION
-            value: us-east-1
-          - name: AWS_DEFAULT_REGION
-            value: us-east-1
-        resources:
-          requests:
-            cpu: "500m"
-            memory: "512Mi"
-          limits:
-            cpu: "1000m"
-            memory: "1Gi"
-      retryStrategy:
-        limit: 2
-        retryPolicy: OnFailure
-        backoff:
-          duration: "60s"
-          factor: 2
-          maxDuration: "10m"
-      activeDeadlineSeconds: 86400  # 24 hours max

From 55f2f530564902b67fe823981e37ea00c7ec4712 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Thu, 19 Feb 2026 20:14:26 -0300
Subject: [PATCH 12/16] Revert app changes to separate PR
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Moves sync_envelope_graph_with_s3.rb and app-deployment.yaml changes
out of this branch — to be delivered in a dedicated PR.
---
 app/services/sync_envelope_graph_with_s3.rb   | 30 -------------------
 .../k8s-manifests-staging/app-deployment.yaml |  5 ----
 2 files changed, 35 deletions(-)

diff --git a/app/services/sync_envelope_graph_with_s3.rb b/app/services/sync_envelope_graph_with_s3.rb
index bea23277..6a9b39e1 100644
--- a/app/services/sync_envelope_graph_with_s3.rb
+++ b/app/services/sync_envelope_graph_with_s3.rb
@@ -27,7 +27,6 @@ def upload
     )
 
     envelope.update_column(:s3_url, s3_object.public_url)
-    trigger_validate_graph_workflow
   end
 
   def remove
@@ -55,33 +54,4 @@ def s3_object
   def s3_resource
     @s3_resource ||= Aws::S3::Resource.new(region: ENV['AWS_REGION'].presence)
   end
-
-  def trigger_validate_graph_workflow
-    argo_token = ENV['ARGO_TOKEN'].presence
-    argo_namespace = ENV['ARGO_NAMESPACE'].presence || 'credreg-staging'
-    dest_bucket = ENV['ARGO_RESOURCE_BUCKET'].presence || 'cer-resources-prod'
-    return unless argo_token
-
-    graph_s3_path = "s3://#{s3_bucket_name}/#{s3_key}"
-    argo_url = "https://argo-server.#{argo_namespace}.svc.cluster.local:2746"
-
-    HTTP.auth("Bearer #{argo_token}")
-        .ssl_context(OpenSSL::SSL::SSLContext.new.tap { |ctx| ctx.verify_mode = OpenSSL::SSL::VERIFY_NONE })
-        .post(
-          "#{argo_url}/api/v1/workflows/#{argo_namespace}/submit",
-          json: {
-            namespace: argo_namespace,
-            resourceKind: 'WorkflowTemplate',
-            resourceName: 'validate-graph-resources',
-            submitOptions: {
-              parameters: [
-                "graph-s3-path=#{graph_s3_path}",
-                "dest-bucket=#{dest_bucket}"
-              ]
-            }
-          }
-        )
-  rescue StandardError => e
-    MR.logger.error("Failed to trigger validate-graph-resources workflow: #{e.message}")
-  end
 end
diff --git a/terraform/environments/eks/k8s-manifests-staging/app-deployment.yaml b/terraform/environments/eks/k8s-manifests-staging/app-deployment.yaml
index f02251ea..1a048426 100644
--- a/terraform/environments/eks/k8s-manifests-staging/app-deployment.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/app-deployment.yaml
@@ -38,11 +38,6 @@ spec:
           env:
             - name: NEW_RELIC_APP_NAME
               value: "Credential Engine Staging"
-            - name: ARGO_TOKEN
-              valueFrom:
-                secretKeyRef:
-                  name: argo-postgres
-                  key: argo_token
           ports:
             - containerPort: 9292
           envFrom:

From 7f60d1b0fcdb348a05a48037994902e0ae944de6 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Thu, 19 Feb 2026 20:19:20 -0300
Subject: [PATCH 13/16] Revert
 k8s-manifests-sandbox/elasticsearch-statefulset.yaml to master

---
 .../k8s-manifests-sandbox/elasticsearch-statefulset.yaml    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-statefulset.yaml b/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-statefulset.yaml
index dfebac0c..71203faf 100644
--- a/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-statefulset.yaml
+++ b/terraform/environments/eks/k8s-manifests-sandbox/elasticsearch-statefulset.yaml
@@ -7,7 +7,7 @@ metadata:
     app: elasticsearch
 spec:
   serviceName: elasticsearch-discovery
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: elasticsearch
@@ -51,8 +51,8 @@ spec:
               value: "false"
             - name: network.host
               value: "0.0.0.0"
-            - name: discovery.type
-              value: "single-node"
+            - name: discovery.seed_hosts
+              value: "elasticsearch-discovery"
           volumeMounts:
             - name: elasticsearch-data
               mountPath: /usr/share/elasticsearch/data

From d81cada7a29162251fe9f78daa173ba2943c9f62 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Thu, 19 Feb 2026 20:22:34 -0300
Subject: [PATCH 14/16] Remove trailing blank line in base.rb

---
 app/api/v1/base.rb | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/api/v1/base.rb b/app/api/v1/base.rb
index e63275b1..a7146caa 100644
--- a/app/api/v1/base.rb
+++ b/app/api/v1/base.rb
@@ -64,7 +64,6 @@ class Base < Grape::API
         mount API::V1::Organizations
         mount API::V1::Publishers
       end
-
     end
   end
 end

From 276d7b20cc520dd20f7465428133acf38c08a495 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Thu, 19 Feb 2026 20:24:52 -0300
Subject: [PATCH 15/16] Revert elasticsearch-statefulset.yaml and
 irsa-iam-policy-and-role.tf to master

---
 .../elasticsearch-statefulset.yaml                     |  6 +++---
 terraform/modules/eks/irsa-iam-policy-and-role.tf      | 10 ++++------
 2 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/terraform/environments/eks/k8s-manifests-staging/elasticsearch-statefulset.yaml b/terraform/environments/eks/k8s-manifests-staging/elasticsearch-statefulset.yaml
index ed9b75ee..b845c13d 100644
--- a/terraform/environments/eks/k8s-manifests-staging/elasticsearch-statefulset.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/elasticsearch-statefulset.yaml
@@ -7,7 +7,7 @@ metadata:
     app: elasticsearch
 spec:
   serviceName: elasticsearch-discovery
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: elasticsearch
@@ -51,8 +51,8 @@ spec:
               value: "false"
             - name: network.host
               value: "0.0.0.0"
-            - name: discovery.type
-              value: "single-node"
+            - name: discovery.seed_hosts
+              value: "elasticsearch-discovery"
           volumeMounts:
             - name: elasticsearch-data
               mountPath: /usr/share/elasticsearch/data
diff --git a/terraform/modules/eks/irsa-iam-policy-and-role.tf b/terraform/modules/eks/irsa-iam-policy-and-role.tf
index c7236b43..4a33ff57 100644
--- a/terraform/modules/eks/irsa-iam-policy-and-role.tf
+++ b/terraform/modules/eks/irsa-iam-policy-and-role.tf
@@ -119,14 +119,13 @@ resource "aws_iam_policy" "application_policy" {
           "s3:DeleteObject"
         ],
         "Resource" : [
+          "arn:aws:s3:::cer-envelope-graphs/*",
           "arn:aws:s3:::cer-envelope-graphs-staging/*",
           "arn:aws:s3:::cer-envelope-graphs-sandbox/*",
           "arn:aws:s3:::cer-envelope-graphs-sandb/*",
           "arn:aws:s3:::cer-envelope-graphs-prod/*",
           "arn:aws:s3:::cer-envelope-downloads/*",
-          "arn:aws:s3:::cer-envelope-graphs/*",
-          "arn:aws:s3:::ocn-exports/*",
-          "arn:aws:s3:::cer-resources*/*"
+          "arn:aws:s3:::ocn-exports/*"
         ]
       },
       {
@@ -138,14 +137,13 @@ resource "aws_iam_policy" "application_policy" {
           "s3:GetBucketLocation"
         ],
         "Resource" : [
+          "arn:aws:s3:::cer-envelope-graphs",
           "arn:aws:s3:::cer-envelope-graphs-staging",
           "arn:aws:s3:::cer-envelope-graphs-sandbox",
           "arn:aws:s3:::cer-envelope-graphs-sandb",
           "arn:aws:s3:::cer-envelope-graphs-prod",
           "arn:aws:s3:::cer-envelope-downloads",
-          "arn:aws:s3:::cer-envelope-graphs",
-          "arn:aws:s3:::ocn-exports",
-          "arn:aws:s3:::cer-resources*"
+          "arn:aws:s3:::ocn-exports"
         ]
       }
     ]

From b0ff7a476e60b231d2bc135b35187f0486680ef8 Mon Sep 17 00:00:00 2001
From: Ariel Rolfo <arielr-lt+username@users.noreply.github.com>
Date: Thu, 19 Feb 2026 20:26:11 -0300
Subject: [PATCH 16/16] skip blank bnodes

---
 .../validate-graph-resources-workflow-template.yaml      | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/validate-graph-resources-workflow-template.yaml b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/validate-graph-resources-workflow-template.yaml
index 13ff9fb0..c1286d89 100644
--- a/terraform/environments/eks/k8s-manifests-staging/argo-workflow/validate-graph-resources-workflow-template.yaml
+++ b/terraform/environments/eks/k8s-manifests-staging/argo-workflow/validate-graph-resources-workflow-template.yaml
@@ -97,13 +97,10 @@ spec:
 
                 resources = []
                 for element in data["@graph"]:
-                    if "ceterms:ctid" not in element:
-                        raise ValueError(
-                            f"Resource is missing 'ceterms:ctid': {json.dumps(element)[:100]}..."
-                        )
-                    ctid = element["ceterms:ctid"]
+                    ctid = element.get("ceterms:ctid")
                     if not ctid:
-                        raise ValueError("'ceterms:ctid' is null.")
+                        # Skip blank nodes and elements without a ctid
+                        continue
                     resources.append((ctid, json.dumps(element)))
 
                 total = len(resources)