feat: Enhance session handling and observability improvements

.github/workflows/components-build-deploy.yml

@@ -84,6 +84,11 @@ jobs:
             image: quay.io/ambient_code/vteam_claude_runner
             dockerfile: ./components/runners/claude-code-runner/Dockerfile
             changed: ${{ needs.detect-changes.outputs.claude-runner }}
+          - name: state-sync
+            context: ./components/runners
+            image: quay.io/ambient_code/vteam_state_sync
+            dockerfile: ./components/runners/state-sync/Dockerfile
+            changed: ${{ needs.detect-changes.outputs.claude-runner }}
     steps:
       - name: Checkout code
         if: matrix.component.changed == 'true' || github.event_name == 'workflow_dispatch'
@@ -163,6 +168,10 @@ jobs:
           oc apply -k components/manifests/base/rbac/
           oc apply -f components/manifests/overlays/production/operator-config-openshift.yaml -n ambient-code
 
+      - name: Deploy observability stack
+        run: |
+          oc apply -k components/manifests/observability/
+
   deploy-to-openshift:
     runs-on: ubuntu-latest
     needs: [detect-changes, build-and-push, update-rbac-and-crd]
@@ -220,6 +229,7 @@ jobs:
           kustomize edit set image quay.io/ambient_code/vteam_backend:latest=quay.io/ambient_code/vteam_backend:${{ steps.image-tags.outputs.backend_tag }}
           kustomize edit set image quay.io/ambient_code/vteam_operator:latest=quay.io/ambient_code/vteam_operator:${{ steps.image-tags.outputs.operator_tag }}
           kustomize edit set image quay.io/ambient_code/vteam_claude_runner:latest=quay.io/ambient_code/vteam_claude_runner:${{ steps.image-tags.outputs.runner_tag }}
+          kustomize edit set image quay.io/ambient_code/vteam_state_sync:latest=quay.io/ambient_code/vteam_state_sync:${{ steps.image-tags.outputs.runner_tag }}
 
       - name: Validate kustomization
         working-directory: components/manifests/overlays/production
@@ -250,7 +260,8 @@ jobs:
         run: |
           oc set env deployment/agentic-operator -n ambient-code -c agentic-operator \
             AMBIENT_CODE_RUNNER_IMAGE="quay.io/ambient_code/vteam_claude_runner:${{ steps.image-tags.outputs.runner_tag }}" \
-            CONTENT_SERVICE_IMAGE="quay.io/ambient_code/vteam_backend:${{ steps.image-tags.outputs.backend_tag }}"
+            CONTENT_SERVICE_IMAGE="quay.io/ambient_code/vteam_backend:${{ steps.image-tags.outputs.backend_tag }}" \
+            STATE_SYNC_IMAGE="quay.io/ambient_code/vteam_state_sync:${{ steps.image-tags.outputs.runner_tag }}"
 
   deploy-with-disptach:
     runs-on: ubuntu-latest
@@ -282,6 +293,7 @@ jobs:
           kustomize edit set image quay.io/ambient_code/vteam_backend:latest=quay.io/ambient_code/vteam_backend:stage
           kustomize edit set image quay.io/ambient_code/vteam_operator:latest=quay.io/ambient_code/vteam_operator:stage
           kustomize edit set image quay.io/ambient_code/vteam_claude_runner:latest=quay.io/ambient_code/vteam_claude_runner:stage
+          kustomize edit set image quay.io/ambient_code/vteam_state_sync:latest=quay.io/ambient_code/vteam_state_sync:stage
 
       - name: Validate kustomization
         working-directory: components/manifests/overlays/production
@@ -309,4 +321,5 @@ jobs:
         run: |
           oc set env deployment/agentic-operator -n ambient-code -c agentic-operator \
             AMBIENT_CODE_RUNNER_IMAGE="quay.io/ambient_code/vteam_claude_runner:stage" \
-            CONTENT_SERVICE_IMAGE="quay.io/ambient_code/vteam_backend:stage"
+            CONTENT_SERVICE_IMAGE="quay.io/ambient_code/vteam_backend:stage" \
+            STATE_SYNC_IMAGE="quay.io/ambient_code/vteam_state_sync:stage"

.github/workflows/prod-release-deploy.yaml

@@ -158,6 +158,10 @@ jobs:
             context: ./components/runners
             image: quay.io/ambient_code/vteam_claude_runner
             dockerfile: ./components/runners/claude-code-runner/Dockerfile
+          - name: state-sync
+            context: ./components/runners
+            image: quay.io/ambient_code/vteam_state_sync
+            dockerfile: ./components/runners/state-sync/Dockerfile
     steps:
       - name: Checkout code from the tag generated above
         uses: actions/checkout@v5
@@ -221,6 +225,10 @@ jobs:
         run: |
           oc login ${{ secrets.PROD_OPENSHIFT_SERVER }} --token=${{ secrets.PROD_OPENSHIFT_TOKEN }} --insecure-skip-tls-verify
 
+      - name: Deploy observability stack
+        run: |
+          oc apply -k components/manifests/observability/
+
       - name: Update kustomization with release image tags
         working-directory: components/manifests/overlays/production
         run: |
@@ -229,6 +237,7 @@ jobs:
           kustomize edit set image quay.io/ambient_code/vteam_backend:latest=quay.io/ambient_code/vteam_backend:${RELEASE_TAG}
           kustomize edit set image quay.io/ambient_code/vteam_operator:latest=quay.io/ambient_code/vteam_operator:${RELEASE_TAG}
           kustomize edit set image quay.io/ambient_code/vteam_claude_runner:latest=quay.io/ambient_code/vteam_claude_runner:${RELEASE_TAG}
+          kustomize edit set image quay.io/ambient_code/vteam_state_sync:latest=quay.io/ambient_code/vteam_state_sync:${RELEASE_TAG}
 
       - name: Validate kustomization
         working-directory: components/manifests/overlays/production
@@ -256,4 +265,5 @@ jobs:
         run: |
           oc set env deployment/agentic-operator -n ambient-code -c agentic-operator \
             AMBIENT_CODE_RUNNER_IMAGE="quay.io/ambient_code/vteam_claude_runner:${{ needs.release.outputs.new_tag }}" \
-            CONTENT_SERVICE_IMAGE="quay.io/ambient_code/vteam_backend:${{ needs.release.outputs.new_tag }}"
+            CONTENT_SERVICE_IMAGE="quay.io/ambient_code/vteam_backend:${{ needs.release.outputs.new_tag }}" \
+            STATE_SYNC_IMAGE="quay.io/ambient_code/vteam_state_sync:${{ needs.release.outputs.new_tag }}"

.gitignore

@@ -140,3 +140,11 @@ reports/
 # Security scan artifacts (transient)
 .security-scan/
 .security-scan.zip
+
+# Secrets (should use .example templates)
+**/minio-credentials-secret.yaml
+
+# Build artifacts and logs
+build.log
+*.log
+!components/**/*.log

Makefile

@@ -1,10 +1,11 @@
-.PHONY: help setup build-all build-frontend build-backend build-operator build-runner deploy clean
+.PHONY: help setup build-all build-frontend build-backend build-operator build-runner build-state-sync deploy clean
 .PHONY: local-up local-down local-clean local-status local-rebuild local-reload-backend local-reload-frontend local-reload-operator local-sync-version
 .PHONY: local-dev-token
 .PHONY: local-logs local-logs-backend local-logs-frontend local-logs-operator local-shell local-shell-frontend
 .PHONY: local-test local-test-dev local-test-quick test-all local-url local-troubleshoot local-port-forward local-stop-port-forward
 .PHONY: push-all registry-login setup-hooks remove-hooks check-minikube check-kubectl
 .PHONY: e2e-test e2e-setup e2e-clean deploy-langfuse-openshift
+.PHONY: setup-minio minio-console minio-logs minio-status
 .PHONY: validate-makefile lint-makefile check-shell makefile-health
 .PHONY: _create-operator-config _auto-port-forward _show-access-info _build-and-load
 
@@ -36,6 +37,7 @@ FRONTEND_IMAGE ?= vteam_frontend:latest
 BACKEND_IMAGE ?= vteam_backend:latest
 OPERATOR_IMAGE ?= vteam_operator:latest
 RUNNER_IMAGE ?= vteam_claude_runner:latest
+STATE_SYNC_IMAGE ?= vteam_state_sync:latest
 
 # Build metadata (captured at build time)
 GIT_COMMIT := $(shell git rev-parse HEAD 2>/dev/null || echo "unknown")
@@ -91,7 +93,7 @@ help: ## Display this help message
 
 ##@ Building
 
-build-all: build-frontend build-backend build-operator build-runner ## Build all container images
+build-all: build-frontend build-backend build-operator build-runner build-state-sync ## Build all container images
 
 build-frontend: ## Build frontend image
 	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Building frontend with $(CONTAINER_ENGINE)..."
@@ -145,6 +147,13 @@ build-runner: ## Build Claude Code runner image
 		-t $(RUNNER_IMAGE) -f claude-code-runner/Dockerfile .
 	@echo "$(COLOR_GREEN)✓$(COLOR_RESET) Runner built: $(RUNNER_IMAGE)"
 
+build-state-sync: ## Build state-sync image for S3 persistence
+	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Building state-sync with $(CONTAINER_ENGINE)..."
+	@echo "  Git: $(GIT_BRANCH)@$(GIT_COMMIT_SHORT)$(GIT_DIRTY)"
+	@cd components/runners/state-sync && $(CONTAINER_ENGINE) build $(PLATFORM_FLAG) $(BUILD_FLAGS) \
+		-t vteam_state_sync:latest .
+	@echo "$(COLOR_GREEN)✓$(COLOR_RESET) State-sync built: vteam_state_sync:latest"
+
 ##@ Git Hooks
 
 setup-hooks: ## Install git hooks for branch protection
@@ -164,13 +173,59 @@ registry-login: ## Login to container registry
 
 push-all: registry-login ## Push all images to registry
 	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Pushing images to $(REGISTRY)..."
-	@for image in $(FRONTEND_IMAGE) $(BACKEND_IMAGE) $(OPERATOR_IMAGE) $(RUNNER_IMAGE); do \
+	@for image in $(FRONTEND_IMAGE) $(BACKEND_IMAGE) $(OPERATOR_IMAGE) $(RUNNER_IMAGE) $(STATE_SYNC_IMAGE); do \
 		echo "  Tagging and pushing $$image..."; \
 		$(CONTAINER_ENGINE) tag $$image $(REGISTRY)/$$image && \
 		$(CONTAINER_ENGINE) push $(REGISTRY)/$$image; \
 	done
 	@echo "$(COLOR_GREEN)✓$(COLOR_RESET) All images pushed"
 
+##@ MinIO S3 Storage
+
+setup-minio: ## Set up MinIO and create initial bucket
+	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Setting up MinIO for S3 state storage..."
+	@./scripts/setup-minio.sh
+	@echo "$(COLOR_GREEN)✓$(COLOR_RESET) MinIO setup complete"
+
+minio-console: ## Open MinIO console (port-forward to localhost:9001)
+	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Opening MinIO console at http://localhost:9001"
+	@echo "  Login: admin / changeme123 (or your configured credentials)"
+	@kubectl port-forward svc/minio 9001:9001 -n $(NAMESPACE)
+
+minio-logs: ## View MinIO logs
+	@kubectl logs -f deployment/minio -n $(NAMESPACE)
+
+minio-status: ## Check MinIO status
+	@echo "$(COLOR_BOLD)MinIO Status$(COLOR_RESET)"
+	@kubectl get deployment,pod,svc,pvc -l app=minio -n $(NAMESPACE)
+
+##@ Observability
+
+deploy-observability: ## Deploy observability (OTel + OpenShift Prometheus)
+	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Deploying observability stack..."
+	@kubectl apply -k components/manifests/observability/
+	@echo "$(COLOR_GREEN)✓$(COLOR_RESET) Observability deployed (OTel + ServiceMonitor)"
+	@echo "  View metrics: OpenShift Console → Observe → Metrics"
+	@echo "  Optional Grafana: make add-grafana"
+
+add-grafana: ## Add Grafana on top of observability stack
+	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Adding Grafana..."
+	@kubectl apply -k components/manifests/observability/overlays/with-grafana/
+	@echo "$(COLOR_GREEN)✓$(COLOR_RESET) Grafana deployed"
+	@echo "  Create route: oc create route edge grafana --service=grafana -n $(NAMESPACE)"
+
+clean-observability: ## Remove observability components
+	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Removing observability..."
+	@kubectl delete -k components/manifests/observability/overlays/with-grafana/ 2>/dev/null || true
+	@kubectl delete -k components/manifests/observability/ 2>/dev/null || true
+	@echo "$(COLOR_GREEN)✓$(COLOR_RESET) Observability removed"
+
+grafana-dashboard: ## Open Grafana (create route first)
+	@echo "$(COLOR_BLUE)▶$(COLOR_RESET) Opening Grafana..."
+	@oc create route edge grafana --service=grafana -n $(NAMESPACE) 2>/dev/null || echo "Route already exists"
+	@echo "  URL: https://$$(oc get route grafana -n $(NAMESPACE) -o jsonpath='{.spec.host}')"
+	@echo "  Login: admin/admin"
+
 ##@ Local Development (Minikube)
 
 local-up: check-minikube check-kubectl ## Start local development environment (minikube)