feat: enhance pipeline cleanup and email notifications

- Add comprehensive resource cleanup in codebuild_deployment.py:
  - ECR repositories cleanup
  - CloudWatch log groups cleanup (Step Functions, Lambda, CodeBuild, AppSync)
  - CloudWatch Logs Resource Policy cleanup to prevent size exceeded errors
  - Enhanced stack deletion with --empty-buckets and --wait flags

- Add enhanced GitLab CI email notifications:
  - Capture actual CodeBuild logs using tracked pipeline execution ID
  - Generate detailed pipeline_summary.txt with deployment context
  - Save execution tracking files as artifacts

- Fix CloudWatch Logs Resource Policy size exceeded error:
  - Root cause was accumulated policy entries from deleted log groups
  - Enhanced cleanup now removes both log groups and policy entries
  - Prevents future Step Functions deployment failures

Resolves CloudWatch Logs Resource Policy size limit issues and improves
pipeline failure debugging with detailed CodeBuild logs in notifications.

Author: Taniya Mathur

.gitlab-ci.yml

@@ -136,3 +136,40 @@ integration_tests:
 
     # Run integration test deployment
     - python3 scripts/integration_test_deployment.py
+
+  after_script:
+    # Capture CodeBuild logs using the tracked execution ID
+    - |
+      echo "=== IDP Pipeline Results ===" > pipeline_summary.txt
+      echo "Branch: $CI_COMMIT_REF_NAME" >> pipeline_summary.txt
+      echo "Commit: $CI_COMMIT_SHA" >> pipeline_summary.txt
+      echo "Status: $CI_JOB_STATUS" >> pipeline_summary.txt
+      echo "" >> pipeline_summary.txt
+      
+      # Get CodeBuild logs using the exact execution ID from Python script
+      if [ -f "pipeline_execution_id.txt" ]; then
+        EXECUTION_ID=$(cat pipeline_execution_id.txt)
+        echo "Pipeline Execution: $EXECUTION_ID" >> pipeline_summary.txt
+        echo "" >> pipeline_summary.txt
+        
+        # Get CodeBuild ID from the pipeline execution
+        BUILD_ID=$(aws codepipeline list-action-executions --pipeline-name ${IDP_PIPELINE_NAME:-idp-sdlc-deploy-pipeline} --filter pipelineExecutionId=$EXECUTION_ID --query 'actionExecutionDetails[?actionName==`Deploy`].output.executionResult.externalExecutionId' --output text 2>/dev/null || echo "")
+        
+        if [ "$BUILD_ID" != "" ] && [ "$BUILD_ID" != "None" ]; then
+          echo "CodeBuild ID: $BUILD_ID" >> pipeline_summary.txt
+          echo "" >> pipeline_summary.txt
+          echo "=== CODEBUILD LOGS ===" >> pipeline_summary.txt
+          aws logs get-log-events --log-group-name "/aws/codebuild/${IDP_PIPELINE_NAME:-idp-sdlc-deploy-pipeline}" --log-stream-name "$BUILD_ID" --limit 100 --query 'events[].message' --output text 2>/dev/null >> pipeline_summary.txt || echo "Could not retrieve CodeBuild logs" >> pipeline_summary.txt
+        else
+          echo "Could not find CodeBuild execution" >> pipeline_summary.txt
+        fi
+      else
+        echo "No pipeline execution ID found" >> pipeline_summary.txt
+      fi
+
+  artifacts:
+    when: always
+    paths:
+      - pipeline_summary.txt
+      - pipeline_execution_id.txt
+    expire_in: 1 week

scripts/codebuild_deployment.py

@@ -60,8 +60,8 @@ def get_env_var(name, default=None):
 
 
 def generate_stack_prefix():
-    """Generate unique stack prefix with timestamp"""
-    timestamp = datetime.now().strftime("%m%d-%H%M")  # Shorter format: MMDD-HHMM
+    """Generate unique stack prefix with timestamp including seconds"""
+    timestamp = datetime.now().strftime("%m%d-%H%M%S")  # Format: MMDD-HHMMSS
     return f"idp-{timestamp}"
 
 
@@ -222,7 +222,35 @@ def cleanup_stack(stack_name, pattern_name):
     """Clean up a deployed stack"""
     print(f"[{pattern_name}] Cleaning up: {stack_name}")
     try:
-        run_command(f"idp-cli delete --stack-name {stack_name} --force", check=False)
+        # Check stack status first
+        result = run_command(f"aws cloudformation describe-stacks --stack-name {stack_name} --query 'Stacks[0].StackStatus' --output text", check=False)
+        stack_status = result.stdout.strip() if result.returncode == 0 else "NOT_FOUND"
+        
+        print(f"[{pattern_name}] Stack status: {stack_status}")
+        
+        # Delete the stack and wait for completion
+        print(f"[{pattern_name}] Attempting stack deletion...")
+        run_command(f"idp-cli delete --stack-name {stack_name} --force --empty-buckets --wait", check=False)
+        
+        # Always clean up orphaned resources after deletion attempt
+        print(f"[{pattern_name}] Cleaning up orphaned resources...")
+        
+        # ECR repositories
+        stack_name_lower = stack_name.lower()
+        run_command(f"aws ecr describe-repositories --query 'repositories[?contains(repositoryName, `{stack_name_lower}`)].repositoryName' --output text | xargs -r -n1 aws ecr delete-repository --repository-name --force", check=False)
+        
+        # CloudWatch log groups
+        run_command(f"aws logs describe-log-groups --log-group-name-prefix '/aws/vendedlogs/states/{stack_name}' --query 'logGroups[].logGroupName' --output text | xargs -r -n1 aws logs delete-log-group --log-group-name", check=False)
+        run_command(f"aws logs describe-log-groups --log-group-name-prefix '/aws/lambda/{stack_name}' --query 'logGroups[].logGroupName' --output text | xargs -r -n1 aws logs delete-log-group --log-group-name", check=False)
+        run_command(f"aws logs describe-log-groups --log-group-name-prefix '/{stack_name}' --query 'logGroups[].logGroupName' --output text | xargs -r -n1 aws logs delete-log-group --log-group-name", check=False)
+        run_command(f"aws logs describe-log-groups --log-group-name-prefix '/aws/codebuild/{stack_name}' --query 'logGroups[].logGroupName' --output text | xargs -r -n1 aws logs delete-log-group --log-group-name", check=False)
+        # AppSync logs (get API ID first, then delete log group)
+        run_command(f"aws appsync list-graphql-apis --query 'graphqlApis[?contains(name, `{stack_name}`)].apiId' --output text | xargs -r -I {{}} aws logs delete-log-group --log-group-name '/aws/appsync/apis/{{}}'", check=False)
+        run_command(f"aws logs describe-log-groups --query 'logGroups[?contains(logGroupName, `{stack_name}`)].logGroupName' --output text | xargs -r -n1 aws logs delete-log-group --log-group-name", check=False)
+        
+        # Clean up CloudWatch Logs Resource Policy entries for deleted log groups
+        run_command(f"aws logs describe-resource-policies --query 'resourcePolicies[0].policyName' --output text | xargs -r aws logs delete-resource-policy --policy-name", check=False)
+        
         print(f"[{pattern_name}] ✅ Cleanup completed")
     except Exception as e:
         print(f"[{pattern_name}] ⚠️ Cleanup failed: {e}")

scripts/integration_test_deployment.py

@@ -173,6 +173,11 @@ def monitor_pipeline(pipeline_name, version_id, max_wait=7200):
 
     if not execution_id:
         return False
+    
+    # Write execution ID to file for GitLab CI to use
+    with open("pipeline_execution_id.txt", "w") as f:
+        f.write(execution_id)
+    print(f"Pipeline execution ID written to file: {execution_id}")
 
     # Then monitor that specific execution
     return monitor_pipeline_execution(pipeline_name, execution_id, max_wait)