Merge branch 'feat/enhanced-pipeline-cleanup-and-notifications' into 'develop'

Enhanced Pipeline Cleanup and Notification Improvements

See merge request genaiic-reusable-assets/engagement-artifacts/genaiic-idp-accelerator!403

Author: rstrahan

.gitlab-ci.yml

@@ -136,3 +136,43 @@ integration_tests:
 
     # Run integration test deployment
     - python3 scripts/integration_test_deployment.py
+
+  after_script:
+    # Capture CodeBuild logs using the tracked execution ID
+    - |
+      echo "=== IDP Pipeline Results ===" > pipeline_summary.txt
+      echo "Branch: $CI_COMMIT_REF_NAME" >> pipeline_summary.txt
+      echo "Commit: $CI_COMMIT_SHA" >> pipeline_summary.txt
+      echo "Status: $CI_JOB_STATUS" >> pipeline_summary.txt
+      echo "" >> pipeline_summary.txt
+      
+      # Get CodeBuild logs using the exact execution ID from Python script
+      if [ -f "pipeline_execution_id.txt" ]; then
+        EXECUTION_ID=$(cat pipeline_execution_id.txt)
+        echo "Pipeline Execution: $EXECUTION_ID" >> pipeline_summary.txt
+        echo "" >> pipeline_summary.txt
+        
+        # Get CodeBuild ID from the pipeline execution
+        BUILD_ID=$(aws codepipeline list-action-executions --pipeline-name ${IDP_PIPELINE_NAME:-idp-sdlc-deploy-pipeline} --filter pipelineExecutionId=$EXECUTION_ID --query 'actionExecutionDetails[?actionName==`BuildAction`].output.executionResult.externalExecutionId' --output text 2>/dev/null || echo "")
+        
+        if [ "$BUILD_ID" != "" ] && [ "$BUILD_ID" != "None" ]; then
+          echo "CodeBuild ID: $BUILD_ID" >> pipeline_summary.txt
+          # Extract just the build ID part (after the colon)
+          LOG_STREAM_NAME="${BUILD_ID#*:}"
+          echo "Log Stream: $LOG_STREAM_NAME" >> pipeline_summary.txt
+          echo "" >> pipeline_summary.txt
+          echo "=== CODEBUILD LOGS ===" >> pipeline_summary.txt
+          aws logs get-log-events --log-group-name "/aws/codebuild/app-sdlc" --log-stream-name "$LOG_STREAM_NAME" --limit 100 --query 'events[].message' --output text 2>/dev/null >> pipeline_summary.txt || echo "Could not retrieve CodeBuild logs" >> pipeline_summary.txt
+        else
+          echo "Could not find CodeBuild execution" >> pipeline_summary.txt
+        fi
+      else
+        echo "No pipeline execution ID found" >> pipeline_summary.txt
+      fi
+
+  artifacts:
+    when: always
+    paths:
+      - pipeline_summary.txt
+      - pipeline_execution_id.txt
+    expire_in: 1 week

scripts/codebuild_deployment.py

@@ -60,8 +60,8 @@ def get_env_var(name, default=None):
 
 
 def generate_stack_prefix():
-    """Generate unique stack prefix with timestamp"""
-    timestamp = datetime.now().strftime("%m%d-%H%M")  # Shorter format: MMDD-HHMM
+    """Generate unique stack prefix with timestamp including seconds"""
+    timestamp = datetime.now().strftime("%m%d-%H%M%S")  # Format: MMDD-HHMMSS
     return f"idp-{timestamp}"
 
 
@@ -222,7 +222,39 @@ def cleanup_stack(stack_name, pattern_name):
     """Clean up a deployed stack"""
     print(f"[{pattern_name}] Cleaning up: {stack_name}")
     try:
-        run_command(f"idp-cli delete --stack-name {stack_name} --force", check=False)
+        # Check stack status first
+        result = run_command(f"aws cloudformation describe-stacks --stack-name {stack_name} --query 'Stacks[0].StackStatus' --output text", check=False)
+        stack_status = result.stdout.strip() if result.returncode == 0 else "NOT_FOUND"
+        
+        print(f"[{pattern_name}] Stack status: {stack_status}")
+        
+        # Delete the stack and wait for completion
+        print(f"[{pattern_name}] Attempting stack deletion...")
+        run_command(f"idp-cli delete --stack-name {stack_name} --force --empty-buckets --wait", check=False)
+        
+        # Always clean up orphaned resources after deletion attempt
+        print(f"[{pattern_name}] Cleaning up orphaned resources...")
+        
+        # Set AWS retry configuration to handle throttling
+        os.environ['AWS_MAX_ATTEMPTS'] = '10'
+        os.environ['AWS_RETRY_MODE'] = 'adaptive'
+        
+        # ECR repositories
+        stack_name_lower = stack_name.lower()
+        run_command(f"aws ecr describe-repositories --query 'repositories[?contains(repositoryName, `{stack_name_lower}`)].repositoryName' --output text | xargs -r -n1 aws ecr delete-repository --repository-name --force", check=False)
+        
+        # S3 buckets (empty and delete orphaned buckets)
+        run_command(f"aws s3api list-buckets --query 'Buckets[?contains(Name, `{stack_name}`)].Name' --output text | xargs -r -n1 -I {{}} sh -c 'aws s3 rm s3://{{}} --recursive && aws s3api delete-bucket --bucket {{}}'", check=False)
+        
+        # CloudWatch log groups (single comprehensive search)
+        run_command(f"aws logs describe-log-groups --query 'logGroups[?contains(logGroupName, `{stack_name}`)].logGroupName' --output text | xargs -r -n1 aws logs delete-log-group --log-group-name", check=False)
+        
+        # AppSync logs (requires separate handling due to random API IDs)
+        run_command(f"aws appsync list-graphql-apis --query 'graphqlApis[?contains(name, `{stack_name}`)].apiId' --output text | xargs -r -I {{}} aws logs delete-log-group --log-group-name '/aws/appsync/apis/{{}}'", check=False)
+        
+        # Clean up CloudWatch Logs Resource Policy (ignore errors if policy doesn't exist)
+        run_command(f"aws logs describe-resource-policies --query 'resourcePolicies[0].policyName' --output text | xargs -r aws logs delete-resource-policy --policy-name || true", check=False)
+        
         print(f"[{pattern_name}] ✅ Cleanup completed")
     except Exception as e:
         print(f"[{pattern_name}] ⚠️ Cleanup failed: {e}")

scripts/integration_test_deployment.py

@@ -173,6 +173,11 @@ def monitor_pipeline(pipeline_name, version_id, max_wait=7200):
 
     if not execution_id:
         return False
+    
+    # Write execution ID to file for GitLab CI to use
+    with open("pipeline_execution_id.txt", "w") as f:
+        f.write(execution_id)
+    print(f"Pipeline execution ID written to file: {execution_id}")
 
     # Then monitor that specific execution
     return monitor_pipeline_execution(pipeline_name, execution_id, max_wait)