feat: enhance pipeline cleanup and notifications

Add comprehensive resource cleanup with throttling fixes and improved CodeBuild log capture in GitLab CI notifications

Author: Taniya Mathur

.gitlab-ci.yml

@@ -153,13 +153,16 @@ integration_tests:
         echo "" >> pipeline_summary.txt
         
         # Get CodeBuild ID from the pipeline execution
-        BUILD_ID=$(aws codepipeline list-action-executions --pipeline-name ${IDP_PIPELINE_NAME:-idp-sdlc-deploy-pipeline} --filter pipelineExecutionId=$EXECUTION_ID --query 'actionExecutionDetails[?actionName==`Deploy`].output.executionResult.externalExecutionId' --output text 2>/dev/null || echo "")
+        BUILD_ID=$(aws codepipeline list-action-executions --pipeline-name ${IDP_PIPELINE_NAME:-idp-sdlc-deploy-pipeline} --filter pipelineExecutionId=$EXECUTION_ID --query 'actionExecutionDetails[?actionName==`BuildAction`].output.executionResult.externalExecutionId' --output text 2>/dev/null || echo "")
         
         if [ "$BUILD_ID" != "" ] && [ "$BUILD_ID" != "None" ]; then
           echo "CodeBuild ID: $BUILD_ID" >> pipeline_summary.txt
+          # Extract just the build ID part (after the colon)
+          LOG_STREAM_NAME="${BUILD_ID#*:}"
+          echo "Log Stream: $LOG_STREAM_NAME" >> pipeline_summary.txt
           echo "" >> pipeline_summary.txt
           echo "=== CODEBUILD LOGS ===" >> pipeline_summary.txt
-          aws logs get-log-events --log-group-name "/aws/codebuild/${IDP_PIPELINE_NAME:-idp-sdlc-deploy-pipeline}" --log-stream-name "$BUILD_ID" --limit 100 --query 'events[].message' --output text 2>/dev/null >> pipeline_summary.txt || echo "Could not retrieve CodeBuild logs" >> pipeline_summary.txt
+          aws logs get-log-events --log-group-name "/aws/codebuild/app-sdlc" --log-stream-name "$LOG_STREAM_NAME" --limit 100 --query 'events[].message' --output text 2>/dev/null >> pipeline_summary.txt || echo "Could not retrieve CodeBuild logs" >> pipeline_summary.txt
         else
           echo "Could not find CodeBuild execution" >> pipeline_summary.txt
         fi

scripts/codebuild_deployment.py

@@ -235,21 +235,25 @@ def cleanup_stack(stack_name, pattern_name):
         # Always clean up orphaned resources after deletion attempt
         print(f"[{pattern_name}] Cleaning up orphaned resources...")
 
+        # Set AWS retry configuration to handle throttling
+        os.environ['AWS_MAX_ATTEMPTS'] = '10'
+        os.environ['AWS_RETRY_MODE'] = 'adaptive'
+        
         # ECR repositories
         stack_name_lower = stack_name.lower()
         run_command(f"aws ecr describe-repositories --query 'repositories[?contains(repositoryName, `{stack_name_lower}`)].repositoryName' --output text | xargs -r -n1 aws ecr delete-repository --repository-name --force", check=False)
 
-        # CloudWatch log groups
-        run_command(f"aws logs describe-log-groups --log-group-name-prefix '/aws/vendedlogs/states/{stack_name}' --query 'logGroups[].logGroupName' --output text | xargs -r -n1 aws logs delete-log-group --log-group-name", check=False)
-        run_command(f"aws logs describe-log-groups --log-group-name-prefix '/aws/lambda/{stack_name}' --query 'logGroups[].logGroupName' --output text | xargs -r -n1 aws logs delete-log-group --log-group-name", check=False)
-        run_command(f"aws logs describe-log-groups --log-group-name-prefix '/{stack_name}' --query 'logGroups[].logGroupName' --output text | xargs -r -n1 aws logs delete-log-group --log-group-name", check=False)
-        run_command(f"aws logs describe-log-groups --log-group-name-prefix '/aws/codebuild/{stack_name}' --query 'logGroups[].logGroupName' --output text | xargs -r -n1 aws logs delete-log-group --log-group-name", check=False)
-        # AppSync logs (get API ID first, then delete log group)
-        run_command(f"aws appsync list-graphql-apis --query 'graphqlApis[?contains(name, `{stack_name}`)].apiId' --output text | xargs -r -I {{}} aws logs delete-log-group --log-group-name '/aws/appsync/apis/{{}}'", check=False)
+        # S3 buckets (empty and delete orphaned buckets)
+        run_command(f"aws s3api list-buckets --query 'Buckets[?contains(Name, `{stack_name}`)].Name' --output text | xargs -r -n1 -I {{}} sh -c 'aws s3 rm s3://{{}} --recursive && aws s3api delete-bucket --bucket {{}}'", check=False)
+        
+        # CloudWatch log groups (single comprehensive search)
         run_command(f"aws logs describe-log-groups --query 'logGroups[?contains(logGroupName, `{stack_name}`)].logGroupName' --output text | xargs -r -n1 aws logs delete-log-group --log-group-name", check=False)
 
-        # Clean up CloudWatch Logs Resource Policy entries for deleted log groups
-        run_command(f"aws logs describe-resource-policies --query 'resourcePolicies[0].policyName' --output text | xargs -r aws logs delete-resource-policy --policy-name", check=False)
+        # AppSync logs (requires separate handling due to random API IDs)
+        run_command(f"aws appsync list-graphql-apis --query 'graphqlApis[?contains(name, `{stack_name}`)].apiId' --output text | xargs -r -I {{}} aws logs delete-log-group --log-group-name '/aws/appsync/apis/{{}}'", check=False)
+        
+        # Clean up CloudWatch Logs Resource Policy (ignore errors if policy doesn't exist)
+        run_command(f"aws logs describe-resource-policies --query 'resourcePolicies[0].policyName' --output text | xargs -r aws logs delete-resource-policy --policy-name || true", check=False)
 
         print(f"[{pattern_name}] ✅ Cleanup completed")
     except Exception as e: