Merge branch 'feature/pipeline-improvement' into 'develop'

## Pipeline Improvements

See merge request genaiic-reusable-assets/engagement-artifacts/genaiic-idp-accelerator!408

Author: rstrahan

.gitlab-ci.yml

@@ -95,6 +95,8 @@ deployment_validation:
 integration_tests:
   stage: integration_tests
   timeout: 2h
+  variables:
+    IDP_ADMIN_EMAIL: ${GITLAB_USER_EMAIL}
   # variables:
   #   # In order to run tests in another account, add a AWS_CREDS_TARGET_ROLE variable to the Gitlab pipeline variables.
   #   AWS_CREDS_TARGET_ROLE: ${AWS_CREDS_TARGET_ROLE}
@@ -138,41 +140,32 @@ integration_tests:
     - python3 scripts/integration_test_deployment.py
 
   after_script:
-    # Capture CodeBuild logs using the tracked execution ID
+    # Display CodeBuild logs directly in GitLab runner console
     - |
-      echo "=== IDP Pipeline Results ===" > pipeline_summary.txt
-      echo "Branch: $CI_COMMIT_REF_NAME" >> pipeline_summary.txt
-      echo "Commit: $CI_COMMIT_SHA" >> pipeline_summary.txt
-      echo "Status: $CI_JOB_STATUS" >> pipeline_summary.txt
-      echo "" >> pipeline_summary.txt
-      
-      # Get CodeBuild logs using the exact execution ID from Python script
       if [ -f "pipeline_execution_id.txt" ]; then
         EXECUTION_ID=$(cat pipeline_execution_id.txt)
-        echo "Pipeline Execution: $EXECUTION_ID" >> pipeline_summary.txt
-        echo "" >> pipeline_summary.txt
+        echo "Pipeline Execution: $EXECUTION_ID"
         
         # Get CodeBuild ID from the pipeline execution
         BUILD_ID=$(aws codepipeline list-action-executions --pipeline-name ${IDP_PIPELINE_NAME:-idp-sdlc-deploy-pipeline} --filter pipelineExecutionId=$EXECUTION_ID --query 'actionExecutionDetails[?actionName==`BuildAction`].output.executionResult.externalExecutionId' --output text 2>/dev/null || echo "")
         
         if [ "$BUILD_ID" != "" ] && [ "$BUILD_ID" != "None" ]; then
-          echo "CodeBuild ID: $BUILD_ID" >> pipeline_summary.txt
+          echo "CodeBuild ID: $BUILD_ID"
           # Extract just the build ID part (after the colon)
           LOG_STREAM_NAME="${BUILD_ID#*:}"
-          echo "Log Stream: $LOG_STREAM_NAME" >> pipeline_summary.txt
-          echo "" >> pipeline_summary.txt
-          echo "=== CODEBUILD LOGS ===" >> pipeline_summary.txt
-          aws logs get-log-events --log-group-name "/aws/codebuild/app-sdlc" --log-stream-name "$LOG_STREAM_NAME" --limit 100 --query 'events[].message' --output text 2>/dev/null >> pipeline_summary.txt || echo "Could not retrieve CodeBuild logs" >> pipeline_summary.txt
+          echo "Log Stream: $LOG_STREAM_NAME"
+          echo ""
+          echo "=== CODEBUILD LOGS ==="
+          aws logs get-log-events --log-group-name "/aws/codebuild/app-sdlc" --log-stream-name "$LOG_STREAM_NAME" --start-from-head --query 'events[].message' --output text 2>/dev/null || echo "Could not retrieve CodeBuild logs"
         else
-          echo "Could not find CodeBuild execution" >> pipeline_summary.txt
+          echo "Could not find CodeBuild execution"
         fi
       else
-        echo "No pipeline execution ID found" >> pipeline_summary.txt
+        echo "No pipeline execution ID found"
       fi
 
   artifacts:
     when: always
     paths:
-      - pipeline_summary.txt
       - pipeline_execution_id.txt
     expire_in: 1 week

scripts/codebuild_deployment.py

@@ -97,7 +97,7 @@ def publish_templates():
         sys.exit(1)
 
 
-def deploy_and_test_pattern(stack_prefix, pattern_config, admin_email, template_url):
+def deploy_test_and_cleanup_pattern(stack_prefix, pattern_config, admin_email, template_url):
     """Deploy and test a specific IDP pattern"""
     pattern_name = pattern_config["name"]
     pattern_id = pattern_config["id"]
@@ -195,28 +195,35 @@ def deploy_and_test_pattern(stack_prefix, pattern_config, admin_email, template_
             print(
                 f"[{pattern_name}] ✅ Found expected verification string: '{verify_string}'"
             )
-            return {
+            
+            success_result = {
                 "stack_name": stack_name,
                 "pattern_name": pattern_name,
                 "success": True,
             }
 
         except Exception as e:
             print(f"[{pattern_name}] ❌ Failed to validate result content: {e}")
-            return {
+            success_result = {
                 "stack_name": stack_name,
                 "pattern_name": pattern_name,
                 "success": False,
             }
 
     except Exception as e:
         print(f"[{pattern_name}] ❌ Testing failed: {e}")
-        return {
+        success_result = {
             "stack_name": stack_name,
             "pattern_name": pattern_name,
             "success": False,
         }
 
+    # Always cleanup the stack regardless of success/failure
+    finally:
+        cleanup_stack(stack_name, pattern_name)
+    
+    return success_result
+
 
 def cleanup_stack(stack_name, pattern_name):
     """Clean up a deployed stack"""
@@ -274,16 +281,15 @@ def main():
     # Step 1: Publish templates to S3
     template_url = publish_templates()
 
-    deployed_stacks = []
     all_success = True
 
-    # Step 2: Deploy and test patterns concurrently
+    # Step 2: Deploy, test, and cleanup patterns concurrently
     print("🚀 Starting concurrent deployment of all patterns...")
     with ThreadPoolExecutor(max_workers=len(DEPLOY_PATTERNS)) as executor:
         # Submit all deployment tasks
         future_to_pattern = {
             executor.submit(
-                deploy_and_test_pattern,
+                deploy_test_and_cleanup_pattern,
                 stack_prefix,
                 pattern_config,
                 admin_email,
@@ -292,33 +298,22 @@ def main():
             for pattern_config in DEPLOY_PATTERNS
         }
 
-        # Collect results as they complete
+        # Collect results as they complete (cleanup happens within each pattern)
         for future in as_completed(future_to_pattern):
             pattern_config = future_to_pattern[future]
             try:
                 result = future.result()
-                deployed_stacks.append(result)
                 if not result["success"]:
                     all_success = False
                     print(f"[{pattern_config['name']}] ❌ Failed")
                 else:
                     print(f"[{pattern_config['name']}] ✅ Success")
+                    
             except Exception as e:
                 print(f"[{pattern_config['name']}] ❌ Exception: {e}")
                 all_success = False
 
-    # Step 3: Cleanup all stacks concurrently
-    print("🧹 Starting concurrent cleanup of all stacks...")
-    with ThreadPoolExecutor(max_workers=len(deployed_stacks)) as executor:
-        cleanup_futures = [
-            executor.submit(cleanup_stack, result["stack_name"], result["pattern_name"])
-            for result in deployed_stacks
-        ]
-
-        # Wait for all cleanups to complete
-        for future in as_completed(cleanup_futures):
-            future.result()  # Wait for completion
-
+    # Check final status after all cleanups are done
     if all_success:
         print("🎉 All pattern deployments completed successfully!")
         sys.exit(0)

scripts/integration_test_deployment.py

@@ -79,10 +79,20 @@ def upload_to_s3(bucket_name):
     s3_client = boto3.client("s3")
 
     try:
+        # Get GitLab user email to pass to CodeBuild
+        gitlab_user_email = os.environ.get("GITLAB_USER_EMAIL", "")
+        
+        # Add metadata to pass email to CodeBuild
+        metadata = {}
+        if gitlab_user_email:
+            metadata["gitlab-user-email"] = gitlab_user_email
+            print(f"Adding GitLab user email to metadata: {gitlab_user_email}")
+
         response = s3_client.put_object(
             Bucket=bucket_name,
             Key="deploy/code.zip",
             Body=open("./dist/code.zip", "rb"),
+            Metadata=metadata,
         )
         version_id = response.get("VersionId", "unknown")
         print(f"✅ Uploaded with version ID: {version_id}")

scripts/sdlc/cfn/codepipeline-s3.yml

@@ -159,42 +159,13 @@ Resources:
               commands:
                 - n 22 && node --version || { echo "Node setup failed"; exit 1; }
                 - npm install -g aws-cdk || { echo "CDK installation failed"; exit 1; }
-                # Check which deployment method to use
-                - |
-                  if [ -d "./scripts/sdlc/idp-cli" ]; then
-                    echo "Using legacy poetry-based deployment"
-                    curl -sSL https://install.python-poetry.org | python3 - || { echo "Poetry installation failed"; exit 1; }
-                    export PATH="/root/.local/bin:$PATH"
-                    cd ./scripts/sdlc/idp-cli
-                    poetry install || { echo "Poetry dependencies installation failed"; exit 1; }
-                    cd ../../..
-                  else
-                    echo "Using new idp-cli deployment"
-                    cd idp_cli && pip install -e . && cd .. || { echo "IDP CLI installation failed"; exit 1; }
-                  fi
+                - export IDP_ADMIN_EMAIL=$(aws s3api head-object --bucket idp-sdlc-sourcecode-${AWS_ACCOUNT_ID:-020432867916}-${AWS_DEFAULT_REGION:-us-east-1} --key deploy/code.zip --query 'Metadata."gitlab-user-email"' --output text 2>/dev/null || echo "")
+                - cd idp_cli && pip install -e . && cd .. || { echo "IDP CLI installation failed"; exit 1; }
             build:
               commands:
-                - |
-                  if [ -d "./scripts/sdlc/idp-cli" ]; then
-                    echo "Running legacy poetry-based build"
-                    cd ./scripts/sdlc/idp-cli
-                    export IDP_CFN_PREFIX=$(make cfn-prefix) || { echo "CFN prefix generation failed"; exit 1; }
-                    make install -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX
-                    make smoketest -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX
-                  else
-                    echo "Running codebuild deployment script"
-                    python3 scripts/codebuild_deployment.py
-                  fi
+                - python3 scripts/codebuild_deployment.py
               finally:
-                - |
-                  if [ -d "./scripts/sdlc/idp-cli" ]; then
-                    echo "Running legacy cleanup"
-                    cd ./scripts/sdlc/idp-cli
-                    make uninstall -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX || echo "Cleanup failed but continuing"
-                    make -n cli-smoketest >/dev/null 2>&1 && make cli-smoketest -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX || echo "CLI smoketest target not found, skipping"
-                  else
-                    echo "Cleanup handled by codebuild deployment script"
-                  fi
+                - echo "Cleanup handled by codebuild deployment script"
 
   DeploymentPipeline:
     Type: 'AWS::CodePipeline::Pipeline'

scripts/sdlc/cfn/credential-vendor.yml

@@ -82,6 +82,7 @@ Resources:
                   - codepipeline:GetPipelineExecution
                   - codepipeline:ListPipelineExecutions
                   - codepipeline:ListPipelines
+                  - codepipeline:ListActionExecutions
                 Resource: !Sub "arn:aws:codepipeline:*:${AWS::AccountId}:*"
         - PolicyName: CodeBuildAccessPolicy
           PolicyDocument: