Merge branch 'fix/image-race-condition' into 'develop'

Fix/image race condition

See merge request genaiic-reusable-assets/engagement-artifacts/genaiic-idp-accelerator!410

Author: rstrahan

CHANGELOG.md

@@ -5,6 +5,18 @@ SPDX-License-Identifier: MIT-0
 
 ## [Unreleased]
 
+### Fixed
+
+- **Pattern-2 Intermittent HITLStatusUpdateFunction ECR Access Failure**
+  - Fixed intermittent "Lambda does not have permission to access the ECR image" (403) errors during Pattern-2 deployment
+  - **Root Cause**: Race condition where Lambda functions were created before ECR images were fully available and scannable
+  - **Solution**: Enhanced CodeBuild custom resource to verify ECR image availability before completing, including:
+    - Verification that all required Lambda images exist in ECR repository
+    - Check that image scanning is complete (repository has `ScanOnPush: true`)
+  - **New Parameter**: Added `EnablePattern2ECRImageScanning` parameter (default: true) to allow users to disable ECR vulnerability scanning if experiencing deployment issues
+    - Recommended: Keep enabled (true) for production to maintain security posture
+    - Optional: Disable (false) only as temporary workaround for deployment reliability
+
 ## [0.4.1]
 
 ### Changed

patterns/pattern-2/template.yaml

@@ -110,6 +110,12 @@ Parameters:
     AllowedValues: ['true', 'false']
     Description: Enable X-Ray tracing
 
+  EnableECRImageScanning:
+    Type: String
+    Default: 'true'
+    AllowedValues: ['true', 'false']
+    Description: Enable automatic vulnerability scanning for Lambda container images in ECR
+
   PermissionsBoundaryArn:
     Type: String
     Default: ""
@@ -147,6 +153,7 @@ Conditions:
     !Not [!Equals [!Ref CustomExtractionModelARN, ""]]
   HasPermissionsBoundary: !Not [!Equals [!Ref PermissionsBoundaryArn, ""]]
   HasAppSyncApi: !Not [!Equals [!Ref AppSyncApiArn, ""]]
+  IsECRImageScanningEnabled: !Equals [!Ref EnableECRImageScanning, "true"]
 
 Resources:
   Pattern2DockerBuildRole:
@@ -202,6 +209,7 @@ Resources:
                 Effect: Allow
                 Action:
                   - ecr:DescribeImageScanFindings
+                  - ecr:DescribeImages
                   - ecr:StartImageScan
               # Required for Amazon Inspector Enhanced Scanning
               # https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning-enhanced-iam.html
@@ -223,7 +231,7 @@ Resources:
     # checkov:skip=CKV_AWS_51: "Mutable tags allowed for workflow flexibility and version updates."
     Properties:
       ImageScanningConfiguration:
-        ScanOnPush: true
+        ScanOnPush: !If [IsECRImageScanningEnabled, true, false]
       EncryptionConfiguration:
         EncryptionType: KMS
         KmsKey: !Ref CustomerManagedEncryptionKeyArn

src/lambda/start_codebuild/index.py

@@ -60,6 +60,77 @@ def create_or_update(event, _):
     raise ValueError(f"invalid resource type: {resource_type}")
 
 
+def _verify_ecr_images_available(ecr_uri: str, image_version: str) -> bool:
+    """Verify all required Lambda images exist in ECR and are pullable.
+    
+    Args:
+        ecr_uri: ECR repository URI (e.g., 123456789012.dkr.ecr.us-east-1.amazonaws.com/repo-name)
+        image_version: Image version tag (e.g., "latest" or "0.3.19")
+    
+    Returns:
+        True if all images are available and scannable, False otherwise
+    """
+    try:
+        repository_name = ecr_uri.split("/")[-1]
+        
+        # List of all image tags used by Lambda functions in Pattern 2
+        required_images = [
+            f"ocr-function-{image_version}",
+            f"classification-function-{image_version}",
+            f"extraction-function-{image_version}",
+            f"assessment-function-{image_version}",
+            f"processresults-function-{image_version}",
+            f"hitl-wait-function-{image_version}",
+            f"hitl-status-update-function-{image_version}",
+            f"hitl-process-function-{image_version}",
+            f"summarization-function-{image_version}",
+        ]
+        
+        LOGGER.info(
+            "verifying %d images in repository %s with version %s",
+            len(required_images),
+            repository_name,
+            image_version,
+        )
+        
+        # Check each image
+        for image_tag in required_images:
+            try:
+                response = ECR_CLIENT.describe_images(
+                    repositoryName=repository_name,
+                    imageIds=[{"imageTag": image_tag}]
+                )
+                
+                images = response.get("imageDetails", [])
+                if not images:
+                    LOGGER.warning("image %s not found in ECR", image_tag)
+                    return False
+                
+                # Check if image scan is complete (repository has ScanOnPush enabled)
+                image = images[0]
+                scan_status = image.get("imageScanStatus", {}).get("status")
+                
+                if scan_status == "IN_PROGRESS":
+                    LOGGER.info("image %s scan still in progress", image_tag)
+                    return False
+                
+                LOGGER.info("image %s verified (scan status: %s)", image_tag, scan_status)
+                    
+            except ClientError as error:
+                if error.response["Error"]["Code"] == "ImageNotFoundException":
+                    LOGGER.warning("image %s not found: %s", image_tag, error)
+                    return False
+                LOGGER.error("error checking image %s: %s", image_tag, error)
+                raise
+        
+        LOGGER.info("all %d required images are available in ECR", len(required_images))
+        return True
+        
+    except Exception as exception:  # pylint: disable=broad-except
+        LOGGER.error("error verifying ECR images: %s", exception)
+        return False
+
+
 @HELPER.poll_create
 @HELPER.poll_update
 def poll_create_or_update(event, _):
@@ -82,7 +153,28 @@ def poll_create_or_update(event, _):
             LOGGER.info("build status: [%s]", build_status)
 
             if build_status == "SUCCEEDED":
-                LOGGER.info("returning True")
+                # Verify ECR images are available before returning success
+                # This prevents Lambda functions from being created before images are pullable
+                env_vars = build.get("environment", {}).get("environmentVariables", [])
+                
+                # Extract ECR URI and image version from build environment
+                ecr_uri = next((v["value"] for v in env_vars if v["name"] == "ECR_URI"), None)
+                image_version = next((v["value"] for v in env_vars if v["name"] == "IMAGE_VERSION"), None)
+                
+                if ecr_uri and image_version:
+                    LOGGER.info("verifying ECR images are available and pullable...")
+                    if _verify_ecr_images_available(ecr_uri, image_version):
+                        LOGGER.info("ECR image verification complete - returning True")
+                        return True
+                    
+                    LOGGER.info("ECR images not yet available - returning None to poll again")
+                    return None
+                
+                # Fallback: if we can't extract variables, proceed without verification
+                LOGGER.warning(
+                    "could not extract ECR_URI or IMAGE_VERSION from build environment, "
+                    "proceeding without ECR verification"
+                )
                 return True
 
             if build_status == "IN_PROGRESS":
@@ -150,7 +242,7 @@ def _delete_all_ecr_images(repository_name: str) -> None:
         if not image_ids:
             continue
         images_to_delete.extend(image_ids)
-        LOGGER.debug(
+        LOGGER.info(
             "queued %s images for deletion from repository %s",
             len(image_ids),
             repository_name,
@@ -162,7 +254,7 @@ def _delete_all_ecr_images(repository_name: str) -> None:
 
     for chunk_start in range(0, len(images_to_delete), 100):
         chunk = images_to_delete[chunk_start : chunk_start + 100]
-        LOGGER.debug(
+        LOGGER.info(
             "deleting %s images from repository %s",
             len(chunk),
             repository_name,

template.yaml

@@ -328,6 +328,17 @@ Parameters:
     AllowedPattern: "^(|arn:aws[a-z-]*:iam::[0-9]{12}:policy/.+)$"
     ConstraintDescription: Must be empty or a valid IAM policy ARN
 
+  EnablePattern2ECRImageScanning:
+    Type: String
+    Default: "true"
+    AllowedValues:
+      - "true"
+      - "false"
+    Description: >-
+      Enable automatic vulnerability scanning for Pattern-2 Lambda container images in ECR. 
+      Disabling may improve deployment reliability but reduces security posture. 
+      Recommended: true for production, false only if experiencing deployment issues.
+
   # Logging configuration
   LogLevel:
     Type: String
@@ -960,6 +971,7 @@ Resources:
         ArtifactPrefix: "<ARTIFACT_PREFIX_TOKEN>"
         Pattern2SourceZipfile: "<PATTERN2_SOURCE_ZIPFILE_TOKEN>"
         EvaluationFunctionArn: !GetAtt EvaluationFunction.Arn
+        EnableECRImageScanning: !Ref EnablePattern2ECRImageScanning
 
   PATTERN3STACK:
     DependsOn: