Fix Pattern-2 ECR image availability race condition during deployment

Author: Bob Strahan

CHANGELOG.md

@@ -5,6 +5,17 @@ SPDX-License-Identifier: MIT-0
 
 ## [Unreleased]
 
+### Fixed
+
+- **Pattern-2 Intermittent HITLStatusUpdateFunction ECR Access Failure**
+  - Fixed intermittent "Lambda does not have permission to access the ECR image" (403) errors during Pattern-2 deployment
+  - **Root Cause**: Race condition where Lambda functions were created before ECR images were fully available and scannable
+  - **Solution**: Enhanced CodeBuild custom resource to verify ECR image availability before completing, including:
+    - Verification that all 9 required Lambda images exist in ECR repository
+    - Check that image scanning is complete (repository has `ScanOnPush: true`)
+    - Polling mechanism that waits for images to be fully ready before allowing Lambda creation
+  - **Impact**: Eliminates deployment failures and ensures reliable stack creation on first attempt
+
 ## [0.4.1]
 
 ### Changed

src/lambda/start_codebuild/index.py

@@ -60,6 +60,77 @@ def create_or_update(event, _):
     raise ValueError(f"invalid resource type: {resource_type}")
 
 
+def _verify_ecr_images_available(ecr_uri: str, image_version: str) -> bool:
+    """Verify all required Lambda images exist in ECR and are pullable.
+    
+    Args:
+        ecr_uri: ECR repository URI (e.g., 123456789012.dkr.ecr.us-east-1.amazonaws.com/repo-name)
+        image_version: Image version tag (e.g., "latest" or "0.3.19")
+    
+    Returns:
+        True if all images are available and scannable, False otherwise
+    """
+    try:
+        repository_name = ecr_uri.split("/")[-1]
+        
+        # List of all image tags used by Lambda functions in Pattern 2
+        required_images = [
+            f"ocr-function-{image_version}",
+            f"classification-function-{image_version}",
+            f"extraction-function-{image_version}",
+            f"assessment-function-{image_version}",
+            f"processresults-function-{image_version}",
+            f"hitl-wait-function-{image_version}",
+            f"hitl-status-update-function-{image_version}",
+            f"hitl-process-function-{image_version}",
+            f"summarization-function-{image_version}",
+        ]
+        
+        LOGGER.info(
+            "verifying %d images in repository %s with version %s",
+            len(required_images),
+            repository_name,
+            image_version,
+        )
+        
+        # Check each image
+        for image_tag in required_images:
+            try:
+                response = ECR_CLIENT.describe_images(
+                    repositoryName=repository_name,
+                    imageIds=[{"imageTag": image_tag}]
+                )
+                
+                images = response.get("imageDetails", [])
+                if not images:
+                    LOGGER.warning("image %s not found in ECR", image_tag)
+                    return False
+                
+                # Check if image scan is complete (repository has ScanOnPush enabled)
+                image = images[0]
+                scan_status = image.get("imageScanStatus", {}).get("status")
+                
+                if scan_status == "IN_PROGRESS":
+                    LOGGER.info("image %s scan still in progress", image_tag)
+                    return False
+                
+                LOGGER.debug("image %s verified (scan status: %s)", image_tag, scan_status)
+                    
+            except ClientError as error:
+                if error.response["Error"]["Code"] == "ImageNotFoundException":
+                    LOGGER.warning("image %s not found: %s", image_tag, error)
+                    return False
+                LOGGER.error("error checking image %s: %s", image_tag, error)
+                raise
+        
+        LOGGER.info("all %d required images are available in ECR", len(required_images))
+        return True
+        
+    except Exception as exception:  # pylint: disable=broad-except
+        LOGGER.error("error verifying ECR images: %s", exception)
+        return False
+
+
 @HELPER.poll_create
 @HELPER.poll_update
 def poll_create_or_update(event, _):
@@ -82,7 +153,28 @@ def poll_create_or_update(event, _):
             LOGGER.info("build status: [%s]", build_status)
 
             if build_status == "SUCCEEDED":
-                LOGGER.info("returning True")
+                # Verify ECR images are available before returning success
+                # This prevents Lambda functions from being created before images are pullable
+                env_vars = build.get("environment", {}).get("environmentVariables", [])
+                
+                # Extract ECR URI and image version from build environment
+                ecr_uri = next((v["value"] for v in env_vars if v["name"] == "ECR_URI"), None)
+                image_version = next((v["value"] for v in env_vars if v["name"] == "IMAGE_VERSION"), None)
+                
+                if ecr_uri and image_version:
+                    LOGGER.info("verifying ECR images are available and pullable...")
+                    if _verify_ecr_images_available(ecr_uri, image_version):
+                        LOGGER.info("ECR image verification complete - returning True")
+                        return True
+                    
+                    LOGGER.info("ECR images not yet available - returning None to poll again")
+                    return None
+                
+                # Fallback: if we can't extract variables, proceed without verification
+                LOGGER.warning(
+                    "could not extract ECR_URI or IMAGE_VERSION from build environment, "
+                    "proceeding without ECR verification"
+                )
                 return True
 
             if build_status == "IN_PROGRESS":