refactor CopyConfigurationFiles to avoid ListBucket permission

Author: Bob Strahan

README.md

@@ -10,7 +10,7 @@ SPDX-License-Identifier: MIT-0
 - [Architecture Overview](#architecture-overview)
 - [Quick Start](#quick-start)
   - [Processing Your First Document](#processing-your-first-document)
-  - [Updating an Existing Deployment](#updating-an-existing-deployment)
+- [Updating an Existing Deployment](#updating-an-existing-deployment)
 - [Detailed Documentation](#detailed-documentation)
   - [Core Documentation](#core-documentation)
   - [Processing Patterns](#processing-patterns)
@@ -84,7 +84,7 @@ After deployment, you can quickly process a document and view results:
 
 See the [Deployment Guide](./docs/deployment.md#testing-the-solution) for more detailed testing instructions.
 
-### Updating an Existing Deployment
+## Updating an Existing Deployment
 
 To update an existing GenAIIDP stack to a new version:
 

publish.sh

@@ -289,6 +289,27 @@ function build_and_package_template() {
   fi
 }
 
+# Generate list of configuration files for explicit copying
+function generate_config_file_list() {
+  local config_dir="config_library"
+  local file_list=""
+  
+  # Find all files in config_library, excluding .checksum files
+  while IFS= read -r -d '' file; do
+    # Get relative path from config_library directory
+    relative_path="${file#$config_dir/}"
+    # Skip .checksum files
+    if [[ "$relative_path" != ".checksum" && "$relative_path" != *"/.checksum" ]]; then
+      if [[ -n "$file_list" ]]; then
+        file_list="$file_list,"
+      fi
+      file_list="$file_list\"$relative_path\""
+    fi
+  done < <(find "$config_dir" -type f -print0)
+  
+  echo "[$file_list]"
+}
+
 # Upload configuration library to S3
 function upload_config_library() {
   local config_dir="config_library"
@@ -386,8 +407,8 @@ function build_main_template() {
   local BUILD_DATE_TIME=$(date -u +"%Y-%m-%d %H:%M:%S")
   local CONFIG_LIBRARY_HASH=$(calculate_hash "config_library")
 
-  # Define configuration S3 base path
-  local CONFIG_BASE_PATH="s3://${BUCKET}/${PREFIX_AND_VERSION}/config_library"
+  # Generate configuration file list for explicit copying
+  local CONFIG_FILE_LIST=$(generate_config_file_list)
 
   echo "Inline edit main template to replace:"
   echo "   <VERSION> with: $VERSION"
@@ -397,8 +418,8 @@ function build_main_template() {
   echo "   <ARTIFACT_PREFIX_TOKEN> with prefix: $PREFIX_AND_VERSION"
   echo "   <WEBUI_ZIPFILE_TOKEN> with filename: $webui_zipfile"
   echo "   <HASH_TOKEN> with: $HASH"
-  echo "   <CONFIG_BASE_PATH_TOKEN> with: $CONFIG_BASE_PATH"
   echo "   <CONFIG_LIBRARY_HASH_TOKEN> with: $CONFIG_LIBRARY_HASH"
+  echo "   <CONFIG_FILES_LIST_TOKEN> with file list: $CONFIG_FILE_LIST"
 
   # Use a more reliable approach for multiple sed replacements
   sed -e "s|<VERSION>|$VERSION|g" \
@@ -408,8 +429,8 @@ function build_main_template() {
       -e "s|<ARTIFACT_PREFIX_TOKEN>|$PREFIX_AND_VERSION|g" \
       -e "s|<WEBUI_ZIPFILE_TOKEN>|$webui_zipfile|g" \
       -e "s|<HASH_TOKEN>|$HASH|g" \
-      -e "s|<CONFIG_BASE_PATH_TOKEN>|$CONFIG_BASE_PATH|g" \
       -e "s|<CONFIG_LIBRARY_HASH_TOKEN>|$CONFIG_LIBRARY_HASH|g" \
+      -e "s|<CONFIG_FILES_LIST_TOKEN>|$CONFIG_FILE_LIST|g" \
       .aws-sam/packaged.yaml > .aws-sam/${MAIN_TEMPLATE}
 
   # Upload and validate main template
@@ -500,4 +521,4 @@ set_public_acls
 print_outputs "$TEMPLATE_URL"
 
 echo "Done"
-exit 0
+exit 0

template.yaml

@@ -543,40 +543,41 @@ Resources:
                 target_bucket = event['ResourceProperties']['TargetBucket']
                 target_prefix = event['ResourceProperties'].get('TargetPrefix', '')
                 
+                file_list = event['ResourceProperties'].get('FileList', [])
+                
                 s3_client = boto3.client('s3')
                 
                 if event['RequestType'] == 'Create' or event['RequestType'] == 'Update':
-                    # List all objects in the source bucket with the given prefix
-                    paginator = s3_client.get_paginator('list_objects_v2')
-                    pages = paginator.paginate(Bucket=source_bucket, Prefix=source_prefix)
-                    
+                    # Copy files explicitly from the provided list
                     copied_count = 0
-                    for page in pages:
-                        if 'Contents' in page:
-                            for obj in page['Contents']:
-                                source_key = obj['Key']
-                                # Remove the source prefix and add to target bucket with target prefix
-                                relative_key = source_key[len(source_prefix):].lstrip('/')
-                                
-                                # Skip if this is just the prefix directory
-                                if not relative_key:
-                                    continue
-                                
-                                # Construct target key with optional target prefix
-                                if target_prefix:
-                                    target_key = f"{target_prefix}/{relative_key}"
-                                else:
-                                    target_key = relative_key
-                                
-                                logger.info(f"Copying {source_bucket}/{source_key} to {target_bucket}/{target_key}")
-                                
-                                copy_source = {'Bucket': source_bucket, 'Key': source_key}
-                                s3_client.copy_object(
-                                    CopySource=copy_source,
-                                    Bucket=target_bucket,
-                                    Key=target_key
-                                )
-                                copied_count += 1
+                    
+                    for relative_file_path in file_list:
+                        # Skip empty entries
+                        if not relative_file_path.strip():
+                            continue
+                            
+                        # Construct source key
+                        source_key = f"{source_prefix}/{relative_file_path}"
+                        
+                        # Construct target key with optional target prefix
+                        if target_prefix:
+                            target_key = f"{target_prefix}/{relative_file_path}"
+                        else:
+                            target_key = relative_file_path
+                        
+                        logger.info(f"Copying {source_bucket}/{source_key} to {target_bucket}/{target_key}")
+                        
+                        try:
+                            copy_source = {'Bucket': source_bucket, 'Key': source_key}
+                            s3_client.copy_object(
+                                CopySource=copy_source,
+                                Bucket=target_bucket,
+                                Key=target_key
+                            )
+                            copied_count += 1
+                        except Exception as copy_error:
+                            logger.warning(f"Failed to copy {source_key}: {str(copy_error)}")
+                            # Continue with other files instead of failing the entire operation
                     
                     logger.info(f"Successfully copied {copied_count} configuration files")
                     cfnresponse.send(event, context, cfnresponse.SUCCESS, {
@@ -633,6 +634,7 @@ Resources:
       TargetBucket: !Ref ConfigurationBucket
       TargetPrefix: "config_library"
       ConfigLibraryHash: "<CONFIG_LIBRARY_HASH_TOKEN>"
+      FileList: <CONFIG_FILES_LIST_TOKEN>
 
   ##########################################################################
   # Nested stack for selected options