[PRMP-939] - Report of anything left in staging

.github/workflows/base-lambdas-reusable-deploy-all.yml

@@ -434,6 +434,20 @@ jobs:
     secrets:
       AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
 
+  deploy_s3_data_collection_lambda:
+    name: Deploy S3 data collection lambda
+    uses: ./.github/workflows/base-lambdas-reusable-deploy.yml
+    with:
+      environment: ${{ inputs.environment}}
+      python_version: ${{ inputs.python_version }}
+      build_branch: ${{ inputs.build_branch}}
+      sandbox: ${{ inputs.sandbox }}
+      lambda_handler_name: data_collection_handler
+      lambda_aws_name: ReportS3Content
+      lambda_layer_names: "core_lambda_layer,data_lambda_layer"
+    secrets:
+      AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
+
   deploy_statistical_report_lambda:
     name: Deploy statistical report lambda
     uses: ./.github/workflows/base-lambdas-reusable-deploy.yml
@@ -838,3 +852,17 @@ jobs:
       lambda_layer_names: "core_lambda_layer"
     secrets:
       AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
+
+#  deploy_report_s3_content_lambda:
+#    name: Deploy Report S3 Content Lambda
+#    uses: ./.github/workflows/base-lambdas-reusable-deploy.yml
+#    with:
+#      environment: ${{ inputs.environment }}
+#      python_version: ${{ inputs.python_version }}
+#      build_branch: ${{ inputs.build_branch }}
+#      sandbox: ${{ inputs.sandbox }}
+#      lambda_handler_name: report_s3_content_handler
+#      lambda_aws_name: ReportS3Content
+#      lambda_layer_names: "core_lambda_layer"
+#    secrets:
+#      AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}

.github/workflows/full-deploy-to-pre-prod.yml

@@ -136,6 +136,17 @@ jobs:
     secrets:
       AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
 
+  deploy_s3_data_collection:
+    name: Deploy S3 Data Collection
+    needs: [ "tag_and_release" ]
+    uses: ./.github/workflows/s3-base-data-collection.yml
+    with:
+      build_branch: ${{ needs.tag_and_release.outputs.version }}
+      environment: pre-prod
+      sandbox: pre-prod
+    secrets:
+      AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
+
   run_fhir_api_e2e_tests:
     name: Run FHIR API E2E Tests
     needs: ["deploy_all_lambdas"]

.github/workflows/full-deploy-to-prod.yml

@@ -75,3 +75,13 @@ jobs:
       sandbox: prod
     secrets:
       AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
+
+  deploy_s3_data_collection:
+    name: Deploy S3 Data Collection
+    uses: ./.github/workflows/s3_base-data-collection.yml
+    with:
+      build_branch: ${{ inputs.tag_version }}
+      environment: prod
+      sandbox: prod
+    secrets:
+      AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}

.github/workflows/full-deploy-to-sandbox.yml

@@ -166,6 +166,17 @@ jobs:
     secrets:
       AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
 
+  deploy_s3_data_collection:
+    name: "Deploy S3 Data Collection (ECS task image)"
+    uses: ./.github/workflows/s3-base-data-collection.yml
+    needs: ["deploy_all_lambdas"]
+    with:
+      build_branch: ${{ inputs.build_branch }}
+      sandbox: ${{ inputs.sandbox }}
+      environment: ${{ inputs.environment }}
+    secrets:
+      AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
+
   bulk_upload:
     name: "Run Bulk Upload"
     if: ${{ inputs.bulk_upload }}

.github/workflows/lambdas-dev-to-main-ci.yml

@@ -80,6 +80,18 @@ jobs:
     secrets:
       AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
 
+  deploy_s3_data_collection:
+    name: Deploy S3 Data Collection
+    needs: [ "run_tests" ]
+    uses: ./.github/workflows/s3-base-data-collection.yml
+    if: github.ref == 'refs/heads/main'
+    with:
+      build_branch: ${{ github.event.pull_request.head.ref }}
+      environment: development
+      sandbox: ndr-dev
+    secrets:
+      AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}
+
   notify-slack:
     name: Notify Slack on Failure
     runs-on: ubuntu-latest
@@ -91,6 +103,7 @@ jobs:
         publish_all_lambda_layers,
         deploy_all_lambdas,
         deploy_data_collection,
+        deploy_s3_data_collection,
       ]
     if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/main'
     steps:
@@ -160,6 +173,10 @@ jobs:
                         {
                           "type": "mrkdwn",
                           "text": "*deploy_data_collection:* ${{ needs.deploy_data_collection.result == 'success' && ':white_check_mark:' || ':x:' }}"
+                        },
+                        {
+                          "type": "mrkdwn",
+                          "text": "*deploy_s3_data_collection:* ${{ needs.deploy_s3_data_collection.result == 'success' && ':white_check_mark:' || ':x:' }}"
                         }
                       ]
                     },

.github/workflows/s3-base-data-collection.yml

@@ -0,0 +1,119 @@
+name: "Z-BASE Deploy S3 Data Collection: Build data collection image"
+
+run-name: "${{ github.event.inputs.build_branch }} | ${{ github.event.inputs.environment }} | ${{ github.event.inputs.sandbox }}"
+
+on:
+  workflow_call:
+    inputs:
+      build_branch:
+        required: true
+        type: string
+      environment:
+        required: true
+        type: string
+      sandbox:
+        required: true
+        type: string
+    secrets:
+      AWS_ASSUME_ROLE:
+        required: true
+
+permissions:
+  pull-requests: write
+  id-token: write
+  contents: read
+
+jobs:
+  s3_data_collection_build_docker_image:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    defaults:
+      run:
+        working-directory: lambdas
+
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          repository: "NHSDigital/national-document-repository"
+          ref: ${{ inputs.build_branch }}
+          fetch-depth: "0"
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v5
+        with:
+          role-to-assume: ${{ secrets.AWS_ASSUME_ROLE }}
+          role-skip-session-tagging: true
+          aws-region: ${{ vars.AWS_REGION }}
+          mask-aws-account-id: true
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build, tag, and push image to Amazon ECR
+        id: build-image
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: ${{ inputs.sandbox }}-s3-data-collection
+          IMAGE_TAG: latest
+          IMAGE_TAG_SHA: ${{ github.sha }}
+        run: |
+          set -euo pipefail
+
+          docker build \
+            -t "$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" \
+            -t "$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA" \
+            -f ecs/s3_data_collection/Dockerfile \
+            .
+
+          docker push "$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG"
+          docker push "$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA"
+
+          echo "image=$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG_SHA" >> "$GITHUB_OUTPUT"
+
+      - name: Download current task definition (raw)
+        id: download-task
+        run: |
+          set -euo pipefail
+
+          aws ecs describe-task-definition \
+            --task-definition "${{ inputs.sandbox }}-task-s3-data-collection" \
+            --query taskDefinition \
+            > task-definition-raw.json
+
+          echo "revision=$(jq -r .revision task-definition-raw.json)" >> "$GITHUB_OUTPUT"
+
+      - name: Render task definition with new image
+        id: task-def
+        uses: aws-actions/amazon-ecs-render-task-definition@v1
+        with:
+          task-definition: lambdas/task-definition-raw.json
+          container-name: ${{ inputs.sandbox }}-container-s3-data-collection
+          image: ${{ steps.build-image.outputs.image }}
+
+      - name: Sanitize task definition JSON for registration
+        run: |
+          set -euo pipefail
+
+          jq 'del(
+                .taskDefinitionArn,
+                .revision,
+                .status,
+                .requiresAttributes,
+                .compatibilities,
+                .registeredAt,
+                .registeredBy
+              )' \
+            "${{ steps.task-def.outputs.task-definition }}" \
+            > task-definition.json
+
+      - name: Register new ECS task definition revision
+        run: |
+          set -euo pipefail
+          aws ecs register-task-definition --cli-input-json file://task-definition.json
+
+      - name: De-register previous revision (optional)
+        run: |
+          set -euo pipefail
+          aws ecs deregister-task-definition \
+            --task-definition "${{ inputs.sandbox }}-task-s3-data-collection:${{ steps.download-task.outputs.revision }}"

.github/workflows/s3-data-collection-deploy-to-sandbox.yml

@@ -0,0 +1,58 @@
+name: "SANDBOX S3 Data Collection - Publish Data Collection Image to ECR"
+
+run-name: "${{ github.event.inputs.build_branch }} | ${{ github.event.inputs.sandbox }} | ${{ github.event.inputs.environment }}"
+
+on:
+  workflow_dispatch:
+    inputs:
+      build_branch:
+        description: "Feature branch to push."
+        required: true
+        type: string
+        default: "main"
+      sandbox:
+        description: "Which Sandbox to push to."
+        required: true
+        type: string
+        default: "ndr"
+      environment:
+        description: "Which Environment settings to use."
+        required: true
+        type: string
+        default: "development"
+  workflow_call:
+    inputs:
+      build_branch:
+        description: "Feature branch to push."
+        required: true
+        type: string
+        default: "main"
+      sandbox:
+        description: "Which Sandbox to push to."
+        required: true
+        type: string
+        default: "ndr"
+      environment:
+        description: "Which Environment settings to use."
+        required: true
+        type: string
+        default: "development"
+    secrets:
+      AWS_ASSUME_ROLE:
+        required: true
+
+permissions:
+  pull-requests: write
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
+
+jobs:
+  push_image:
+    name: Push Image
+    uses: ./.github/workflows/s3-base-data-collection.yml
+    with:
+      build_branch: ${{ inputs.build_branch }}
+      environment: ${{ inputs.environment }}
+      sandbox: ${{ inputs.sandbox }}
+    secrets:
+      AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }}

lambdas/ecs/s3_data_collection/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.11
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONPATH=/lambdas
+
+WORKDIR /lambdas
+COPY requirements /lambdas/requirements
+
+RUN pip install -r requirements/layers/requirements_core_lambda_layer.txt
+RUN pip install -r requirements/layers/requirements_data_lambda_layer.txt
+
+COPY . /lambdas
+
+ENTRYPOINT ["python", "-m", "handlers.report_s3_content_handler"]

lambdas/handlers/report_s3_content_handler.py

@@ -0,0 +1,35 @@
+from services.reporting.report_s3_content_service import ReportS3ContentService
+from utils.audit_logging_setup import LoggingService
+from utils.decorators.ensure_env_var import ensure_environment_variables
+from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions
+from utils.decorators.override_error_check import override_error_check
+from utils.decorators.set_audit_arg import set_request_context_for_logging
+
+logger = LoggingService(__name__)
+
+@ensure_environment_variables(names=["LLOYD_GEORGE_BUCKET_NAME"])
+@ensure_environment_variables(names=["BULK_STAGING_BUCKET_NAME"])
+@ensure_environment_variables(names=["STATISTICAL_REPORTS_BUCKET"])
+@override_error_check
+@handle_lambda_exceptions
+@set_request_context_for_logging
+def lambda_handler(event, context):
+    logger.info("Report S3 content lambda invoked")
+
+    service = ReportS3ContentService()
+
+    service.process_s3_content(
+    )
+
+
+
+class _EcsContext:
+    aws_request_id = "ecs-run"
+    function_name = "ReportS3ContentEcsTask"
+    invoked_function_arn = "ecs://ReportS3ContentEcsTask"
+    log_group_name = "ecs"
+    log_stream_name = "ecs"
+
+
+if __name__ == "__main__":
+    lambda_handler({}, _EcsContext())

lambdas/services/base/s3_service.py

@@ -255,3 +255,24 @@ def save_or_create_file(self, source_bucket: str, file_key: str, body: bytes):
         return self.client.put_object(
             Bucket=source_bucket, Key=file_key, Body=BytesIO(body)
         )
+
+    def list_all_object_versions(self, bucket_name: str):
+        paginator = self.client.get_paginator("list_object_versions")
+        versions, delete_markers = [], []
+
+        for page in paginator.paginate(Bucket=bucket_name):
+            versions.extend(page.get("Versions", []))
+            delete_markers.extend(page.get("DeleteMarkers", []))
+
+        return versions, delete_markers
+
+    def get_object_tags_versioned(self, bucket: str, key: str, version_id: str | None):
+        try:
+            params = {"Bucket": bucket, "Key": key}
+            if version_id:
+                params["VersionId"] = version_id
+            response = self.client.get_object_tagging(**params)
+            return response.get("TagSet", [])
+        except ClientError:
+            return []
+