Merge branch 'feature/add-p2-pipeline' into 'develop'

Pipeline improvements for integrating idp-cli and cf role changes to support p2

See merge request genaiic-reusable-assets/engagement-artifacts/genaiic-idp-accelerator!372

Author: rstrahan

.gitlab-ci.yml

@@ -4,6 +4,10 @@
 # Any publicly available image
 image: public.ecr.aws/docker/library/python:3.13-bookworm
 
+# Global timeout for all jobs
+default:
+  timeout: 2h
+
 # Change pip's cache directory to be inside the project directory since we can
 # only cache local items.
 # variables:
@@ -51,6 +55,7 @@ developer_tests:
 
 integration_tests:
   stage: integration_tests
+  timeout: 2h
   # variables:
   #   # In order to run tests in another account, add a AWS_CREDS_TARGET_ROLE variable to the Gitlab pipeline variables.
   #   AWS_CREDS_TARGET_ROLE: ${AWS_CREDS_TARGET_ROLE}

iam-roles/cloudformation-management/IDP-Cloudformation-Service-Role.yaml

@@ -59,6 +59,10 @@ Resources:
                   - iam:DeletePolicy
                   - iam:GetPolicy
                   - iam:ListPolicies
+                  - iam:ListPolicyVersions
+                  - iam:CreatePolicyVersion
+                  - iam:DeletePolicyVersion
+                  - iam:SetDefaultPolicyVersion
                   - iam:AttachRolePolicy
                   - iam:DetachRolePolicy
                   - iam:PutRolePolicy
@@ -122,6 +126,7 @@ Resources:
                   - ec2:DeleteTags
                   - ec2:DescribeTags
                   - ec2:DescribeAvailabilityZones
+                  - ecr:*
                 Resource: '*'
 
   PassRolePolicy:

scripts/sdlc/cfn/codepipeline-s3.yml

@@ -168,8 +168,10 @@ Resources:
                 - export IDP_CFN_PREFIX=$(make cfn-prefix) || { echo "CFN prefix generation failed"; exit 1; }
                 - make install -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX
                 - make smoketest -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX
-                - echo "All tests passed successfully, running cleanup..."
-                - make uninstall -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX && aws cloudformation delete-stack --stack-name ${IDP_CFN_PREFIX}-cloudformation-service-role --region us-east-1 || echo "Cleanup failed but continuing"
+              finally:
+                - echo "Running cleanup regardless of build result..."
+                - make uninstall -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX || echo "Cleanup failed but continuing"
+                - make -n cli-smoketest >/dev/null 2>&1 && make cli-smoketest -e IDP_CFN_PREFIX=$IDP_CFN_PREFIX || echo "CLI smoketest target not found, skipping"
 
   DeploymentPipeline:
     Type: 'AWS::CodePipeline::Pipeline'

scripts/sdlc/cfn/sdlc-iam-role.yml

@@ -99,6 +99,7 @@ Resources:
     Properties:
       RoleName: !Ref BuilderRoleName
       Description: 'Role for application builders with PowerUser and Limited IAM access'
+      MaxSessionDuration: 7200  # 2 hours for long-running CI/CD operations
       AssumeRolePolicyDocument:
         Version: '2012-10-17'
         Statement:

scripts/sdlc/idp-cli/Makefile

@@ -10,6 +10,12 @@ IDP_CFN_PREFIX?=idp-dev
 IDP_STACK_NAME?=$(IDP_CFN_PREFIX)
 IDP_PIPELINE_NAME?=idp-sdlc-deploy-pipeline
 
+# Pattern-specific configurations
+IDP_PATTERN1_STACK_NAME?=$(IDP_CFN_PREFIX)-p1
+IDP_PATTERN2_STACK_NAME?=$(IDP_CFN_PREFIX)-p2
+IDP_PATTERN1_NAME?=Pattern1 - Packet or Media processing with Bedrock Data Automation (BDA)
+IDP_PATTERN2_NAME?=Pattern2 - OCR → Bedrock Classification (page-level or holistic) → Bedrock Extraction
+
 test:
 	poetry run pytest -s --log-cli-level=DEBUG
 
@@ -27,20 +33,28 @@ install:
 uninstall:
 	export S3UTIL_ALLOW_BUCKET_DESTRUCT=true && \
 	poetry run idpcli uninstall \
-	--stack-name=$(IDP_STACK_NAME) \
+	--stack-name-prefix=$(IDP_CFN_PREFIX) \
 	--account-id=$(IDP_ACCOUNT_ID) \
 	--cfn-prefix=$(IDP_CFN_PREFIX)
-	# Clean up CloudWatch Logs resource policy after uninstall
-	cd $(IDP_CWD) && python3 scripts/cleanup-logs.py || echo "Log cleanup failed but continuing" \
+	cd $(IDP_CWD) && python3 scripts/cleanup-logs.py || echo "Log cleanup failed but continuing"
 
 smoketest:
 	poetry run idpcli smoketest \
-	--stack-name=$(IDP_STACK_NAME)
+	--stack-name-prefix=$(IDP_CFN_PREFIX) \
+	--file-path="../../../samples/lending_package.pdf"
+
+cli-smoketest:
+	poetry run idpcli idp-cli-smoketest \
+	--cfn-prefix=$(IDP_CFN_PREFIX) \
+	--admin-email=$(IDP_ADMIN_EMAIL) \
+	--account-id=$(IDP_ACCOUNT_ID) \
+	--cwd=$(IDP_CWD)
+
 
 wait:
 	$(eval VERSION_ID := $(shell cat $(IDP_CWD)/.upload_version_id 2>/dev/null || echo ""))
 	@if [ -z "$(VERSION_ID)" ]; then echo "Error: No version ID found"; exit 1; fi
-	$(eval EXECUTION_ID := $(shell poetry run python -c "from idp_cli.util.codepipeline_util import CodePipelineUtil; print(CodePipelineUtil.get_execution_id_by_version('$(IDP_PIPELINE_NAME)', '$(VERSION_ID)'))" || echo "FAILED"))
+	$(eval EXECUTION_ID := $(shell poetry run python -c "from idpcli.util.codepipeline_util import CodePipelineUtil; print(CodePipelineUtil.get_execution_id_by_version('$(IDP_PIPELINE_NAME)', '$(VERSION_ID)'))" || echo "FAILED"))
 	@if [ "$(EXECUTION_ID)" = "FAILED" ]; then echo "Error: Failed to find pipeline execution for version $(VERSION_ID)"; exit 1; fi
 	poetry run idpcli monitor-pipeline \
 	--pipeline-name=$(IDP_PIPELINE_NAME) \

scripts/sdlc/idp-cli/pyproject.toml

@@ -19,10 +19,10 @@ dependencies = [
 ]
 
 [tool.poetry]
-packages = [{include = "idp_cli", from = "src"}]
+packages = [{include = "idpcli", from = "src"}]
 
 [tool.poetry.scripts]
-idpcli = "idp_cli.cli.main:app"
+idpcli = "idpcli.cli.main:app"
 
 [tool.poetry.group.dev.dependencies]
 ipykernel = "^6.29.5"

scripts/sdlc/idp-cli/src/idp_cli/service/uninstall_service.py

@@ -0,0 +1,2 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT-0

…pts/sdlc/idp-cli/src/idp_cli/__init__.py …ipts/sdlc/idp-cli/src/idpcli/__init__.pyscripts/sdlc/idp-cli/src/idp_cli/__init__.py renamed to scripts/sdlc/idp-cli/src/idpcli/__init__.py scripts/sdlc/idp-cli/src/idp_cli/__init__.py renamed to scripts/sdlc/idp-cli/src/idpcli/__init__.py

@@ -3,11 +3,12 @@
 
 import os
 import sys
-from idp_cli.service.install_service import InstallService
-from idp_cli.util.codepipeline_util import CodePipelineUtil
+from idpcli.service.install_service import InstallService
+from idpcli.util.codepipeline_util import CodePipelineUtil
 import typer
-from idp_cli.service.uninstall_service import UninstallService
-from idp_cli.service.smoketest_service import SmokeTestService
+from idpcli.service.uninstall_service import UninstallService
+from idpcli.service.smoketest_service import SmokeTestService
+from idpcli.service.smoketest_idp_cli_service import SmokeTestIdpCliService
 from dotenv import load_dotenv
 
 from loguru import logger
@@ -28,7 +29,6 @@ def install(
     account_id: str = typer.Option(..., "--account-id", help="AWS Account ID"),
     cfn_prefix: str = typer.Option("idp-dev", "--cfn-prefix", help="An identifier to prefix the stack"),
     admin_email: str = typer.Option(..., "--admin-email", help="The admin email"),
-    idp_pattern: str = typer.Option("Pattern1 - Packet or Media processing with Bedrock Data Automation (BDA)", "--idp-pattern", help="The IDP Pattern to install"),
     cwd: str = typer.Option("./", "--cwd", help="Current working directory"),
     debug: bool = typer.Option(False, "--debug", help="Enable debug mode"),
     publish: bool = typer.Option(True, "--publish", help="Control publishing"),
@@ -43,27 +43,35 @@ def install(
         service.publish()
 
     if deploy:
-        service.install(admin_email=admin_email, idp_pattern=idp_pattern)
-        typer.echo("Install Complete!")
+        all_patterns_succeeded = service.install(admin_email=admin_email)
+        if all_patterns_succeeded:
+            typer.echo("Install Complete!")
+        else:
+            typer.echo("Install failed!", err=True)
+            sys.exit(1)
 
 
 @app.command()
 def uninstall(
-    stack_name: str = typer.Option(..., "--stack-name", help="Name of the stack to uninstall"),
+    stack_name_prefix: str = typer.Option(..., "--stack-name-prefix", help="Prefix of the stacks to uninstall"),
     account_id: str = typer.Option(..., "--account-id", help="AWS Account ID"),
     cfn_prefix: str = typer.Option("idp-dev", "--cfn-prefix", help="An identifier to prefix the stack")
 ):
     """
     Uninstall IDP Accelerator
     """
     try:
-        typer.echo(f"Uninstalling stack: {stack_name}")
-
-        service = UninstallService(stack_name=stack_name, account_id=account_id, cfn_prefix=cfn_prefix)
+        typer.echo(f"Uninstalling stacks with prefix: {stack_name_prefix}")
 
-        service.uninstall()
+        service = UninstallService(stack_name_prefix=stack_name_prefix, account_id=account_id, cfn_prefix=cfn_prefix)
 
-        typer.echo("Uninstall Complete!")
+        all_patterns_succeeded = service.uninstall()
+        
+        if all_patterns_succeeded:
+            typer.echo("Uninstall Complete!")
+        else:
+            typer.echo("Uninstall failed!", err=True)
+            sys.exit(1)
     except Exception as e:
         logger.exception(f"Error during uninstall process: {str(e)}")
         typer.echo(f"Uninstall failed: {str(e)}", err=True)
@@ -72,26 +80,26 @@ def uninstall(
 
 @app.command()
 def smoketest(
-    stack_name: str = typer.Option("idp-Stack", "--stack-name", help="Name of the deployed stack to test"),
+    stack_name_prefix: str = typer.Option("idp-Stack", "--stack-name-prefix", help="Prefix of the deployed stacks to test"),
     file_path: str = typer.Option("../../../samples/lending_package.pdf", "--file-path", help="Path to the test file"),
     verify_string: str = typer.Option("ANYTOWN, USA 12345", "--verify-string", help="String to verify in the processed output")
 ):
     """
-    Run a smoke test on the deployed IDP Accelerator
+    Run a smoke test on both deployed IDP patterns
     """
     try:
-        typer.echo(f"Running smoke test on stack: {stack_name}")
+        typer.echo(f"Running smoke test on stacks with prefix: {stack_name_prefix}")
 
         service = SmokeTestService(
-            stack_name=stack_name,
+            stack_name_prefix=stack_name_prefix,
             file_path=file_path,
             verify_string=verify_string
         )
 
         result = service.do_smoketest()
 
         if result:
-            typer.echo("Smoke test passed successfully!")
+            typer.echo("All smoke tests passed successfully!")
         else:
             typer.echo("Smoke test failed!", err=True)
             sys.exit(1)
@@ -100,6 +108,38 @@ def smoketest(
         typer.echo(f"Smoke test failed: {str(e)}", err=True)
         sys.exit(1)
 
+@app.command()
+def idp_cli_smoketest(
+    cfn_prefix: str = typer.Option(..., "--cfn-prefix", help="CloudFormation prefix for stack naming"),
+    admin_email: str = typer.Option(..., "--admin-email", help="Admin email for deployment"),
+    account_id: str = typer.Option(..., "--account-id", help="AWS account ID"),
+    cwd: str = typer.Option("../../../", "--cwd", help="Working directory path")
+):
+    """
+    End-to-end smoketest: install CLI, deploy stack, run inference, verify results
+    """
+    try:
+        typer.echo(f"Running IDP CLI smoketest with prefix: {cfn_prefix}")
+        
+        service = SmokeTestIdpCliService(
+            cfn_prefix=cfn_prefix,
+            admin_email=admin_email,
+            account_id=account_id,
+            cwd=cwd
+        )
+        
+        result = service.do_smoketest()
+        
+        if result:
+            typer.echo("IDP CLI smoketest passed successfully!")
+        else:
+            typer.echo("IDP CLI smoketest failed!", err=True)
+            sys.exit(1)
+    except Exception as e:
+        logger.exception(f"Error during IDP CLI smoketest: {str(e)}")
+        typer.echo(f"IDP CLI smoketest failed: {str(e)}", err=True)
+        sys.exit(1)
+
 @app.command()
 def monitor_pipeline(
     pipeline_name: str = typer.Option(..., "--pipeline-name", help="Name of the CodePipeline to monitor"),