From 2a1def527f6c5024b527994ffc291a3a0e81d350 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 17:55:05 +0530 Subject: [PATCH 01/12] fix the preview workflow --- .github/workflows/preview_create.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/preview_create.yml b/.github/workflows/preview_create.yml index 93d1e63..877aece 100644 --- a/.github/workflows/preview_create.yml +++ b/.github/workflows/preview_create.yml @@ -7,7 +7,6 @@ on: - ./*.md - LICENSE - .circleci/* - - .gitlab-ci.yml jobs: test: @@ -30,13 +29,13 @@ jobs: pip install awscli-local - name: Deploy Preview - uses: LocalStack/setup-localstack@v0.2.2 + uses: LocalStack/setup-localstack@v0.2.4 env: AWS_DEFAULT_REGION: us-east-1 AWS_REGION: us-east-1 AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test - LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} with: github-token: ${{ secrets.GITHUB_TOKEN }} state-backend: ephemeral @@ -46,4 +45,4 @@ jobs: preview-cmd: | # Add your custom deployment commands here. # Below is an example for the Image resizer application. - bin/build_lambdas.sh && deployment/awslocal/deploy.sh \ No newline at end of file + bin/build_lambdas.sh && deployment/awslocal/deploy.sh From 84ff9afa0e351b1b8f60bd05af1886825a740523 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 18:03:14 +0530 Subject: [PATCH 02/12] fix --- .github/workflows/preview_create.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview_create.yml b/.github/workflows/preview_create.yml index 877aece..0794b97 100644 --- a/.github/workflows/preview_create.yml +++ b/.github/workflows/preview_create.yml @@ -40,8 +40,8 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} state-backend: ephemeral state-action: start - skip-ephemeral-stop: 'true' include-preview: 'true' + install-awslocal: 'true' preview-cmd: | # Add your custom deployment commands here. # Below is an example for the Image resizer application. From 10bba6e994f4f77ba07a8961421bf3b4fefd8906 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 18:06:54 +0530 Subject: [PATCH 03/12] fix --- .github/workflows/preview_create.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/preview_create.yml b/.github/workflows/preview_create.yml index 0794b97..5ba27fc 100644 --- a/.github/workflows/preview_create.yml +++ b/.github/workflows/preview_create.yml @@ -29,12 +29,8 @@ jobs: pip install awscli-local - name: Deploy Preview - uses: LocalStack/setup-localstack@v0.2.4 + uses: LocalStack/setup-localstack@main env: - AWS_DEFAULT_REGION: us-east-1 - AWS_REGION: us-east-1 - AWS_ACCESS_KEY_ID: test - AWS_SECRET_ACCESS_KEY: test LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} with: github-token: ${{ secrets.GITHUB_TOKEN }} From 7ceda8ebdd90f97d8fcf716851cb390b13778c4a Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 19:18:31 +0530 Subject: [PATCH 04/12] test against a branch --- .github/workflows/preview_create.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview_create.yml b/.github/workflows/preview_create.yml index 5ba27fc..fb27e01 100644 --- a/.github/workflows/preview_create.yml +++ b/.github/workflows/preview_create.yml @@ -29,7 +29,7 @@ jobs: pip install awscli-local - name: Deploy Preview - uses: LocalStack/setup-localstack@main + uses: LocalStack/setup-localstack@HarshCasper-patch-1 env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} with: From 1db5982b19ddcef64c0a6aeaae79aef4064b26a7 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 20:07:55 +0530 Subject: [PATCH 05/12] remove the branch --- .github/workflows/preview_create.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview_create.yml b/.github/workflows/preview_create.yml index fb27e01..5ba27fc 100644 --- a/.github/workflows/preview_create.yml +++ b/.github/workflows/preview_create.yml @@ -29,7 +29,7 @@ jobs: pip install awscli-local - name: Deploy Preview - uses: LocalStack/setup-localstack@HarshCasper-patch-1 + uses: LocalStack/setup-localstack@main env: LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} with: From 4f3328e52ea4a3523b2dd9a2b91bc2f87021fda2 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 20:54:13 +0530 Subject: [PATCH 06/12] fix integration test with auth token --- .github/workflows/integration-test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 619adb4..373ead7 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -50,14 +50,15 @@ jobs: pip install -r requirements-dev.txt - name: Start LocalStack - uses: LocalStack/setup-localstack@v0.2.2 + uses: LocalStack/setup-localstack@v0.2.4 with: image-tag: 'latest' use-pro: 'true' configuration: LS_LOG=trace install-awslocal: 'true' env: - LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + - name: Build lambdas run: | bin/build_lambdas.sh From dcce5df799514beae1886ae92fd2ff19e3a8d001 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 20:59:34 +0530 Subject: [PATCH 07/12] try running on macos --- .github/workflows/integration-test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 373ead7..7305eec 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -27,7 +27,12 @@ on: jobs: run-it-tests-job: - runs-on: ${{ inputs.runner-os || 'ubuntu-latest' }} + runs-on: ${{ matrix.runner }} + strategy: + matrix: + runner: + - ubuntu-latest + - macos-latest steps: - name: Checkout uses: actions/checkout@v2 From 25a2c01dca466e64c68dcd881c50daaba0742148 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 21:02:27 +0530 Subject: [PATCH 08/12] setup docker on macos --- .github/workflows/integration-test.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 7305eec..8729c02 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -22,7 +22,7 @@ on: type: choice options: - ubuntu-latest - - macos-latest + - macos-13 jobs: @@ -30,12 +30,10 @@ jobs: runs-on: ${{ matrix.runner }} strategy: matrix: - runner: - - ubuntu-latest - - macos-latest + runner: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.runner-os && fromJSON(format('["{0}"]', github.event.inputs.runner-os))) || fromJSON('["ubuntu-latest", "macos-13"]') }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python 3.11 id: setup-python @@ -43,10 +41,19 @@ jobs: with: python-version: 3.11 + - name: Debug Runner Context (Ubuntu) + if: runner.os == 'Linux' # Only run this debug step on Linux + run: | + echo "DEBUG: Runner OS is ${{ runner.os }}" + echo "DEBUG: Event name is ${{ github.event_name }}" + echo "DEBUG: Evaluating runner.os == 'macOS' -> ${{ runner.os == 'macOS' }}" + echo "DEBUG: Evaluating github.event_name == 'schedule' -> ${{ github.event_name == 'schedule' }}" + echo "DEBUG: Evaluating full condition -> ${{ runner.os == 'macOS' && github.event_name == 'schedule' }}" + - name: Docker setup (macos only) - id: setup-docker-mac - if: ${{ runner.os == 'macOS' }} + if: ${{ runner.os == 'macOS' && github.event_name == 'schedule' }} run: | + brew install colima brew install docker colima start From e864987165b52d15272676f9e67d1f51ade7c2f0 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 23:01:57 +0530 Subject: [PATCH 09/12] run macOS tests only on schedule --- .github/workflows/integration-test.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 8729c02..9890482 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -27,10 +27,7 @@ on: jobs: run-it-tests-job: - runs-on: ${{ matrix.runner }} - strategy: - matrix: - runner: ${{ (github.event_name == 'workflow_dispatch' && github.event.inputs.runner-os && fromJSON(format('["{0}"]', github.event.inputs.runner-os))) || fromJSON('["ubuntu-latest", "macos-13"]') }} + runs-on: ${{ inputs.runner-os || 'ubuntu-latest' }} steps: - name: Checkout uses: actions/checkout@v4 @@ -41,17 +38,8 @@ jobs: with: python-version: 3.11 - - name: Debug Runner Context (Ubuntu) - if: runner.os == 'Linux' # Only run this debug step on Linux - run: | - echo "DEBUG: Runner OS is ${{ runner.os }}" - echo "DEBUG: Event name is ${{ github.event_name }}" - echo "DEBUG: Evaluating runner.os == 'macOS' -> ${{ runner.os == 'macOS' }}" - echo "DEBUG: Evaluating github.event_name == 'schedule' -> ${{ github.event_name == 'schedule' }}" - echo "DEBUG: Evaluating full condition -> ${{ runner.os == 'macOS' && github.event_name == 'schedule' }}" - - name: Docker setup (macos only) - if: ${{ runner.os == 'macOS' && github.event_name == 'schedule' }} + if: ${{ runner.os == 'macOS' }} run: | brew install colima brew install docker From 2645fee853ec415f625ee45345ca7b0eaebe8dc1 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 23:21:03 +0530 Subject: [PATCH 10/12] remove cloud pods release workflow --- .github/workflows/cloudpod_release.yml | 88 -------------------------- 1 file changed, 88 deletions(-) delete mode 100644 .github/workflows/cloudpod_release.yml diff --git a/.github/workflows/cloudpod_release.yml b/.github/workflows/cloudpod_release.yml deleted file mode 100644 index e695759..0000000 --- a/.github/workflows/cloudpod_release.yml +++ /dev/null @@ -1,88 +0,0 @@ -on: - workflow_dispatch: - inputs: - release-tag: - type: string - required: true - description: This will be the version of the release, but will also be used as 'tag' for the localstack docker image - push: - paths-ignore: - - ./*.md - - LICENSE - - .circleci/* - - .gitlab-ci.yml - branches: - - main - -permissions: - contents: write - -name: Create Release -jobs: - release: - name: Create Release for Cloud Pod - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Python 3.11 - id: setup-python - uses: actions/setup-python@v2 - with: - python-version: 3.11 - - - name: Set up Project - run: | - pip install -r requirements-dev.txt - - - name: Start LocalStack - uses: LocalStack/setup-localstack@v0.2.2 - with: - image-tag: ${{ inputs.release-tag || 'latest'}} - use-pro: 'true' - install-awslocal: 'true' - env: - LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} - - - name: Build lambdas - run: | - bin/build_lambdas.sh - - - name: Deploy infrastructure - run: | - deployment/awslocal/deploy.sh - - - name: Run Tests - env: - AWS_DEFAULT_REGION: us-east-1 - AWS_REGION: us-east-1 - AWS_ACCESS_KEY_ID: test - AWS_SECRET_ACCESS_KEY: test - run: | - pytest tests - - # Not using action as state is not stored as an artifact - - name: Save the Cloud Pod - env: - LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} - run: | - localstack state export release-pod.zip - - - name: Prepare Release Notes - run: | - echo "This release includes the Cloud Pod of the sample created with LocalStack Version \`${{ inputs.release-tag || 'latest'}}\`." > Release.txt - echo "You can download the \`release-pod.zip\` and inject it manually by running \`localstack state import release-pod.zip\`, or use the Cloud Pods Launchpad." >> Release.txt - echo "### Cloud Pods Launchpad" >> Release.txt - echo "You can click the Launchpad to inject the the pod into your running LocalStack instance using the WebUI:" >> Release.txt - echo "[![LocalStack Pods Launchpad](https://localstack.cloud/gh/launch-pod-badge.svg)](https://app.localstack.cloud/launchpad?url=https://github.com/$GITHUB_REPOSITORY/releases/download/${{ inputs.release-tag || 'latest'}}/release-pod.zip)" >> Release.txt - - - name: Create Release - id: create_release - uses: softprops/action-gh-release@v1 - with: - tag_name: "${{ inputs.release-tag || 'latest'}}" - name: "Cloud Pod for LocalStack Version '${{ inputs.release-tag || 'latest'}}'" - body_path: ./Release.txt - files: | - ./release-pod.zip From fa8e769f3e1afcc999d5c90d09f95082a6cb17f8 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 23:29:42 +0530 Subject: [PATCH 11/12] reform the cloud pods action --- .github/workflows/test_cloudpods.yml | 141 ++++++++++++++++++--------- 1 file changed, 93 insertions(+), 48 deletions(-) diff --git a/.github/workflows/test_cloudpods.yml b/.github/workflows/test_cloudpods.yml index c76a979..18891fd 100644 --- a/.github/workflows/test_cloudpods.yml +++ b/.github/workflows/test_cloudpods.yml @@ -1,75 +1,109 @@ -name: Test Released Cloud Pods +name: Create and Test LocalStack Cloud Pod on: schedule: - # “At 00:00 on Saturday.” + # At 00:00 on Saturday. - cron: "0 0 * * 6" + push: + branches: + - main + pull_request: + branches: + - main workflow_dispatch: permissions: contents: write + actions: read jobs: - get-releases: - name: Retrieve Released Cloud Pods + create-pod: + name: Create Cloud Pod runs-on: ubuntu-latest outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} + pod_artifact_name: cloud-pod-${{ github.run_id }} steps: - - id: set-matrix - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - output=$(gh api repos/$GITHUB_REPOSITORY/releases | jq -r '[.[] | select(.tag_name|startswith("v")|not) | .tag_name]') - output=$(echo $output | tr '\n' ' ') - echo "matrix=$output" >> $GITHUB_OUTPUT - - test-pod-release: - needs: get-releases - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - tag: ${{ fromJson(needs.get-releases.outputs.matrix) }} - steps: - # checkout to run the tests later on - - name: Checkout - uses: actions/checkout@v3 + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Dependencies + run: | + pip install -r requirements-dev.txt --upgrade + + - name: Start LocalStack + uses: LocalStack/setup-localstack@main + with: + use-pro: 'true' + install-awslocal: 'true' + env: + DEBUG: 1 + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + + - name: Deploy Infrastructure (Example) + run: | + bin/build_lambdas.sh && deployment/awslocal/deploy.sh - # Loading it manually as we're storing the state as a release and not an artifact - - name: Retrieve Pod + - name: Export LocalStack State (Cloud Pod) + id: export_state env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} run: | - # TODO the download url seems to follow the pattern $GITHUB_REPOSITORY/releases/download/{TAG}/{ASSET_NAME} - # alternatively we can query the asset-id, and browser_download_url, but it seems like an overhead - # asset_id=$(gh api repos/$GITHUB_REPOSITORY/releases/tags/latest | jq -r '.assets[]' | jq --arg DB $DB -c 'select(.name=="release-pod-\( $DB ).zip") | .id) - # download_url=$(gh api repos/$GITHUB_REPOSITORY/releases/assets/$asset_id | jq -r ".browser_download_url") - download_url="https://github.com/$GITHUB_REPOSITORY/releases/download/${{ matrix.tag }}/release-pod.zip" - curl -L $download_url --output release-pod.zip - ls -la + echo "Exporting LocalStack state..." + localstack state export cloud-pod.zip + ls -la cloud-pod.zip # Verify the file exists + echo "pod_artifact_name=cloud-pod-${{ github.run_id }}" >> $GITHUB_OUTPUT + + - name: Upload Cloud Pod Artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.export_state.outputs.pod_artifact_name }} + path: cloud-pod.zip + retention-days: 1 + + test-pod: + name: Test Cloud Pod + needs: create-pod + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.11' + - name: Install Test Dependencies + run: | + pip install -r requirements-dev.txt --upgrade + - name: Start LocalStack - uses: LocalStack/setup-localstack@v0.2.2 + uses: LocalStack/setup-localstack@main with: - image-tag: ${{ matrix.tag }} use-pro: 'true' install-awslocal: 'true' env: DEBUG: 1 POD_LOAD_CLI_TIMEOUT: 300 - LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + + - name: Download Cloud Pod Artifact + uses: actions/download-artifact@v4 + with: + name: ${{ needs.create-pod.outputs.pod_artifact_name }} - - name: Inject Pod + - name: Inject Pod (Import State) env: - LOCALSTACK_API_KEY: ${{ secrets.LOCALSTACK_API_KEY }} + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} run: | - localstack state import release-pod.zip + echo "Importing LocalStack state from cloud-pod.zip..." + ls -la # Check if download was successful + localstack state import cloud-pod.zip - name: Run Tests env: @@ -78,12 +112,12 @@ jobs: AWS_ACCESS_KEY_ID: test AWS_SECRET_ACCESS_KEY: test run: | - pip install -r requirements-dev.txt pytest tests - - name: Show Logs - if: failure() + - name: Show LocalStack Logs + if: always() run: | + echo "::error::Tests failed. Dumping LocalStack logs:" localstack logs - name: Send a Slack notification @@ -99,10 +133,21 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - - name: Prevent Workflows from getting Stale - if: always() + - name: Generate a Diagnostic Report + if: failure() + run: | + curl -s localhost:4566/_localstack/diagnose | gzip -cf > diagnose.json.gz + + - name: Upload the Diagnostic Report + if: failure() + uses: actions/upload-artifact@v4 + with: + name: diagnose.json.gz + path: ./diagnose.json.gz + + - name: Prevent Workflow from becoming Stale + if: always() && github.ref == 'refs/heads/main' uses: gautamkrishnar/keepalive-workflow@v1 with: - # this message should prevent automatic triggering of workflows - # see https://docs.github.com/en/actions/managing-workflow-runs/skipping-workflow-runs commit_message: "[skip ci] Automated commit by Keepalive Workflow to keep the repository active" + gh_token: ${{ secrets.GITHUB_TOKEN }} From f2dd1bf95d7eb636c50e56ed7ad0dca95d78a945 Mon Sep 17 00:00:00 2001 From: Harsh Mishra Date: Tue, 22 Apr 2025 23:33:54 +0530 Subject: [PATCH 12/12] remove keepalive workflow --- .github/workflows/test_cloudpods.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test_cloudpods.yml b/.github/workflows/test_cloudpods.yml index 18891fd..f193c54 100644 --- a/.github/workflows/test_cloudpods.yml +++ b/.github/workflows/test_cloudpods.yml @@ -117,7 +117,6 @@ jobs: - name: Show LocalStack Logs if: always() run: | - echo "::error::Tests failed. Dumping LocalStack logs:" localstack logs - name: Send a Slack notification @@ -147,7 +146,4 @@ jobs: - name: Prevent Workflow from becoming Stale if: always() && github.ref == 'refs/heads/main' - uses: gautamkrishnar/keepalive-workflow@v1 - with: - commit_message: "[skip ci] Automated commit by Keepalive Workflow to keep the repository active" - gh_token: ${{ secrets.GITHUB_TOKEN }} + uses: liskin/gh-workflow-keepalive@v1