diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 42ae6df492..1e80365116 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -10,14 +10,28 @@ RUN apt-get update && apt-get upgrade -y && apt-get install -y \ strip-nondeterminism \ uuid-runtime \ parallel \ - bc + bc \ + curl \ + ca-certificates \ + gnupg + +# Prepare github-cli install +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ + | gpg --dearmor \ + | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \ + && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] \ + https://cli.github.com/packages stable main" \ + | tee /etc/apt/sources.list.d/github-cli.list > /dev/null # Install CLI tools. RUN apt-get update && apt-get install -y \ vim \ ranger \ tmux \ - fzf + fzf \ + gh # Install base neovim RUN set -eux; \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ce88bee614..afb1f79547 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -48,7 +48,9 @@ "source=${localEnv:HOME}/.ssh,target=/home/vscode/.ssh,type=bind,consistency=cached", "source=${localEnv:HOME}/.gnupg,target=/home/vscode/.gnupg,type=bind,consistency=cached", "source=${localWorkspaceFolder}/.devcontainer/config/nvim,target=/home/vscode/.config/nvim,type=bind", - "source=${env:HOME}/.gitconfig,target=/home/vscode/.gitconfig,type=bind,consistency=cached" + "source=${env:HOME}/.gitconfig,target=/home/vscode/.gitconfig,type=bind,consistency=cached", + "source=${localEnv:HOME}/.config/gh,target=/home/vscode/.config/gh,type=bind", + "source=${localEnv:NDRI_LOCATION},target=/workspaces/national-document-repository-infrastructure,type=bind,consistency=cached" ], "postCreateCommand": "HOST_PWD=${localWorkspaceFolder} bash -c '.devcontainer/src/create.sh' ", "runArgs": [ diff --git a/Makefile b/Makefile index 4400570e9b..11013a969c 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ ZIP_BASE_PATH = ./$(LAMBDAS_BUILD_PATH)/$(lambda_name)/tmp ZIP_COMMON_FILES = lambdas/utils lambdas/models lambdas/services lambdas/repositories lambdas/enums lambdas/scripts CONTAINER ?= false -.PHONY: install clean help format list requirements ruff +.PHONY: install clean help format list requirements ruff build-and-deploy-sandbox default: help @@ -68,6 +68,17 @@ check-packages: ./lambdas/venv/bin/pip-audit -r $(REPORTS_REQUIREMENTS) ./lambdas/venv/bin/pip-audit -r $(ALERTING_REQUIREMENTS) +build-and-deploy-sandbox: ## Build a sandbox and deploy code. If no SANDBOX_NAME is provided it will use your current branch as the name. It will default to building and deploying using 'main', You can skip building infrastructure by BUILD_INFRA=false. Usage: make build-and-deploy-sandbox SANDBOX_NAME= NDRI_WORKFLOW_BRANCH= NDRI_BRANCH= NDR_WORKFLOW_BRANCH= NDR_BRANCH= BUILD_INFRA= NDRI_DIR_LOC_OVERRIDE= + @./scripts/build_and_deploy_sandbox.sh \ + $(if $(NDRI_WORKFLOW_BRANCH),--ndri_workflow_branch=$(NDRI_WORKFLOW_BRANCH)) \ + $(if $(NDRI_BRANCH),--ndri_branch=$(NDRI_BRANCH)) \ + $(if $(NDR_WORKFLOW_BRANCH),--ndr_workflow_branch=$(NDR_WORKFLOW_BRANCH)) \ + $(if $(NDR_BRANCH),--ndr_branch=$(NDR_BRANCH)) \ + $(if $(SANDBOX_NAME),--sandbox_name=$(SANDBOX_NAME)) \ + $(if $(BUILD_INFRA),--build_infra=$(BUILD_INFRA)) \ + $(if $(FULL_DEPLOY),--full_deploy=$(FULL_DEPLOY)) \ + $(if $(NDRI_DIR_LOC_OVERRIDE),--ndri_dir_loc_override=$(NDRI_DIR_LOC_OVERRIDE)) + download-api-certs: ## Downloads mTLS certificates (use with dev envs only). Usage: make download-api-certs WORKSPACE= rm -rf ./lambdas/mtls_env_certs/$(WORKSPACE) ./scripts/aws/download-api-certs.sh $(WORKSPACE) diff --git a/README.md b/README.md index 9621ae9fde..5c4b0c3086 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,15 @@ The following tools are required for all options: - [Git](https://git-scm.com/) - Docker (e.g. via [Brew](https://formulae.brew.sh/formula/docker)) +Setup an environment variable on your local system. The environment variable points to your national-document-repository-infrastructure directory on your local system. +For Linux/MacOS users add the following to your ~/.zshrc or ~/.bashrc file + +```bash +export NDRI_LOCATION= +``` + +For Windows users, please follow Microsoft's recommendations for creating persistent environment variables + ### Method 1 - Dev container within VS Code (recommended) > [!IMPORTANT] @@ -70,6 +79,14 @@ lazygit - [Node@24](https://formulae.brew.sh/formula/node@24) - [Python@3.11](https://formulae.brew.sh/formula/python@3.11) +### Initial Setup of the container + +1. Configure Github-CLI with + +```bash +gh auth login +``` + ## Monitoring We have configured AWS CloudWatch to provide alarm notifications whenever one of a number of metrics exceeds its normal diff --git a/scripts/build_and_deploy_sandbox.sh b/scripts/build_and_deploy_sandbox.sh new file mode 100755 index 0000000000..c9c6bfd9df --- /dev/null +++ b/scripts/build_and_deploy_sandbox.sh @@ -0,0 +1,228 @@ +#!/bin/bash +set -euo pipefail + +# Initialize variables to track options +NDR_DIRECTORY="$(pwd)" +PARENT_DIR="$(dirname "$NDR_DIRECTORY")" +NDRI_DIRECTORY="$PARENT_DIR/national-document-repository-infrastructure" +BUILD_INFRA=true +NDRI_BRANCH="main" +NDRI_WORKFLOW_BRANCH="main" +NDRI_WORKFLOW_FILE="deploy-sandbox.yml" +NDR_BRANCH="main" +NDR_WORKFLOW_BRANCH="main" +NDR_WORKFLOW_FILE="lambdas-deploy-feature-to-sandbox.yml" +NDR_WORKFLOW_FILE_FULL="full-deploy-to-sandbox.yml" +FULL_DEPLOY=false +SANDBOX_NAME="" +START_TIME="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + +spinner=(⠋ ⠙ ⠹ ⠸ ⠼ ⠴ ⠦ ⠧ ⠇ ⠏) +spin_i=0 +poll_interval=10 +spin_interval=0.15 +last_poll=0 + +# Parse arguments +for arg in "$@"; do + case $arg in + --ndri_workflow_branch=*) + NDRI_WORKFLOW_BRANCH="${arg#*=}" + ;; + --ndri_branch=*) + NDRI_BRANCH="${arg#*=}" + ;; + --ndr_workflow_branch=*) + NDR_WORKFLOW_BRANCH="${arg#*=}" + ;; + --ndr_branch=*) + NDR_BRANCH="${arg#*=}" + ;; + --sandbox_name=*) + SANDBOX_NAME="${arg#*=}" + ;; + --build_infra=*) + BUILD_INFRA="${arg#*=}" + ;; + --full_deploy=*) + FULL_DEPLOY="${arg#*=}" + ;; + --ndri_dir_loc_override=*) + NDRI_DIRECTORY="${arg#*=}" + ;; + *) + echo "Unknown argument: $arg" + echo "Usage: $0 [--ndri_workflow_branch=] [--ndri_branch=] [--ndr_branch=] [--ndr_workflow_branch=] [--sandbox_name=] [--build_infra=] [--ndri_dir_loc_override=]" + exit 1 + ;; + esac +done + +if [[ -z "$SANDBOX_NAME" ]]; then + branch=$(git rev-parse --abbrev-ref HEAD) + branch=$(echo "$branch" | sed 's/[^a-zA-Z0-9]//g') + branch="${branch,,}" + SANDBOX_NAME="$branch" +fi + +case "$SANDBOX_NAME" in +main | dev | ndr-dev | ndr-test | pre-prod | prod) + echo "Error: sandbox_name '$SANDBOX_NAME' is not allowed." + echo "Refusing to run against protected environments (main, dev, ndr-dev)." + exit 1 + ;; +esac + +if [[ "$BUILD_INFRA" == "true" ]]; then + echo "🏗️ Running infrastructure build" + cd "$NDRI_DIRECTORY" + echo "🔁 Triggering infrastructure workflow '$NDRI_WORKFLOW_FILE' from '$NDRI_WORKFLOW_BRANCH' with branch '$NDRI_BRANCH' to '$SANDBOX_NAME'..." + # Trigger the workflow and capture the run ID + gh workflow run "$NDRI_WORKFLOW_FILE" --ref "$NDRI_WORKFLOW_BRANCH" --field git_ref="$NDRI_BRANCH" --field sandbox_name="$SANDBOX_NAME" >/dev/null + + for i in {1..10}; do + run_id=$( + gh run list \ + --workflow "$NDRI_WORKFLOW_FILE" \ + --event workflow_dispatch \ + --json status,databaseId,createdAt,displayTitle \ + --jq ".[] + | select(.displayTitle == \"$NDRI_WORKFLOW_BRANCH | $SANDBOX_NAME\") + | select(.createdAt >= \"$START_TIME\") + | select(.status == \"queued\" or .status == \"in_progress\") + | .databaseId" | + head -n1 + ) + + [[ -n "$run_id" ]] && break + sleep 1 + done + + if [[ -z "$run_id" ]]; then + echo "❌ Could not find a workflow run to monitor." + exit 1 + fi + + echo "✅ Workflow triggered successfully (run ID: $run_id)" + echo "⏳ Monitoring workflow progress..." + + printf "\n" + + while true; do + now=$(date +%s) + + # Poll GitHub only every $poll_interval seconds + if ((now - last_poll >= poll_interval)); then + read -r status conclusion < <( + gh run view "$run_id" --json status,conclusion \ + -q '.status + " " + (.conclusion // "")' + ) + last_poll=$now + fi + + case "$status" in + queued) + printf "\r🕐 Deploy - Sandbox workflow queued... %s" "${spinner[spin_i++ % ${#spinner[@]}]}" + ;; + in_progress) + printf "\r🏃 Deploy - Sandbox workflow is in progress... %s" "${spinner[spin_i++ % ${#spinner[@]}]}" + ;; + completed) + printf "\r\033[K" + if [[ "$conclusion" == "success" ]]; then + echo "✅ Deploy - Sandbox workflow completed successfully." + printf "\n" + break + else + echo "❌ Deploy - Sandbox workflow failed with conclusion: $conclusion" + printf "\n" + exit 1 + fi + ;; + esac + + sleep "$spin_interval" + done +else + echo "🏃 Skipping infrastructure build" +fi + +echo "🏗️ Running Lambda deployment" +cd "$NDR_DIRECTORY" +if [[ "$FULL_DEPLOY" == "true" ]]; then + echo "🔁 Triggering Full Deploy to Sandbox workflow '$NDR_WORKFLOW_FILE_FULL' from '$NDR_WORKFLOW_BRANCH' with branch '$NDR_BRANCH' to '$SANDBOX_NAME'..." + # Trigger the workflow and capture the run ID + gh workflow run "$NDR_WORKFLOW_FILE_FULL" --ref "$NDR_WORKFLOW_BRANCH" --field build_branch="$NDR_BRANCH" --field sandbox="$SANDBOX_NAME" --field environment="development" >/dev/null + DISPLAY_TITLE="$NDR_BRANCH | $SANDBOX_NAME | development | false | true | true" + WORKFLOW_FILE="$NDR_WORKFLOW_FILE_FULL" +else + echo "🔁 Triggering Deploy lambdas to Sandbox workflow '$NDR_WORKFLOW_FILE' from '$NDR_WORKFLOW_BRANCH' with branch '$NDR_BRANCH' to '$SANDBOX_NAME'..." + # Trigger the workflow and capture the run ID + gh workflow run "$NDR_WORKFLOW_FILE" --ref "$NDR_WORKFLOW_BRANCH" --field build_branch="$NDR_BRANCH" --field sandbox="$SANDBOX_NAME" --field environment="development" >/dev/null + DISPLAY_TITLE="$NDR_BRANCH | $SANDBOX_NAME | development | true" + WORKFLOW_FILE="$NDR_WORKFLOW_FILE" +fi + +for i in {1..10}; do + lambda_run_id=$( + gh run list \ + --workflow "$WORKFLOW_FILE" \ + --event workflow_dispatch \ + --json status,databaseId,createdAt,displayTitle \ + --jq ".[] + | select(.displayTitle == \"$DISPLAY_TITLE\") + | select(.createdAt >= \"$START_TIME\") + | select(.status == \"queued\" or .status == \"in_progress\") + | .databaseId" | + head -n1 + ) + + [[ -n "$lambda_run_id" ]] && break + sleep 1 +done + +if [[ -z "$lambda_run_id" ]]; then + echo "❌ Could not find a Deploy to Sandbox workflow run to monitor." + exit 1 +fi + +echo "✅ Deploy to Sandbox workflow triggered successfully (run ID: $lambda_run_id)" +echo "⏳ Monitoring Deploy to Sandbox workflow progress..." + +spin_i=0 +last_poll=0 +printf "\n" + +while true; do + now=$(date +%s) + + # Poll GitHub only every $poll_interval seconds + if ((now - last_poll >= poll_interval)); then + read -r status conclusion < <( + gh run view "$lambda_run_id" --json status,conclusion \ + -q '.status + " " + (.conclusion // "")' + ) + last_poll=$now + fi + + case "$status" in + queued) + printf "\r🕐 Deploy to Sandbox workflow queued... %s" "${spinner[spin_i++ % ${#spinner[@]}]}" + ;; + in_progress) + printf "\r🏃 Deploy to Sandbox workflow in progress... %s" "${spinner[spin_i++ % ${#spinner[@]}]}" + ;; + completed) + printf "\r\033[K" + if [[ "$conclusion" == "success" ]]; then + echo "✅ Deploy to Sandbox workflow completed successfully." + break + else + echo "❌ Deploy to Sandbox workflow failed with conclusion: $conclusion" + exit 1 + fi + ;; + esac + + sleep "$spin_interval" +done