diff --git a/.github/actions/check-todo-usage/action.yaml b/.github/actions/check-todo-usage/action.yaml new file mode 100644 index 00000000..a403d588 --- /dev/null +++ b/.github/actions/check-todo-usage/action.yaml @@ -0,0 +1,10 @@ +name: "Check Todo usage" +description: "Check Todo usage" +runs: + using: "composite" + steps: + - name: "Check Todo usage" + shell: bash + run: | + export BRANCH_NAME=origin/${{ github.event.repository.default_branch }} + check=branch ./scripts/githooks/check-todos.sh diff --git a/.github/workflows/cicd-3-deploy.yaml b/.github/workflows/cicd-3-deploy.yaml index 383678ca..3c63d21b 100644 --- a/.github/workflows/cicd-3-deploy.yaml +++ b/.github/workflows/cicd-3-deploy.yaml @@ -48,7 +48,6 @@ jobs: echo "nodejs_version=$(grep "^nodejs\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT echo "python_version=$(grep "^python\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT echo "terraform_version=$(grep "^terraform\s" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT - # TODO: Get the version, but it may not be the .version file as this should come from the CI/CD Pull Request Workflow echo "version=$(head -n 1 .version 2> /dev/null || echo unknown)" >> $GITHUB_OUTPUT # echo "tag=${{ github.event.inputs.tag }}" >> $GITHUB_OUTPUT - name: "List variables" diff --git a/.github/workflows/scheduled-repository-template-sync.yaml b/.github/workflows/scheduled-repository-template-sync.yaml index 9df295a9..e9114865 100644 --- a/.github/workflows/scheduled-repository-template-sync.yaml +++ b/.github/workflows/scheduled-repository-template-sync.yaml @@ -27,7 +27,7 @@ jobs: - name: Run syncronisation script run: | - ./scripts/githooks/sync-template-repo.sh + ./nhs-notify-repository-template/scripts/githooks/sync-template-repo.sh rm -Rf ./nhs-notify-repository-template - name: Create Pull Request diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 8bf13f1d..df4947e5 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -27,17 +27,17 @@ jobs: # Needed to publish results and get a badge (see publish_results below). id-token: write # Uncomment the permissions below if installing in a private repository. - # contents: read - # actions: read + contents: read + actions: read steps: - name: "Checkout code" - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1 + uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2 with: results_file: results.sarif results_format: sarif @@ -45,7 +45,7 @@ jobs: # - you want to enable the Branch-Protection check on a *public* repository, or # - you are installing Scorecard on a *private* repository # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional. - # repo_token: ${{ secrets.SCORECARD_TOKEN }} + repo_token: ${{ secrets.SCORECARD_TOKEN }} # Public repositories: # - Publish results to OpenSSF REST API for easy access by consumers diff --git a/.github/workflows/stage-1-commit.yaml b/.github/workflows/stage-1-commit.yaml index e168ba96..ececcb36 100644 --- a/.github/workflows/stage-1-commit.yaml +++ b/.github/workflows/stage-1-commit.yaml @@ -36,7 +36,7 @@ jobs: scan-secrets: name: "Scan secrets" runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 @@ -47,7 +47,7 @@ jobs: check-file-format: name: "Check file format" runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 @@ -58,7 +58,7 @@ jobs: check-markdown-format: name: "Check Markdown format" runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 @@ -93,7 +93,7 @@ jobs: check-english-usage: name: "Check English usage" runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 @@ -101,6 +101,17 @@ jobs: fetch-depth: 0 # Full history is needed to compare branches - name: "Check English usage" uses: ./.github/actions/check-english-usage + check-todo-usage: + name: "Check TODO usage" + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: "Checkout code" + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history is needed to compare branches + - name: "Check TODO usage" + uses: ./.github/actions/check-todo-usage detect-terraform-changes: name: "Detect Terraform Changes" runs-on: ubuntu-latest @@ -127,7 +138,7 @@ jobs: lint-terraform: name: "Lint Terraform" runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 5 needs: detect-terraform-changes if: needs.detect-terraform-changes.outputs.terraform_changed == 'true' steps: @@ -145,7 +156,7 @@ jobs: - name: "Checkout code" uses: actions/checkout@v4 - name: "Setup ASDF" - uses: asdf-vm/actions/setup@v3 + uses: asdf-vm/actions/setup@v4 - name: "Perform Setup" uses: ./.github/actions/setup - name: "Trivy Scan" @@ -156,7 +167,7 @@ jobs: permissions: id-token: write contents: read - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 @@ -175,7 +186,7 @@ jobs: permissions: id-token: write contents: read - timeout-minutes: 2 + timeout-minutes: 5 steps: - name: "Checkout code" uses: actions/checkout@v4 diff --git a/.github/workflows/stage-3-build.yaml b/.github/workflows/stage-3-build.yaml index 2ffa10e2..f6115459 100644 --- a/.github/workflows/stage-3-build.yaml +++ b/.github/workflows/stage-3-build.yaml @@ -97,7 +97,7 @@ jobs: # - name: "Upload artefact 1" # run: | # echo "Uploading artefact 1 ..." - # # TODO: Use either action/cache or action/upload-artifact + # # Use either action/cache or action/upload-artifact # artefact-n: # name: "Artefact n" # runs-on: ubuntu-latest @@ -114,4 +114,4 @@ jobs: # - name: "Upload artefact n" # run: | # echo "Uploading artefact n ..." - # # TODO: Use either action/cache or action/upload-artifact + # # Use either action/cache or action/upload-artifact diff --git a/scripts/config/.repository-template-sync-ignore b/scripts/config/.repository-template-sync-ignore index dc208412..7b6cbfc1 100644 --- a/scripts/config/.repository-template-sync-ignore +++ b/scripts/config/.repository-template-sync-ignore @@ -4,19 +4,30 @@ nhs-notify-repository-template/ # Files and Folders in this repository to ignore .editorconfig .github/CODEOWNERS +.github/ISSUE_TEMPLATE +.github/workflows/cicd-*.yaml +.github/workflows/stage-*.yaml .gitleaksignore .vscode/ -/Makefile +Makefile CHANGELOG.md -README.md -VERSION project.code-workspace +README.md scripts/config/sonar-scanner.properties scripts/tests/ +VERSION # Files and Folders in the template repository to disregard .devcontainer/ -.github/workflows/cicd-*.yaml +.github/actions/build-docs +.github/workflows/*.disabled +*/examples/ docs/ +eslint.config.mjs infrastructure/terraform/components/ +lambdas/example-lambda/ +package-lock.json +package.json scripts/**/examples/ +scripts/terraform/terraform.mk +src/.vscode/ diff --git a/scripts/config/.repository-template-sync-merge b/scripts/config/.repository-template-sync-merge index 9ecb4bf9..471223e8 100644 --- a/scripts/config/.repository-template-sync-merge +++ b/scripts/config/.repository-template-sync-merge @@ -1,6 +1,9 @@ # Files and folders to merge when syncing nhs-notify-repository-template back in to this repository +.github/workflows/cicd-*.yaml +.github/workflows/stage-*.yaml scripts/config/.repository-template-sync-ignore scripts/config/.repository-template-sync-merge +scripts/config/pre-commit.yaml .tool-versions .gitignore scripts/config/vale/vale.ini diff --git a/scripts/config/markdownlint.yaml b/scripts/config/markdownlint.yaml index 29802df6..554ab554 100644 --- a/scripts/config/markdownlint.yaml +++ b/scripts/config/markdownlint.yaml @@ -1,4 +1,11 @@ +# SEE: https://github.com/DavidAnson/markdownlint/blob/main/schema/.markdownlint.yaml + +# https://github.com/DavidAnson/markdownlint/blob/main/doc/md013.md MD013: false + +# https://github.com/DavidAnson/markdownlint/blob/main/doc/md024.md MD024: siblings_only: true + +# https://github.com/DavidAnson/markdownlint/blob/main/doc/md033.md MD033: false diff --git a/scripts/config/pre-commit.yaml b/scripts/config/pre-commit.yaml index 81ca421e..1bd20f28 100644 --- a/scripts/config/pre-commit.yaml +++ b/scripts/config/pre-commit.yaml @@ -70,3 +70,10 @@ repos: entry: ./scripts/githooks/check-terraform-docs.sh language: script pass_filenames: false + - repo: local + hooks: + - id: check-todo-usage + name: Check TODO usage + entry: /usr/bin/env check=branch ./scripts/githooks/check-todos.sh + language: script + pass_filenames: false diff --git a/scripts/githooks/check-english-usage.sh b/scripts/githooks/check-english-usage.sh index 43e2bf23..b3942deb 100755 --- a/scripts/githooks/check-english-usage.sh +++ b/scripts/githooks/check-english-usage.sh @@ -25,6 +25,7 @@ set -euo pipefail # ============================================================================== function main() { + cd "$(git rev-parse --show-toplevel)" check=${check:-working-tree-changes} @@ -57,6 +58,7 @@ function main() { # Arguments (provided as environment variables): # filter=[git command to filter the files to check] function run-vale-natively() { + # shellcheck disable=SC2046 vale \ --config "$PWD/scripts/config/vale/vale.ini" \ @@ -67,29 +69,23 @@ function run-vale-natively() { # Arguments (provided as environment variables): # filter=[git command to filter the files to check] function run-vale-in-docker() { + # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh # shellcheck disable=SC2155 local image=$(name=jdkato/vale docker-get-image-version-and-pull) - - echo "Image is: $image" - echo "Filter is: $filter" # We use /dev/null here to stop `vale` from complaining that it's # not been called correctly if the $filter happens to return an # empty list. As long as there's a filename, even if it's one that # will be ignored, `vale` is happy. # shellcheck disable=SC2046,SC2086 - - set -x docker run --rm --platform linux/amd64 \ --volume "$PWD:/workdir" \ --workdir /workdir \ "$image" \ --config /workdir/scripts/config/vale/vale.ini \ $($filter) /dev/null - - set +x } # ============================================================================== diff --git a/scripts/githooks/check-file-format.sh b/scripts/githooks/check-file-format.sh index 7fe89c5d..d7c94747 100755 --- a/scripts/githooks/check-file-format.sh +++ b/scripts/githooks/check-file-format.sh @@ -89,6 +89,7 @@ function run-editorconfig-natively() { # dry_run_opt=[dry run option] # filter=[git command to filter the files to check] function run-editorconfig-in-docker() { + # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh @@ -97,12 +98,10 @@ function run-editorconfig-in-docker() { # We use /dev/null here as a backstop in case there are no files in the state # we choose. If the filter comes back empty, adding `/dev/null` onto it has # the effect of preventing `ec` from treating "no files" as "all the files". - set -x docker run --rm --platform linux/amd64 \ --volume "$PWD":/check \ "$image" \ - sh -c "ec --exclude '.git/' --config .editorconfig-checker.json $dry_run_opt \$($filter) /dev/null" - set +x + sh -c "ec --exclude '.git/' $dry_run_opt \$($filter) /dev/null" } # ============================================================================== diff --git a/scripts/githooks/check-markdown-format.sh b/scripts/githooks/check-markdown-format.sh index 89ae1c00..c39a080d 100755 --- a/scripts/githooks/check-markdown-format.sh +++ b/scripts/githooks/check-markdown-format.sh @@ -32,7 +32,7 @@ set -euo pipefail # ============================================================================== function main() { - echo "CHECKING MARKDOWN!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + cd "$(git rev-parse --show-toplevel)" check=${check:-working-tree-changes} @@ -64,7 +64,7 @@ function main() { # Arguments (provided as environment variables): # files=[files to check] function run-markdownlint-natively() { - echo "RUNNING MARKDOWN LINT NATIVELY" + # shellcheck disable=SC2086 markdownlint \ $files \ @@ -75,25 +75,18 @@ function run-markdownlint-natively() { # Arguments (provided as environment variables): # files=[files to check] function run-markdownlint-in-docker() { - echo "RUNNING MARKDOWN LINT IN DOCKER" + # shellcheck disable=SC1091 source ./scripts/docker/docker.lib.sh # shellcheck disable=SC2155 local image=$(name=ghcr.io/igorshubovych/markdownlint-cli docker-get-image-version-and-pull) # shellcheck disable=SC2086 - - echo "Config:" - cat scripts/config/markdownlint.yaml - - set -x docker run --rm --platform linux/amd64 \ --volume "$PWD":/workdir \ "$image" \ $files \ --config /workdir/scripts/config/markdownlint.yaml - - set +x } # ============================================================================== diff --git a/scripts/githooks/check-todos.sh b/scripts/githooks/check-todos.sh new file mode 100755 index 00000000..83b7a80e --- /dev/null +++ b/scripts/githooks/check-todos.sh @@ -0,0 +1,238 @@ +#!/bin/bash + +# WARNING: Please, DO NOT edit this file! It is maintained in the Repository Template (https://github.com/nhs-england-tools/repository-template). Raise a PR instead. + +set -euo pipefail + +# Pre-commit git hook to scan for secrets hard-coded in the codebase. This is a +# gitleaks command wrapper. It will run gitleaks natively if it is installed, +# otherwise it will run it in a Docker container. +# +# Usage: +# $ [options] ./scan-secrets.sh +# +# Options: +# check=all # check all files in the repository +# check=staged-changes # check only files staged for commit. +# check=working-tree-changes # check modified, unstaged files. This is the default. +# check=branch # check for all changes since branching from $BRANCH_NAME +# VERBOSE=true # Show all the executed commands, default is 'false' +# +# Exit codes: +# 0 - No Todos +# 1 - Todos found or error encountered +# 126 - Unknown flag + +# ============================================================================== + +EXCLUDED_FILES=( + ".devcontainer/devcontainer.json" + ".tool-versions" + ".vscode/extensions.json" + "infrastructure/terraform/bin/terraform.sh" + "Makefile" + "project.code-workspace" + "src/jekyll-devcontainer/src/.devcontainer/devcontainer.json" +) + +EXCLUDED_DIRS=( + ".git/" + ".venv/" + "docs/" + "node_modules/" +) + + +# Get files to check based on mode +function get_files_to_check() { + local mode="$1" + case "$mode" in + staged-changes) + git diff --diff-filter=ACMRT --name-only --cached # ACMRT only show files added, copied, modified, renamed or that had their type changed (eg. file → symlink) in this commit. This leaves out deleted files. + ;; + working-tree-changes) + git ls-files --others --exclude-standard && git diff --diff-filter=ACMRT --name-only + ;; + branch) + git diff --diff-filter=ACMRT --name-only ${BRANCH_NAME:-origin/main} + ;; + all) + git ls-files && git ls-files --others --exclude-standard + ;; + *) + echo "Unknown check mode: $mode" >&2 + exit 126 + ;; + esac +} + + +function build_exclude_args() { + local args=( + --exclude=".github/actions/check-todo-usage/action.yaml" + --exclude=".github/workflows/stage-1-commit.yaml" + --exclude="scripts/config/pre-commit.yaml" + --exclude="scripts/githooks/check-todos.sh" + ) # Exclude this script and its references by default, as it naturally contains TODOs. Todo todo todo <- see? + + if [ ${#EXCLUDED_DIRS[@]} -gt 0 ]; then + for dir in "${EXCLUDED_DIRS[@]}"; do + args+=(--exclude-dir="$dir") + done + fi + + if [ ${#EXCLUDED_FILES[@]} -gt 0 ]; then + for file in "${EXCLUDED_FILES[@]}"; do + args+=(--exclude="$file") + done + fi + echo "${args[@]}" +} + + +function search_todos() { + local mode="$1" + shift # Shift positional parameters so $@ contains only exclude_args + local exclude_args=("$@") + local todos="" + + local files + files=$(get_files_to_check "$mode") + # flatten files to unique list + files=$(echo "$files" | tr ' ' '\n' | sort -u) + + for file in $files; do + skip=false + + # Check if the file matches any exclude patterns + # Exclude files based on provided arguments and predefined directories + for ex in "${exclude_args[@]}"; do + if [[ "$ex" == --exclude* ]]; then + pattern=${ex#--exclude=} + [[ "$file" == $pattern ]] && skip=true && break + fi + done + + # Check if the file is in any of the excluded directories + for exdir in "${EXCLUDED_DIRS[@]}"; do + [[ "$file" == $exdir* ]] && skip=true && break + done + + # If the file is excluded, skip it + if [ "$skip" = false ] && [ -f "$file" ]; then + file_todos=$(grep -nHiE '\bTODO\b' "$file" || true) + [ -n "$file_todos" ] && todos+="$file_todos\n" + fi + done + + echo -e "$todos" +} + + +function filter_todos_with_valid_jira_ticket() { + local todos="$1" + local jira_regex="[A-Z][A-Z0-9]+-[0-9]+" + local todos_without_ticket="" + + while IFS= read -r line; do + # Only lines with TODO but without a valid JIRA ticket + if grep -qnHiE '\bTODO\b' <<< "$line"; then + if ! [[ "$line" =~ $jira_regex ]]; then + todos_without_ticket+="$line\n" + fi + fi + done <<< "$(echo -e "$todos")" + + # Output only TODOs without a valid JIRA ticket + echo -e "$todos_without_ticket" +} + + +function print_output() { + local todos="$1" + local exclude_args="$2" + local todo_count=$(line_count "$todos") + + echo "TODO Check Configuration:" + echo "=========================================" + echo " Check Mode: ${check:-working-tree-changes}" + echo " Total TODOs found: $todo_count" + + if [ ${#EXCLUDED_DIRS[@]} -gt 0 ]; then + echo " Excluded Directories: ${EXCLUDED_DIRS[*]}" + else + echo " Excluded Directories: (none)" + fi + + if [ ${#EXCLUDED_FILES[@]} -gt 0 ]; then + echo " Excluded Files: ${EXCLUDED_FILES[*]}" + else + echo " Excluded Files: (none)" + fi + + if is-arg-true "${VERBOSE:-false}"; then + echo "Grep Exclude Args: $exclude_args" + fi + + echo -e "\n=========================================" + echo "All TODOs found: $todo_count" + echo "=========================================" + + if [ "$todo_count" -gt 0 ]; then + echo "$todos" + else + echo "No TODOs found." + fi + + local results=$(filter_todos_with_valid_jira_ticket "$todos") + local results_count=$(line_count "$results") + + echo -e "\n=========================================" + echo "TODOs without a Jira ticket: $results_count" + echo "=========================================" + + if [ "$results_count" -gt 0 ]; then + echo "$results" + exit 1 + else + echo "No TODOs found without a Jira reference." + fi +} + + +function main() { + cd "$(git rev-parse --show-toplevel)" + + local check_mode="${check:-working-tree-changes}" + local exclude_args=$(build_exclude_args) + local todos=$(search_todos "$check_mode" $exclude_args) + print_output "$todos" "$exclude_args" +} + +# ============================================================================== + +# Count non-empty lines in a string +function line_count() { + local input="$1" + if [ -n "$input" ]; then + echo -e "$input" | wc -l + else + echo 0 + fi +} + +function is-arg-true() { + if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then + return 0 + else + return 1 + fi +} + +# ============================================================================== + +is-arg-true "${VERBOSE:-false}" && set -x + +main "$@" + +exit 0