From d87d8cf991e3974b9be3810e7d921a89d54c4bc1 Mon Sep 17 00:00:00 2001 From: Val Redchenko Date: Mon, 26 Jan 2026 18:22:09 +0000 Subject: [PATCH 1/2] feat: add secrets scanning workflow and pre-commit hooks - Add leaked-secrets-scan.yml workflow for daily and PR-based scanning - Move osv-scanner.toml from webui/ to repo root (consolidate config) - Add .pre-commit-config.yaml with detect-secrets hook - Add .secrets.baseline (no secrets detected) --- .github/workflows/leaked-secrets-scan.yml | 99 ++++++++++ .pre-commit-config.yaml | 15 ++ .secrets.baseline | 214 +++++++++++++++++++++ webui/osv-scanner.toml => osv-scanner.toml | 4 + 4 files changed, 332 insertions(+) create mode 100644 .github/workflows/leaked-secrets-scan.yml create mode 100644 .pre-commit-config.yaml create mode 100644 .secrets.baseline rename webui/osv-scanner.toml => osv-scanner.toml (60%) diff --git a/.github/workflows/leaked-secrets-scan.yml b/.github/workflows/leaked-secrets-scan.yml new file mode 100644 index 0000000..ad0c971 --- /dev/null +++ b/.github/workflows/leaked-secrets-scan.yml @@ -0,0 +1,99 @@ +name: Leaked Secrets Scan + +on: + workflow_call: + schedule: + - cron: '0 3 * * *' # Daily at 3 AM UTC (3-4 AM UK time) + workflow_dispatch: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + secrets-scan: + runs-on: ubuntu-latest + name: ${{ github.event_name == 'schedule' && 'Scheduled Secrets Scan' || 'Secrets Scan' }} + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Install detect-secrets + run: | + python -m pip install --upgrade pip + pip install detect-secrets + + - name: Verify baseline exists + run: | + if [ ! -f .secrets.baseline ]; then + echo "::error::.secrets.baseline not found!" + exit 1 + fi + echo "Found .secrets.baseline" + + - name: Scan for secrets + run: | + echo "Scanning for secrets..." + detect-secrets scan \ + --baseline .secrets.baseline \ + --exclude-files '.*\.lock$' \ + --force-use-all-plugins + + - name: Audit baseline for unaudited secrets + run: | + echo "Auditing secrets baseline..." + if grep -q '"is_secret": null' .secrets.baseline; then + echo "::error::Found unaudited secrets in baseline! Run: detect-secrets audit .secrets.baseline" + detect-secrets audit .secrets.baseline --report + exit 1 + fi + echo "All secrets in baseline have been audited" + detect-secrets audit .secrets.baseline --report + + - name: Check for new secrets in PR + if: github.event_name == 'pull_request' + run: | + echo "Checking for new secrets in PR..." + mkdir -p /tmp/pr-scan + git diff origin/main...HEAD --name-only | while read -r file; do + if [ -f "$file" ]; then + mkdir -p "/tmp/pr-scan/$(dirname "$file")" 2>/dev/null || true + cp "$file" "/tmp/pr-scan/$file" 2>/dev/null || true + fi + done + + if [ "$(ls -A /tmp/pr-scan 2>/dev/null)" ]; then + echo "Scanning changed files..." + detect-secrets scan \ + --baseline .secrets.baseline \ + --exclude-files '.*\.lock$' \ + --force-use-all-plugins \ + /tmp/pr-scan || echo "No new secrets found" + else + echo "No files to scan" + fi + + - name: Full repository scan (scheduled) + if: github.event_name == 'schedule' + run: | + echo "Performing full repository scan..." + detect-secrets scan \ + --exclude-files '.*\.lock$' \ + --force-use-all-plugins + + - name: Upload baseline on failure + uses: actions/upload-artifact@v4 + if: failure() + with: + name: secrets-scan-results + path: .secrets.baseline + retention-days: 30 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6fb5a36 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-yaml + args: ['--allow-multiple-documents'] + - id: check-merge-conflict + - id: end-of-file-fixer + + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + args: ['--baseline', '.secrets.baseline', '--exclude-files', '.*\.lock$'] diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..28a89c4 --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,214 @@ +{ + "version": "1.5.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "GitLabTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "OpenAIDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TelegramBotTokenDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + }, + { + "path": "detect_secrets.filters.regex.should_exclude_file", + "pattern": [ + ".*\\.lock$" + ] + } + ], + "results": { + "claude-code/smartem-frontend/skills/playwright/API_REFERENCE.md": [ + { + "type": "Secret Keyword", + "filename": "claude-code/smartem-frontend/skills/playwright/API_REFERENCE.md", + "hashed_secret": "f0578f1e7174b1a41c4ea8c6e17f7a8a3b88c92a", + "is_verified": false, + "line_number": 499 + }, + { + "type": "Secret Keyword", + "filename": "claude-code/smartem-frontend/skills/playwright/API_REFERENCE.md", + "hashed_secret": "8be52126a6fde450a7162a3651d589bb51e9579d", + "is_verified": false, + "line_number": 500 + } + ], + "docs/decision-records/smartem-workspace-developer-guide.md": [ + { + "type": "Secret Keyword", + "filename": "docs/decision-records/smartem-workspace-developer-guide.md", + "hashed_secret": "8bed7940e0c3ea61eca107fdb36282d4ceef45d4", + "is_verified": false, + "line_number": 1135 + }, + { + "type": "Secret Keyword", + "filename": "docs/decision-records/smartem-workspace-developer-guide.md", + "hashed_secret": "46192f4a6b473b1d14528bad8da6300e84cb4d0e", + "is_verified": false, + "line_number": 1140 + } + ], + "docs/development/e2e-simulation.md": [ + { + "type": "Basic Auth Credentials", + "filename": "docs/development/e2e-simulation.md", + "hashed_secret": "35675e68f4b5af7b995d9205ad0fc43842f16450", + "is_verified": false, + "line_number": 410 + } + ], + "docs/operations/kubernetes-secrets.md": [ + { + "type": "Secret Keyword", + "filename": "docs/operations/kubernetes-secrets.md", + "hashed_secret": "142879ed2e01ae92cf503554bfd2241c3e024a0f", + "is_verified": false, + "line_number": 115 + }, + { + "type": "Secret Keyword", + "filename": "docs/operations/kubernetes-secrets.md", + "hashed_secret": "3ba67f062408ecadaabf84c340240f69c1a0a318", + "is_verified": false, + "line_number": 117 + }, + { + "type": "Secret Keyword", + "filename": "docs/operations/kubernetes-secrets.md", + "hashed_secret": "3a12b92524e2bb602238d9f3e28f885d097668bb", + "is_verified": false, + "line_number": 318 + }, + { + "type": "Secret Keyword", + "filename": "docs/operations/kubernetes-secrets.md", + "hashed_secret": "76a3330d1ebe1e20c9824c59cafac491bcfa43c3", + "is_verified": false, + "line_number": 334 + } + ], + "k8s/secret.example.yaml": [ + { + "type": "Secret Keyword", + "filename": "k8s/secret.example.yaml", + "hashed_secret": "da19880411eca9df7c34f4f4690fb783e9c38d38", + "is_verified": false, + "line_number": 9 + } + ] + }, + "generated_at": "2026-01-26T18:21:21Z" +} diff --git a/webui/osv-scanner.toml b/osv-scanner.toml similarity index 60% rename from webui/osv-scanner.toml rename to osv-scanner.toml index 87671e4..b927a18 100644 --- a/webui/osv-scanner.toml +++ b/osv-scanner.toml @@ -1,3 +1,7 @@ +# OSV Scanner configuration +# https://google.github.io/osv-scanner/configuration/ + +# Migrated from webui/osv-scanner.toml [[IgnoredVulns]] id = "GHSA-73rr-hh4g-fpgx" reason = "Dev tooling only - diff is used by @tanstack/router-utils for build-time code generation, no user input reaches parsePatch/applyPatch" From e4afd9fa4d762cf83871ebd0a4ebd8242b2c70fa Mon Sep 17 00:00:00 2001 From: Val Redchenko Date: Fri, 30 Jan 2026 21:04:16 +0000 Subject: [PATCH 2/2] feat: add local vulnerability scanning scripts - osv-scan-repos.sh: multi-language scanner using osv-scanner Go binary - pip-audit-scan-repos.sh: Python-only scanner using uvx pip-audit Both scripts scan ERIC workspace repos by default and support markdown output via -o flag. Relates to #153 --- scripts/osv-scan-repos.sh | 164 ++++++++++++++++++++++++++++++++ scripts/pip-audit-scan-repos.sh | 158 ++++++++++++++++++++++++++++++ 2 files changed, 322 insertions(+) create mode 100755 scripts/osv-scan-repos.sh create mode 100755 scripts/pip-audit-scan-repos.sh diff --git a/scripts/osv-scan-repos.sh b/scripts/osv-scan-repos.sh new file mode 100755 index 0000000..0e115ef --- /dev/null +++ b/scripts/osv-scan-repos.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# +# Local OSV vulnerability scanner for ERIC workspace repos +# Scans specified repositories and outputs results to stdout or a file +# +# Usage: +# ./osv-scan-repos.sh # Scan default repos, output to stdout +# ./osv-scan-repos.sh -o results.md # Output as markdown to file +# ./osv-scan-repos.sh repo1 repo2 # Scan specific repo paths +# +# Requirements: +# - osv-scanner binary in PATH or at ~/.asdf/installs/golang/*/bin/osv-scanner + +set -euo pipefail + +# Find osv-scanner binary +find_osv_scanner() { + if command -v osv-scanner &>/dev/null; then + echo "osv-scanner" + return + fi + + # Check asdf golang installations + local asdf_bin + asdf_bin=$(find ~/.asdf/installs/golang -name "osv-scanner" -type f 2>/dev/null | head -1) + if [[ -n "$asdf_bin" ]]; then + echo "$asdf_bin" + return + fi + + echo "ERROR: osv-scanner not found. Install with: go install github.com/google/osv-scanner/cmd/osv-scanner@latest" >&2 + exit 1 +} + +OSV_SCANNER=$(find_osv_scanner) + +# Default repos relative to ERIC workspace root +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ERIC_ROOT="${SCRIPT_DIR}/../../../.." + +DEFAULT_REPOS=( + "repos/DiamondLightSource/smartem-decisions" + "repos/DiamondLightSource/smartem-frontend" + "repos/DiamondLightSource/smartem-devtools" + "repos/DiamondLightSource/fandanGO-cryoem-dls" + "repos/FragmentScreen/fandanGO-aria" + "repos/FragmentScreen/fandanGO-core" + "repos/FragmentScreen/fandanGO-cryoem-cnb" + "repos/FragmentScreen/fandanGO-nmr-cerm" + "repos/FragmentScreen/fandanGO-nmr-guf" + "repos/FragmentScreen/ddapi-record-logs" + "repos/FragmentScreen/ddapi-record-oscem" +) + +OUTPUT_FILE="" +REPOS=() + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + -o|--output) + OUTPUT_FILE="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [-o output.md] [repo_path ...]" + echo "" + echo "Options:" + echo " -o, --output FILE Write results to FILE (markdown format)" + echo " -h, --help Show this help" + echo "" + echo "If no repos specified, scans all ERIC workspace repos." + exit 0 + ;; + *) + REPOS+=("$1") + shift + ;; + esac +done + +# Use default repos if none specified +if [[ ${#REPOS[@]} -eq 0 ]]; then + for repo in "${DEFAULT_REPOS[@]}"; do + REPOS+=("${ERIC_ROOT}/${repo}") + done +fi + +scan_repo() { + local repo_path="$1" + local repo_name + repo_name=$(basename "$repo_path") + + if [[ ! -d "$repo_path" ]]; then + echo "SKIP: $repo_name (directory not found)" + return + fi + + echo "Scanning: $repo_name" + "$OSV_SCANNER" --recursive "$repo_path" 2>&1 || true +} + +scan_repo_markdown() { + local repo_path="$1" + local repo_name + repo_name=$(basename "$repo_path") + + if [[ ! -d "$repo_path" ]]; then + echo "### $repo_name" + echo "" + echo "**Status:** Skipped (directory not found)" + echo "" + return + fi + + echo "### $repo_name" + echo "" + + local output + output=$("$OSV_SCANNER" --recursive "$repo_path" 2>&1) || true + + if echo "$output" | grep -q "No issues found"; then + echo "**Status:** Clean - no vulnerabilities found" + elif echo "$output" | grep -q "OSV URL"; then + echo "**Status:** Vulnerabilities found" + echo "" + echo '```' + echo "$output" | grep -A1000 "OSV URL" | head -50 + echo '```' + else + echo "**Status:** No dependencies detected or scan error" + echo "" + echo '```' + echo "$output" | tail -10 + echo '```' + fi + echo "" +} + +# Run scans +if [[ -n "$OUTPUT_FILE" ]]; then + { + echo "# OSV Scanner Results" + echo "" + echo "**Scan date:** $(date -Iseconds)" + echo "" + echo "**Scanner version:** $("$OSV_SCANNER" --version 2>&1 | head -1)" + echo "" + echo "---" + echo "" + + for repo in "${REPOS[@]}"; do + scan_repo_markdown "$repo" + done + } > "$OUTPUT_FILE" + + echo "Results written to: $OUTPUT_FILE" +else + for repo in "${REPOS[@]}"; do + echo "========================================" + scan_repo "$repo" + echo "" + done +fi diff --git a/scripts/pip-audit-scan-repos.sh b/scripts/pip-audit-scan-repos.sh new file mode 100755 index 0000000..b6e4a69 --- /dev/null +++ b/scripts/pip-audit-scan-repos.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# +# Local pip-audit vulnerability scanner for ERIC workspace repos (Python only) +# Scans specified repositories and outputs results to stdout or a file +# Uses uvx to run pip-audit without permanent installation +# +# Usage: +# ./pip-audit-scan-repos.sh # Scan default repos, output to stdout +# ./pip-audit-scan-repos.sh -o results.md # Output as markdown to file +# ./pip-audit-scan-repos.sh repo1 repo2 # Scan specific repo paths +# +# Requirements: +# - uvx (from uv package manager) + +set -euo pipefail + +# Default repos relative to ERIC workspace root +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ERIC_ROOT="${SCRIPT_DIR}/../../../.." + +DEFAULT_REPOS=( + "repos/DiamondLightSource/smartem-decisions" + "repos/DiamondLightSource/smartem-frontend" + "repos/DiamondLightSource/smartem-devtools" + "repos/DiamondLightSource/fandanGO-cryoem-dls" + "repos/FragmentScreen/fandanGO-aria" + "repos/FragmentScreen/fandanGO-core" + "repos/FragmentScreen/fandanGO-cryoem-cnb" + "repos/FragmentScreen/fandanGO-nmr-cerm" + "repos/FragmentScreen/fandanGO-nmr-guf" + "repos/FragmentScreen/ddapi-record-logs" + "repos/FragmentScreen/ddapi-record-oscem" +) + +OUTPUT_FILE="" +REPOS=() + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + -o|--output) + OUTPUT_FILE="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [-o output.md] [repo_path ...]" + echo "" + echo "Options:" + echo " -o, --output FILE Write results to FILE (markdown format)" + echo " -h, --help Show this help" + echo "" + echo "If no repos specified, scans all ERIC workspace Python repos." + echo "Note: Only scans Python repos with requirements.txt files." + exit 0 + ;; + *) + REPOS+=("$1") + shift + ;; + esac +done + +# Use default repos if none specified +if [[ ${#REPOS[@]} -eq 0 ]]; then + for repo in "${DEFAULT_REPOS[@]}"; do + REPOS+=("${ERIC_ROOT}/${repo}") + done +fi + +scan_repo() { + local repo_path="$1" + local repo_name + repo_name=$(basename "$repo_path") + + if [[ ! -d "$repo_path" ]]; then + echo "SKIP: $repo_name (directory not found)" + return + fi + + if [[ ! -f "$repo_path/requirements.txt" ]]; then + echo "SKIP: $repo_name (no requirements.txt)" + return + fi + + echo "Scanning: $repo_name" + uvx pip-audit -r "$repo_path/requirements.txt" 2>&1 || true +} + +scan_repo_markdown() { + local repo_path="$1" + local repo_name + repo_name=$(basename "$repo_path") + + echo "### $repo_name" + echo "" + + if [[ ! -d "$repo_path" ]]; then + echo "**Status:** Skipped (directory not found)" + echo "" + return + fi + + if [[ ! -f "$repo_path/requirements.txt" ]]; then + echo "**Status:** Skipped (no requirements.txt - not a Python project or uses different dep format)" + echo "" + return + fi + + local output + output=$(uvx pip-audit -r "$repo_path/requirements.txt" 2>&1) || true + + if echo "$output" | grep -q "No known vulnerabilities found"; then + echo "**Status:** Clean - no vulnerabilities found" + elif echo "$output" | grep -q "found [0-9]* known vulnerabilit"; then + local vuln_count + vuln_count=$(echo "$output" | grep -oP "found \K[0-9]+" | head -1) + echo "**Status:** $vuln_count vulnerabilities found" + echo "" + echo '```' + echo "$output" + echo '```' + else + echo "**Status:** Scan completed" + echo "" + echo '```' + echo "$output" + echo '```' + fi + echo "" +} + +# Run scans +if [[ -n "$OUTPUT_FILE" ]]; then + { + echo "# pip-audit Scanner Results" + echo "" + echo "**Scan date:** $(date -Iseconds)" + echo "" + echo "**Scanner:** pip-audit (via uvx)" + echo "" + echo "**Note:** pip-audit only scans Python dependencies (requirements.txt). PHP repos are skipped." + echo "" + echo "---" + echo "" + + for repo in "${REPOS[@]}"; do + scan_repo_markdown "$repo" + done + } > "$OUTPUT_FILE" + + echo "Results written to: $OUTPUT_FILE" +else + for repo in "${REPOS[@]}"; do + echo "========================================" + scan_repo "$repo" + echo "" + done +fi