From 86deac2abe40a21f58e6f8992478e12cea270022 Mon Sep 17 00:00:00 2001 From: Ammar Date: Thu, 4 Dec 2025 11:25:13 -0600 Subject: [PATCH 1/6] =?UTF-8?q?=F0=9F=A4=96=20feat:=20add=20selective=20in?= =?UTF-8?q?tegration=20test=20execution=20system?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements a coverage-based test selection system to reduce CI time by running only the integration tests affected by code changes. Components: - generate-coverage-map.ts: Runs tests individually with coverage to build a reverse index (source file → tests that cover it) - select-affected-tests.ts: Selects tests based on changed files, with safe fallbacks for infrastructure changes, new tests, and unmapped files - coverage-map.yml: Daily workflow to regenerate the map and cache it - Updated ci.yml: Restores map from cache and runs selective tests Safety features: - Falls back to all tests when map is missing/stale (>7 days) - Infrastructure files (jest config, package.json, etc.) trigger all tests - New/unmapped source files trigger all tests - Exit code 2 signals fallback, ensuring CI always runs correct tests _Generated with `mux`_ --- .github/workflows/ci.yml | 56 ++- .github/workflows/coverage-map.yml | 83 +++ scripts/selective-tests/README.md | 107 ++++ .../selective-tests/generate-coverage-map.ts | 244 +++++++++ .../selective-tests/run-selective-tests.sh | 132 +++++ .../selective-tests/select-affected-tests.ts | 472 ++++++++++++++++++ scripts/selective-tests/types.ts | 77 +++ 7 files changed, 1169 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/coverage-map.yml create mode 100644 scripts/selective-tests/README.md create mode 100644 scripts/selective-tests/generate-coverage-map.ts create mode 100755 scripts/selective-tests/run-selective-tests.sh create mode 100644 scripts/selective-tests/select-affected-tests.ts create mode 100644 scripts/selective-tests/types.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc13a60377..c4f407a959 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,14 +100,66 @@ jobs: - name: Checkout code uses: actions/checkout@v4 with: - fetch-depth: 0 # Required for git describe to find tags + fetch-depth: 0 # Required for git describe to find tags and diff - uses: ./.github/actions/setup-mux - name: Build worker files run: make build-main - - name: Run all integration tests with coverage + # Try to restore coverage map from cache for selective testing + - name: Restore coverage map + if: github.event.inputs.test_filter == '' + id: coverage-map-cache + uses: actions/cache/restore@v4 + with: + path: coverage-map.json + key: coverage-map-${{ github.sha }} + restore-keys: | + coverage-map-latest- + coverage-map- + + - name: Select affected tests + if: github.event.inputs.test_filter == '' && steps.coverage-map-cache.outputs.cache-hit != '' + id: select-tests + run: | + # Run selection script and capture output + set +e + SELECTED=$(bun scripts/selective-tests/select-affected-tests.ts \ + --map coverage-map.json \ + --base origin/${{ github.base_ref || 'main' }} \ + --head ${{ github.sha }} \ + --output jest \ + --verbose) + EXIT_CODE=$? + set -e + + echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT + + if [[ $EXIT_CODE -eq 0 ]]; then + echo "selected_tests=$SELECTED" >> $GITHUB_OUTPUT + echo "run_selective=true" >> $GITHUB_OUTPUT + else + echo "run_selective=false" >> $GITHUB_OUTPUT + fi + + - name: Run selective integration tests + if: steps.select-tests.outputs.run_selective == 'true' && steps.select-tests.outputs.selected_tests != '--testPathPattern=^$' + run: | + echo "Running selective tests: ${{ steps.select-tests.outputs.selected_tests }}" + TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent ${{ steps.select-tests.outputs.selected_tests }} + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Skip tests (no affected tests) + if: steps.select-tests.outputs.run_selective == 'true' && steps.select-tests.outputs.selected_tests == '--testPathPattern=^$' + run: | + echo "::notice::No integration tests affected by changes - skipping" + echo "No integration tests were affected by the changes in this PR." >> $GITHUB_STEP_SUMMARY + + - name: Run all integration tests (fallback or manual filter) + if: steps.select-tests.outputs.run_selective != 'true' # --silent suppresses per-test output (17+ test files × workers = overwhelming logs) run: TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent ${{ github.event.inputs.test_filter || 'tests' }} env: diff --git a/.github/workflows/coverage-map.yml b/.github/workflows/coverage-map.yml new file mode 100644 index 0000000000..3660b8d2d1 --- /dev/null +++ b/.github/workflows/coverage-map.yml @@ -0,0 +1,83 @@ +name: Coverage Map Generation + +on: + # Run daily at 2 AM UTC + schedule: + - cron: "0 2 * * *" + # Allow manual trigger + workflow_dispatch: + # Also regenerate when the selective test system changes + push: + branches: [main] + paths: + - "scripts/selective-tests/**" + - "tests/integration/**" + - ".github/workflows/coverage-map.yml" + +concurrency: + group: coverage-map-${{ github.ref }} + cancel-in-progress: true + +jobs: + generate-coverage-map: + name: Generate Coverage Map + runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} + timeout-minutes: 60 + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: ./.github/actions/setup-mux + + - name: Build worker files + run: make build-main + + - name: Generate coverage map + run: bun scripts/selective-tests/generate-coverage-map.ts --output coverage-map.json + env: + TEST_INTEGRATION: "1" + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Upload coverage map artifact + uses: actions/upload-artifact@v4 + with: + name: coverage-map + path: coverage-map.json + retention-days: 30 + + - name: Cache coverage map for CI + uses: actions/cache/save@v4 + with: + path: coverage-map.json + key: coverage-map-${{ github.sha }} + + # Also save with a "latest" key that PRs can restore from + - name: Cache coverage map (latest) + uses: actions/cache/save@v4 + with: + path: coverage-map.json + key: coverage-map-latest-${{ github.run_id }} + # This creates a new cache entry each run, but restore-keys in CI + # will match the most recent one + + - name: Summary + run: | + echo "## Coverage Map Generated" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "- **Commit:** ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY + echo "- **Generated at:** $(date -u)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "### Map Statistics" >> $GITHUB_STEP_SUMMARY + echo '```json' >> $GITHUB_STEP_SUMMARY + jq '{ + version, + generatedAt, + commitSha, + testCount: (.allTests | length), + filesWithCoverage: ([.fileToTests | to_entries[] | select(.value | length > 0)] | length), + totalMappings: ([.fileToTests | to_entries[] | .value | length] | add) + }' coverage-map.json >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY diff --git a/scripts/selective-tests/README.md b/scripts/selective-tests/README.md new file mode 100644 index 0000000000..c2bb3cb712 --- /dev/null +++ b/scripts/selective-tests/README.md @@ -0,0 +1,107 @@ +# Selective Test Execution System + +This system reduces CI time by running only the integration tests affected by code changes, rather than the full test suite on every PR. + +## How It Works + +1. **Coverage Map Generation**: A daily workflow runs all integration tests individually with code coverage, building a reverse index from source files to tests. + +2. **Affected Test Selection**: When a PR runs, the system: + - Restores the coverage map from cache + - Identifies changed files via git diff + - Looks up which tests cover those files + - Runs only the affected tests + +3. **Safe Fallbacks**: The system falls back to running all tests when: + - Coverage map is missing or stale (>7 days) + - Infrastructure files change (jest.config, package.json, etc.) + - New test files are added + - Changed source files aren't in the coverage map + - Any error occurs during selection + +## Files + +- `types.ts` - Shared types and constants +- `generate-coverage-map.ts` - Generates the coverage map by running tests with coverage +- `select-affected-tests.ts` - Selects tests based on changed files +- `run-selective-tests.sh` - CI wrapper script with fallback handling + +## Usage + +### Generate Coverage Map (local testing) + +```bash +bun scripts/selective-tests/generate-coverage-map.ts --output coverage-map.json +``` + +This takes ~30-60 minutes as it runs each test file individually. + +### Select Affected Tests + +```bash +# Using git diff +bun scripts/selective-tests/select-affected-tests.ts \ + --map coverage-map.json \ + --base origin/main \ + --head HEAD \ + --output jest + +# Using explicit file list +bun scripts/selective-tests/select-affected-tests.ts \ + --map coverage-map.json \ + --changed "src/node/services/workspaceService.ts,src/node/config.ts" \ + --output json +``` + +### Output Formats + +- `jest` - Space-separated test files for Jest CLI, or `tests` for all tests +- `json` - Full result object with reasoning +- `list` - Newline-separated test files, or `ALL` for all tests + +### Exit Codes + +- `0` - Selection successful (may be empty test list) +- `2` - Fallback triggered (run all tests) +- `1` - Error + +## CI Integration + +The system integrates with `.github/workflows/ci.yml`: + +1. Restores coverage map from cache +2. Runs selection script +3. Either runs selected tests or falls back to all tests +4. Skips tests entirely if no tests are affected + +The coverage map is regenerated daily by `.github/workflows/coverage-map.yml` and cached for PR use. + +## Infrastructure Files + +These files trigger a full test run when changed (see `INFRASTRUCTURE_PATTERNS` in `types.ts`): + +- Test configuration: `jest.config.cjs`, `babel.config.cjs` +- Build config: `tsconfig.json`, `package.json`, `bun.lockb` +- Test infrastructure: `tests/setup.ts`, `tests/integration/helpers.ts` +- Service container: `src/node/services/serviceContainer.ts` +- Shared types: `src/types/**`, `src/constants/**` + +## Shadow Mode + +For validation, run with `--shadow-mode` to log what would have been selected while still running all tests: + +```bash +./scripts/selective-tests/run-selective-tests.sh --shadow-mode +``` + +## Debugging + +Use `--verbose` for detailed logging: + +```bash +bun scripts/selective-tests/select-affected-tests.ts \ + --map coverage-map.json \ + --changed "src/node/services/aiService.ts" \ + --output json \ + --verbose +``` diff --git a/scripts/selective-tests/generate-coverage-map.ts b/scripts/selective-tests/generate-coverage-map.ts new file mode 100644 index 0000000000..afd3fb31ae --- /dev/null +++ b/scripts/selective-tests/generate-coverage-map.ts @@ -0,0 +1,244 @@ +#!/usr/bin/env bun +/** + * Generate a coverage map by running each integration test individually + * and recording which source files it covers. + * + * Usage: bun scripts/selective-tests/generate-coverage-map.ts [--output coverage-map.json] + * + * This script: + * 1. Discovers all integration test files + * 2. Runs each test individually with coverage enabled + * 3. Parses the coverage output to extract covered files + * 4. Builds a reverse index: source file → tests that cover it + * 5. Outputs a JSON coverage map + */ + +import { execSync, spawnSync } from "child_process"; +import * as fs from "fs"; +import * as path from "path"; +import { createHash } from "crypto"; +import type { CoverageMap } from "./types"; +import { INFRASTRUCTURE_PATTERNS } from "./types"; + +const TESTS_DIR = "tests/integration"; +const COVERAGE_DIR = "coverage"; +const DEFAULT_OUTPUT = "coverage-map.json"; + +function log(message: string): void { + console.error(`[generate-coverage-map] ${message}`); +} + +function getGitCommitSha(): string { + try { + return execSync("git rev-parse HEAD", { encoding: "utf-8" }).trim(); + } catch { + return "unknown"; + } +} + +function hashFiles(files: string[]): string { + const hash = createHash("sha256"); + for (const file of files.sort()) { + if (fs.existsSync(file)) { + hash.update(file); + hash.update(fs.readFileSync(file)); + } + } + return hash.digest("hex").substring(0, 16); +} + +function discoverTestFiles(): string[] { + const testFiles: string[] = []; + const entries = fs.readdirSync(TESTS_DIR, { withFileTypes: true }); + + for (const entry of entries) { + if (entry.isFile() && entry.name.endsWith(".test.ts")) { + testFiles.push(path.join(TESTS_DIR, entry.name)); + } + } + + return testFiles.sort(); +} + +function discoverSourceFiles(): string[] { + const sourceFiles: string[] = []; + + function walkDir(dir: string): void { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory() && !entry.name.startsWith(".")) { + walkDir(fullPath); + } else if (entry.isFile() && entry.name.endsWith(".ts")) { + sourceFiles.push(fullPath); + } + } + } + + walkDir("src"); + return sourceFiles.sort(); +} + +interface CoverageData { + [filePath: string]: { + s: Record; // statements + f?: Record; // functions + b?: Record; // branches + }; +} + +function extractCoveredFiles(coverageJsonPath: string): string[] { + if (!fs.existsSync(coverageJsonPath)) { + return []; + } + + const coverage: CoverageData = JSON.parse( + fs.readFileSync(coverageJsonPath, "utf-8") + ); + const coveredFiles: string[] = []; + + for (const [filePath, data] of Object.entries(coverage)) { + // Check if any statement was executed + const hasExecutedStatements = Object.values(data.s).some( + (count) => count > 0 + ); + if (hasExecutedStatements) { + // Convert absolute path to relative + const relativePath = path.relative(process.cwd(), filePath); + if (relativePath.startsWith("src/")) { + coveredFiles.push(relativePath); + } + } + } + + return coveredFiles; +} + +function runTestWithCoverage(testFile: string): string[] { + // Clean coverage directory first + if (fs.existsSync(COVERAGE_DIR)) { + fs.rmSync(COVERAGE_DIR, { recursive: true }); + } + + log(`Running: ${testFile}`); + + const result = spawnSync( + "bun", + [ + "x", + "jest", + "--coverage", + "--coverageReporters=json", + "--maxWorkers=1", + "--silent", + "--forceExit", + testFile, + ], + { + env: { + ...process.env, + TEST_INTEGRATION: "1", + // Disable color output for cleaner logs + FORCE_COLOR: "0", + }, + stdio: ["ignore", "pipe", "pipe"], + timeout: 5 * 60 * 1000, // 5 minute timeout per test + } + ); + + if (result.error) { + log(` Error running test: ${result.error.message}`); + return []; + } + + // Extract covered files from coverage-final.json + const coverageJsonPath = path.join(COVERAGE_DIR, "coverage-final.json"); + const coveredFiles = extractCoveredFiles(coverageJsonPath); + log(` Covered ${coveredFiles.length} source files`); + + return coveredFiles; +} + +function buildCoverageMap( + testFiles: string[], + sourceFiles: string[] +): CoverageMap { + const fileToTests: Record = {}; + const commitSha = getGitCommitSha(); + const sourceHash = hashFiles(sourceFiles); + + // Initialize all source files with empty arrays + for (const sourceFile of sourceFiles) { + fileToTests[sourceFile] = []; + } + + // Run each test and record coverage + for (let i = 0; i < testFiles.length; i++) { + const testFile = testFiles[i]; + log(`[${i + 1}/${testFiles.length}] Processing ${testFile}`); + + const coveredFiles = runTestWithCoverage(testFile); + + for (const sourceFile of coveredFiles) { + if (!fileToTests[sourceFile]) { + fileToTests[sourceFile] = []; + } + if (!fileToTests[sourceFile].includes(testFile)) { + fileToTests[sourceFile].push(testFile); + } + } + } + + return { + version: 1, + generatedAt: new Date().toISOString(), + commitSha, + sourceHash, + fileToTests, + allTests: testFiles, + infrastructureFiles: INFRASTRUCTURE_PATTERNS, + }; +} + +function main(): void { + const args = process.argv.slice(2); + let outputPath = DEFAULT_OUTPUT; + + for (let i = 0; i < args.length; i++) { + if (args[i] === "--output" && args[i + 1]) { + outputPath = args[i + 1]; + i++; + } + } + + log("Discovering test files..."); + const testFiles = discoverTestFiles(); + log(`Found ${testFiles.length} integration test files`); + + log("Discovering source files..."); + const sourceFiles = discoverSourceFiles(); + log(`Found ${sourceFiles.length} source files`); + + log("Building coverage map (this may take a while)..."); + const coverageMap = buildCoverageMap(testFiles, sourceFiles); + + // Write output + fs.writeFileSync(outputPath, JSON.stringify(coverageMap, null, 2)); + log(`Coverage map written to ${outputPath}`); + + // Print summary + const totalMappings = Object.values(coverageMap.fileToTests).reduce( + (sum, tests) => sum + tests.length, + 0 + ); + const filesWithCoverage = Object.values(coverageMap.fileToTests).filter( + (tests) => tests.length > 0 + ).length; + + log(`Summary:`); + log(` Total test files: ${coverageMap.allTests.length}`); + log(` Source files with coverage: ${filesWithCoverage}`); + log(` Total file→test mappings: ${totalMappings}`); +} + +main(); diff --git a/scripts/selective-tests/run-selective-tests.sh b/scripts/selective-tests/run-selective-tests.sh new file mode 100755 index 0000000000..3996b86198 --- /dev/null +++ b/scripts/selective-tests/run-selective-tests.sh @@ -0,0 +1,132 @@ +#!/bin/bash +# Run integration tests selectively based on changed files. +# +# This script wraps the TypeScript selection logic and handles CI-specific +# concerns like caching, fallback behavior, and output formatting. +# +# Usage: ./scripts/selective-tests/run-selective-tests.sh [--verbose] [--shadow-mode] +# +# Environment variables: +# COVERAGE_MAP_PATH Path to coverage map (default: coverage-map.json) +# FORCE_ALL_TESTS If set to "true", skip selection and run all tests +# GITHUB_BASE_REF Base ref for PR comparison (set by GitHub Actions) +# GITHUB_HEAD_REF Head ref for PR comparison (set by GitHub Actions) +# +# Exit codes: +# 0 - Tests passed (or no tests to run) +# 1 - Tests failed or error +# 2 - Fallback to all tests was triggered (informational in shadow mode) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# Defaults +COVERAGE_MAP_PATH="${COVERAGE_MAP_PATH:-$PROJECT_ROOT/coverage-map.json}" +FORCE_ALL_TESTS="${FORCE_ALL_TESTS:-false}" +VERBOSE="" +SHADOW_MODE="" + +# Parse arguments +while [[ $# -gt 0 ]]; do + case $1 in + --verbose) + VERBOSE="--verbose" + shift + ;; + --shadow-mode) + SHADOW_MODE="true" + shift + ;; + *) + shift + ;; + esac +done + +log() { + echo "[run-selective-tests] $1" >&2 +} + +# Determine git refs for comparison +if [[ -n "${GITHUB_BASE_REF:-}" ]]; then + BASE_REF="origin/$GITHUB_BASE_REF" +else + BASE_REF="origin/main" +fi + +if [[ -n "${GITHUB_SHA:-}" ]]; then + HEAD_REF="$GITHUB_SHA" +else + HEAD_REF="HEAD" +fi + +log "Base ref: $BASE_REF" +log "Head ref: $HEAD_REF" +log "Coverage map: $COVERAGE_MAP_PATH" + +# Check for force-all flag +FORCE_FLAG="" +if [[ "$FORCE_ALL_TESTS" == "true" ]]; then + log "FORCE_ALL_TESTS is set, will run all tests" + FORCE_FLAG="--force-all" +fi + +# Run the selection script +log "Selecting affected tests..." + +set +e +SELECTED_TESTS=$(bun "$SCRIPT_DIR/select-affected-tests.ts" \ + --map "$COVERAGE_MAP_PATH" \ + --base "$BASE_REF" \ + --head "$HEAD_REF" \ + --output jest \ + $VERBOSE \ + $FORCE_FLAG) +SELECT_EXIT_CODE=$? +set -e + +log "Selection exit code: $SELECT_EXIT_CODE" +log "Selected tests: $SELECTED_TESTS" + +# Handle shadow mode - run selection but always run all tests +if [[ -n "$SHADOW_MODE" ]]; then + log "Shadow mode enabled - will run all tests regardless of selection" + + # Log what would have happened + if [[ $SELECT_EXIT_CODE -eq 0 ]]; then + log "SHADOW: Would have run selective tests: $SELECTED_TESTS" + else + log "SHADOW: Would have fallen back to all tests (exit code $SELECT_EXIT_CODE)" + fi + + # Run all tests + log "Running all integration tests..." + TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent tests + exit $? +fi + +# Handle the selection result +if [[ $SELECT_EXIT_CODE -eq 2 ]]; then + # Fallback triggered - run all tests + log "Fallback triggered, running all integration tests..." + TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent tests + exit $? +elif [[ $SELECT_EXIT_CODE -ne 0 ]]; then + # Error in selection script + log "Error in selection script (exit code $SELECT_EXIT_CODE), falling back to all tests" + TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent tests + exit $? +fi + +# Check if no tests need to run +if [[ "$SELECTED_TESTS" == "--testPathPattern=^$" ]]; then + log "No tests affected by changes, skipping integration tests" + echo "::notice::No integration tests affected by changes - skipping" + exit 0 +fi + +# Run selected tests +log "Running selected tests: $SELECTED_TESTS" +TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent $SELECTED_TESTS diff --git a/scripts/selective-tests/select-affected-tests.ts b/scripts/selective-tests/select-affected-tests.ts new file mode 100644 index 0000000000..cb6c3c60b8 --- /dev/null +++ b/scripts/selective-tests/select-affected-tests.ts @@ -0,0 +1,472 @@ +#!/usr/bin/env bun +/** + * Select which integration tests to run based on changed files. + * + * Usage: + * bun scripts/selective-tests/select-affected-tests.ts [options] + * + * Options: + * --map Path to coverage map JSON (default: coverage-map.json) + * --base Git base ref for comparison (default: origin/main) + * --head Git head ref for comparison (default: HEAD) + * --changed Comma-separated list of changed files (overrides git diff) + * --output Output format: json, list, or jest (default: jest) + * --max-staleness Maximum map age in days (default: 7) + * --force-all Force running all tests (for debugging) + * --verbose Enable verbose logging + * + * Exit codes: + * 0 - Success, selective tests determined + * 2 - Fallback triggered, run all tests + * 1 - Error + * + * Output (stdout): + * - json: Full AffectedTestsResult as JSON + * - list: Newline-separated list of test files + * - jest: Space-separated test files suitable for jest CLI + */ + +import { execSync } from "child_process"; +import * as fs from "fs"; +import type { CoverageMap, AffectedTestsResult } from "./types"; +import { EXIT_CODES, INFRASTRUCTURE_PATTERNS } from "./types"; + +// minimatch is CommonJS, need to use require +// eslint-disable-next-line @typescript-eslint/no-require-imports +const minimatch = require("minimatch") as ( + file: string, + pattern: string, + options?: { matchBase?: boolean } +) => boolean; + +const DEFAULT_MAP_PATH = "coverage-map.json"; +const DEFAULT_BASE_REF = "origin/main"; +const DEFAULT_HEAD_REF = "HEAD"; +const DEFAULT_MAX_STALENESS_DAYS = 7; + +interface Options { + mapPath: string; + baseRef: string; + headRef: string; + changedFiles: string[] | null; + outputFormat: "json" | "list" | "jest"; + maxStalenessDays: number; + forceAll: boolean; + verbose: boolean; +} + +function log(message: string, verbose: boolean, force = false): void { + if (verbose || force) { + console.error(`[select-affected-tests] ${message}`); + } +} + +function parseArgs(): Options { + const args = process.argv.slice(2); + const options: Options = { + mapPath: DEFAULT_MAP_PATH, + baseRef: DEFAULT_BASE_REF, + headRef: DEFAULT_HEAD_REF, + changedFiles: null, + outputFormat: "jest", + maxStalenessDays: DEFAULT_MAX_STALENESS_DAYS, + forceAll: false, + verbose: false, + }; + + for (let i = 0; i < args.length; i++) { + switch (args[i]) { + case "--map": + options.mapPath = args[++i]; + break; + case "--base": + options.baseRef = args[++i]; + break; + case "--head": + options.headRef = args[++i]; + break; + case "--changed": + options.changedFiles = args[++i].split(",").filter(Boolean); + break; + case "--output": + options.outputFormat = args[++i] as "json" | "list" | "jest"; + break; + case "--max-staleness": + options.maxStalenessDays = parseInt(args[++i], 10); + break; + case "--force-all": + options.forceAll = true; + break; + case "--verbose": + options.verbose = true; + break; + } + } + + return options; +} + +function getChangedFiles(baseRef: string, headRef: string): string[] { + try { + // First, try to get the merge base for more accurate diffing + const mergeBase = execSync(`git merge-base ${baseRef} ${headRef}`, { + encoding: "utf-8", + }).trim(); + + const output = execSync(`git diff --name-only ${mergeBase} ${headRef}`, { + encoding: "utf-8", + }); + + return output + .split("\n") + .map((f) => f.trim()) + .filter(Boolean); + } catch { + // Fallback to direct diff if merge-base fails + try { + const output = execSync(`git diff --name-only ${baseRef} ${headRef}`, { + encoding: "utf-8", + }); + return output + .split("\n") + .map((f) => f.trim()) + .filter(Boolean); + } catch { + return []; + } + } +} + +function matchesPattern(file: string, pattern: string): boolean { + // Handle glob patterns + if (pattern.includes("*")) { + return minimatch(file, pattern, { matchBase: true }); + } + // Exact match or directory prefix + return file === pattern || file.startsWith(pattern + "/"); +} + +function isInfrastructureFile( + file: string, + patterns: string[] = INFRASTRUCTURE_PATTERNS +): boolean { + return patterns.some((pattern) => matchesPattern(file, pattern)); +} + +function isNewTestFile(file: string, coverageMap: CoverageMap): boolean { + return ( + file.startsWith("tests/integration/") && + file.endsWith(".test.ts") && + !coverageMap.allTests.includes(file) + ); +} + +function loadCoverageMap(mapPath: string): CoverageMap | null { + if (!fs.existsSync(mapPath)) { + return null; + } + + try { + const content = fs.readFileSync(mapPath, "utf-8"); + return JSON.parse(content) as CoverageMap; + } catch { + return null; + } +} + +function isMapStale(map: CoverageMap, maxDays: number): boolean { + const generatedAt = new Date(map.generatedAt); + const now = new Date(); + const ageMs = now.getTime() - generatedAt.getTime(); + const ageDays = ageMs / (1000 * 60 * 60 * 24); + return ageDays > maxDays; +} + +function selectAffectedTests( + options: Options +): AffectedTestsResult & { exitCode: number } { + const verbose = options.verbose; + + // Check for force-all flag + if (options.forceAll) { + log("Force-all flag set, running all tests", verbose, true); + return { + runAll: true, + reason: "Force-all flag set", + tests: [], + changedFiles: [], + unmappedFiles: [], + exitCode: EXIT_CODES.FALLBACK_TRIGGERED, + }; + } + + // Load coverage map + const coverageMap = loadCoverageMap(options.mapPath); + + if (!coverageMap) { + log(`Coverage map not found at ${options.mapPath}`, verbose, true); + return { + runAll: true, + reason: `Coverage map not found at ${options.mapPath}`, + tests: [], + changedFiles: [], + unmappedFiles: [], + exitCode: EXIT_CODES.FALLBACK_TRIGGERED, + }; + } + + log(`Loaded coverage map from ${options.mapPath}`, verbose); + log(` Generated: ${coverageMap.generatedAt}`, verbose); + log(` Commit: ${coverageMap.commitSha}`, verbose); + log(` Tests: ${coverageMap.allTests.length}`, verbose); + + // Check map staleness + if (isMapStale(coverageMap, options.maxStalenessDays)) { + log( + `Coverage map is stale (> ${options.maxStalenessDays} days old)`, + verbose, + true + ); + return { + runAll: true, + reason: `Coverage map is stale (> ${options.maxStalenessDays} days old)`, + tests: [], + changedFiles: [], + unmappedFiles: [], + exitCode: EXIT_CODES.FALLBACK_TRIGGERED, + }; + } + + // Get changed files + const changedFiles = + options.changedFiles ?? getChangedFiles(options.baseRef, options.headRef); + + if (changedFiles.length === 0) { + log("No changed files detected", verbose, true); + return { + runAll: false, + reason: "No changed files detected", + tests: [], + changedFiles: [], + unmappedFiles: [], + exitCode: EXIT_CODES.SUCCESS, + }; + } + + log(`Changed files (${changedFiles.length}):`, verbose); + for (const file of changedFiles) { + log(` ${file}`, verbose); + } + + // Check for infrastructure files + const infrastructurePatterns = + coverageMap.infrastructureFiles || INFRASTRUCTURE_PATTERNS; + const infrastructureChanges = changedFiles.filter((f) => + isInfrastructureFile(f, infrastructurePatterns) + ); + + if (infrastructureChanges.length > 0) { + log( + `Infrastructure files changed: ${infrastructureChanges.join(", ")}`, + verbose, + true + ); + return { + runAll: true, + reason: `Infrastructure files changed: ${infrastructureChanges.join(", ")}`, + tests: [], + changedFiles, + unmappedFiles: [], + exitCode: EXIT_CODES.FALLBACK_TRIGGERED, + }; + } + + // Check for new test files + const newTests = changedFiles.filter((f) => isNewTestFile(f, coverageMap)); + if (newTests.length > 0) { + log(`New test files detected: ${newTests.join(", ")}`, verbose, true); + return { + runAll: true, + reason: `New test files detected: ${newTests.join(", ")}`, + tests: [], + changedFiles, + unmappedFiles: [], + exitCode: EXIT_CODES.FALLBACK_TRIGGERED, + }; + } + + // Filter to source files only + const sourceChanges = changedFiles.filter( + (f) => + f.startsWith("src/") && + f.endsWith(".ts") && + !f.endsWith(".test.ts") && + !f.endsWith(".d.ts") + ); + + // Also include test file changes that are in the coverage map + const testChanges = changedFiles.filter( + (f) => + f.startsWith("tests/integration/") && + f.endsWith(".test.ts") && + coverageMap.allTests.includes(f) + ); + + // Non-source changes that we can ignore (docs, configs we don't care about, etc.) + const ignoredPatterns = [ + "docs/**", + "*.md", + "*.mdx", + ".gitignore", + ".editorconfig", + "LICENSE", + ".vscode/**", + "storybook/**", + ".storybook/**", + "*.stories.tsx", + ]; + + const ignoredChanges = changedFiles.filter((f) => + ignoredPatterns.some((p) => matchesPattern(f, p)) + ); + + // Find unmapped source files + const unmappedSourceFiles = sourceChanges.filter( + (f) => !coverageMap.fileToTests[f] + ); + + // If we have unmapped source files that aren't in the map at all, be conservative + if (unmappedSourceFiles.length > 0) { + log( + `Unmapped source files found: ${unmappedSourceFiles.join(", ")}`, + verbose, + true + ); + return { + runAll: true, + reason: `Unmapped source files: ${unmappedSourceFiles.join(", ")}`, + tests: [], + changedFiles, + unmappedFiles: unmappedSourceFiles, + exitCode: EXIT_CODES.FALLBACK_TRIGGERED, + }; + } + + // Collect all affected tests + const affectedTests = new Set(); + + // Add tests for changed source files + for (const sourceFile of sourceChanges) { + const tests = coverageMap.fileToTests[sourceFile] || []; + for (const test of tests) { + affectedTests.add(test); + } + } + + // Add any changed test files directly + for (const testFile of testChanges) { + affectedTests.add(testFile); + } + + const testsList = Array.from(affectedTests).sort(); + + // Check if we have remaining changes that aren't accounted for + const accountedChanges = new Set([ + ...sourceChanges, + ...testChanges, + ...ignoredChanges, + ]); + const unaccountedChanges = changedFiles.filter( + (f) => !accountedChanges.has(f) + ); + + // If there are unaccounted changes that are TypeScript files, be conservative + const unaccountedTsChanges = unaccountedChanges.filter((f) => + f.endsWith(".ts") + ); + if (unaccountedTsChanges.length > 0) { + log( + `Unaccounted TypeScript changes: ${unaccountedTsChanges.join(", ")}`, + verbose, + true + ); + return { + runAll: true, + reason: `Unaccounted TypeScript changes: ${unaccountedTsChanges.join(", ")}`, + tests: [], + changedFiles, + unmappedFiles: unaccountedTsChanges, + exitCode: EXIT_CODES.FALLBACK_TRIGGERED, + }; + } + + log(`Affected tests (${testsList.length}):`, verbose); + for (const test of testsList) { + log(` ${test}`, verbose); + } + + return { + runAll: false, + reason: + testsList.length > 0 + ? `Selected ${testsList.length} tests for ${sourceChanges.length} changed source files` + : "No tests affected by changes", + tests: testsList, + changedFiles, + unmappedFiles: [], + exitCode: EXIT_CODES.SUCCESS, + }; +} + +function formatOutput( + result: AffectedTestsResult, + format: "json" | "list" | "jest" +): string { + switch (format) { + case "json": + return JSON.stringify(result, null, 2); + case "list": + if (result.runAll) { + return "ALL"; + } + return result.tests.join("\n"); + case "jest": + if (result.runAll) { + // Return 'tests' to run all integration tests + return "tests"; + } + if (result.tests.length === 0) { + // No tests to run - return a pattern that matches nothing + return "--testPathPattern=^$"; + } + // Return test files space-separated for jest CLI + return result.tests.join(" "); + } +} + +function main(): void { + const options = parseArgs(); + const result = selectAffectedTests(options); + + // Output the result + const output = formatOutput(result, options.outputFormat); + console.log(output); + + // Log summary + log(`Result: ${result.reason}`, options.verbose, true); + if (result.runAll) { + log("Fallback triggered: running all tests", options.verbose, true); + } else if (result.tests.length > 0) { + log( + `Running ${result.tests.length} selected tests`, + options.verbose, + true + ); + } else { + log("No tests need to run", options.verbose, true); + } + + process.exit(result.exitCode); +} + +main(); diff --git a/scripts/selective-tests/types.ts b/scripts/selective-tests/types.ts new file mode 100644 index 0000000000..73e8d080c5 --- /dev/null +++ b/scripts/selective-tests/types.ts @@ -0,0 +1,77 @@ +/** + * Types for the selective test filtering system. + * + * This system uses runtime coverage data to determine which integration tests + * need to run based on changed source files. + */ + +/** Coverage map: source file → list of test files that cover it */ +export interface CoverageMap { + version: 1; + generatedAt: string; + /** Git commit SHA when the map was generated */ + commitSha: string; + /** Hash of all source files for staleness detection */ + sourceHash: string; + /** Map from relative source file path to array of test file paths */ + fileToTests: Record; + /** List of all test files included in the map */ + allTests: string[]; + /** Files that are considered "infrastructure" - changes trigger all tests */ + infrastructureFiles: string[]; +} + +/** Result from the affected tests selection */ +export interface AffectedTestsResult { + /** Whether to run all tests (fallback triggered) */ + runAll: boolean; + /** Reason for the decision */ + reason: string; + /** List of test files to run (empty if runAll) */ + tests: string[]; + /** Changed files that triggered the selection */ + changedFiles: string[]; + /** Files that were not found in the coverage map */ + unmappedFiles: string[]; +} + +/** Exit codes for scripts */ +export const EXIT_CODES = { + SUCCESS: 0, + FALLBACK_TRIGGERED: 2, + ERROR: 1, +} as const; + +/** Infrastructure files that should trigger all tests when changed */ +export const INFRASTRUCTURE_PATTERNS = [ + // Core configuration + "jest.config.cjs", + "babel.config.cjs", + "tsconfig.json", + "tsconfig.*.json", + "package.json", + "bun.lockb", + + // Test infrastructure + "tests/setup.ts", + "tests/integration/helpers.ts", + "tests/__mocks__/**", + + // Service container (imports all services) + "src/node/services/serviceContainer.ts", + + // Core shared types + "src/types/**", + "src/constants/**", + + // Build configuration + "vite.*.ts", + "electron-builder.yml", + + // CI configuration + ".github/workflows/**", + ".github/actions/**", + + // This selective test system itself + "scripts/selective-tests/**", +]; From 5cb21bc40bd388395157981742ead6c7d5545989 Mon Sep 17 00:00:00 2001 From: Ammar Date: Thu, 4 Dec 2025 14:00:40 -0600 Subject: [PATCH 2/6] cleanup: simplify selective test system - Remove run-selective-tests.sh (logic is inline in CI workflow) - Move CI/.github from infrastructure to ignored (doesn't affect tests) - Add more ignored patterns (e2e tests, electron-builder) - Fix verbose output capture in CI workflow - Minor comment improvements --- .github/workflows/ci.yml | 9 +- scripts/selective-tests/README.md | 11 +- .../selective-tests/generate-coverage-map.ts | 1 + .../selective-tests/run-selective-tests.sh | 132 ------------------ .../selective-tests/select-affected-tests.ts | 16 ++- scripts/selective-tests/types.ts | 9 +- 6 files changed, 22 insertions(+), 156 deletions(-) delete mode 100755 scripts/selective-tests/run-selective-tests.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4f407a959..df5b66ee5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,21 +123,22 @@ jobs: if: github.event.inputs.test_filter == '' && steps.coverage-map-cache.outputs.cache-hit != '' id: select-tests run: | - # Run selection script and capture output set +e SELECTED=$(bun scripts/selective-tests/select-affected-tests.ts \ --map coverage-map.json \ --base origin/${{ github.base_ref || 'main' }} \ --head ${{ github.sha }} \ --output jest \ - --verbose) + --verbose 2>&1) EXIT_CODE=$? set -e - echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT + # Extract just the test list (last line of output) + TEST_LIST=$(echo "$SELECTED" | tail -1) + echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT if [[ $EXIT_CODE -eq 0 ]]; then - echo "selected_tests=$SELECTED" >> $GITHUB_OUTPUT + echo "selected_tests=$TEST_LIST" >> $GITHUB_OUTPUT echo "run_selective=true" >> $GITHUB_OUTPUT else echo "run_selective=false" >> $GITHUB_OUTPUT diff --git a/scripts/selective-tests/README.md b/scripts/selective-tests/README.md index c2bb3cb712..431aa33779 100644 --- a/scripts/selective-tests/README.md +++ b/scripts/selective-tests/README.md @@ -21,10 +21,9 @@ This system reduces CI time by running only the integration tests affected by co ## Files -- `types.ts` - Shared types and constants +- `types.ts` - Shared types and infrastructure file patterns - `generate-coverage-map.ts` - Generates the coverage map by running tests with coverage - `select-affected-tests.ts` - Selects tests based on changed files -- `run-selective-tests.sh` - CI wrapper script with fallback handling ## Usage @@ -86,14 +85,6 @@ These files trigger a full test run when changed (see `INFRASTRUCTURE_PATTERNS` - Service container: `src/node/services/serviceContainer.ts` - Shared types: `src/types/**`, `src/constants/**` -## Shadow Mode - -For validation, run with `--shadow-mode` to log what would have been selected while still running all tests: - -```bash -./scripts/selective-tests/run-selective-tests.sh --shadow-mode -``` - ## Debugging Use `--verbose` for detailed logging: diff --git a/scripts/selective-tests/generate-coverage-map.ts b/scripts/selective-tests/generate-coverage-map.ts index afd3fb31ae..6a8c8ed2a5 100644 --- a/scripts/selective-tests/generate-coverage-map.ts +++ b/scripts/selective-tests/generate-coverage-map.ts @@ -20,6 +20,7 @@ import { createHash } from "crypto"; import type { CoverageMap } from "./types"; import { INFRASTRUCTURE_PATTERNS } from "./types"; +// Directories and defaults const TESTS_DIR = "tests/integration"; const COVERAGE_DIR = "coverage"; const DEFAULT_OUTPUT = "coverage-map.json"; diff --git a/scripts/selective-tests/run-selective-tests.sh b/scripts/selective-tests/run-selective-tests.sh deleted file mode 100755 index 3996b86198..0000000000 --- a/scripts/selective-tests/run-selective-tests.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash -# Run integration tests selectively based on changed files. -# -# This script wraps the TypeScript selection logic and handles CI-specific -# concerns like caching, fallback behavior, and output formatting. -# -# Usage: ./scripts/selective-tests/run-selective-tests.sh [--verbose] [--shadow-mode] -# -# Environment variables: -# COVERAGE_MAP_PATH Path to coverage map (default: coverage-map.json) -# FORCE_ALL_TESTS If set to "true", skip selection and run all tests -# GITHUB_BASE_REF Base ref for PR comparison (set by GitHub Actions) -# GITHUB_HEAD_REF Head ref for PR comparison (set by GitHub Actions) -# -# Exit codes: -# 0 - Tests passed (or no tests to run) -# 1 - Tests failed or error -# 2 - Fallback to all tests was triggered (informational in shadow mode) - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" - -# Defaults -COVERAGE_MAP_PATH="${COVERAGE_MAP_PATH:-$PROJECT_ROOT/coverage-map.json}" -FORCE_ALL_TESTS="${FORCE_ALL_TESTS:-false}" -VERBOSE="" -SHADOW_MODE="" - -# Parse arguments -while [[ $# -gt 0 ]]; do - case $1 in - --verbose) - VERBOSE="--verbose" - shift - ;; - --shadow-mode) - SHADOW_MODE="true" - shift - ;; - *) - shift - ;; - esac -done - -log() { - echo "[run-selective-tests] $1" >&2 -} - -# Determine git refs for comparison -if [[ -n "${GITHUB_BASE_REF:-}" ]]; then - BASE_REF="origin/$GITHUB_BASE_REF" -else - BASE_REF="origin/main" -fi - -if [[ -n "${GITHUB_SHA:-}" ]]; then - HEAD_REF="$GITHUB_SHA" -else - HEAD_REF="HEAD" -fi - -log "Base ref: $BASE_REF" -log "Head ref: $HEAD_REF" -log "Coverage map: $COVERAGE_MAP_PATH" - -# Check for force-all flag -FORCE_FLAG="" -if [[ "$FORCE_ALL_TESTS" == "true" ]]; then - log "FORCE_ALL_TESTS is set, will run all tests" - FORCE_FLAG="--force-all" -fi - -# Run the selection script -log "Selecting affected tests..." - -set +e -SELECTED_TESTS=$(bun "$SCRIPT_DIR/select-affected-tests.ts" \ - --map "$COVERAGE_MAP_PATH" \ - --base "$BASE_REF" \ - --head "$HEAD_REF" \ - --output jest \ - $VERBOSE \ - $FORCE_FLAG) -SELECT_EXIT_CODE=$? -set -e - -log "Selection exit code: $SELECT_EXIT_CODE" -log "Selected tests: $SELECTED_TESTS" - -# Handle shadow mode - run selection but always run all tests -if [[ -n "$SHADOW_MODE" ]]; then - log "Shadow mode enabled - will run all tests regardless of selection" - - # Log what would have happened - if [[ $SELECT_EXIT_CODE -eq 0 ]]; then - log "SHADOW: Would have run selective tests: $SELECTED_TESTS" - else - log "SHADOW: Would have fallen back to all tests (exit code $SELECT_EXIT_CODE)" - fi - - # Run all tests - log "Running all integration tests..." - TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent tests - exit $? -fi - -# Handle the selection result -if [[ $SELECT_EXIT_CODE -eq 2 ]]; then - # Fallback triggered - run all tests - log "Fallback triggered, running all integration tests..." - TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent tests - exit $? -elif [[ $SELECT_EXIT_CODE -ne 0 ]]; then - # Error in selection script - log "Error in selection script (exit code $SELECT_EXIT_CODE), falling back to all tests" - TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent tests - exit $? -fi - -# Check if no tests need to run -if [[ "$SELECTED_TESTS" == "--testPathPattern=^$" ]]; then - log "No tests affected by changes, skipping integration tests" - echo "::notice::No integration tests affected by changes - skipping" - exit 0 -fi - -# Run selected tests -log "Running selected tests: $SELECTED_TESTS" -TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent $SELECTED_TESTS diff --git a/scripts/selective-tests/select-affected-tests.ts b/scripts/selective-tests/select-affected-tests.ts index cb6c3c60b8..72b181a08f 100644 --- a/scripts/selective-tests/select-affected-tests.ts +++ b/scripts/selective-tests/select-affected-tests.ts @@ -39,10 +39,11 @@ const minimatch = require("minimatch") as ( options?: { matchBase?: boolean } ) => boolean; +// Defaults const DEFAULT_MAP_PATH = "coverage-map.json"; const DEFAULT_BASE_REF = "origin/main"; const DEFAULT_HEAD_REF = "HEAD"; -const DEFAULT_MAX_STALENESS_DAYS = 7; +const DEFAULT_MAX_STALENESS_DAYS = 7; // Fallback if map older than this interface Options { mapPath: string; @@ -312,18 +313,27 @@ function selectAffectedTests( coverageMap.allTests.includes(f) ); - // Non-source changes that we can ignore (docs, configs we don't care about, etc.) + // Non-source changes that we can safely ignore const ignoredPatterns = [ + // Documentation "docs/**", "*.md", "*.mdx", + // Editor/repo config ".gitignore", ".editorconfig", - "LICENSE", ".vscode/**", + "LICENSE", + // Storybook (has its own tests) "storybook/**", ".storybook/**", "*.stories.tsx", + // CI workflows (changes here don't affect test results) + ".github/**", + // E2E tests (separate from integration tests) + "tests/e2e/**", + // Build artifacts + "electron-builder.yml", ]; const ignoredChanges = changedFiles.filter((f) => diff --git a/scripts/selective-tests/types.ts b/scripts/selective-tests/types.ts index 73e8d080c5..523e3d541a 100644 --- a/scripts/selective-tests/types.ts +++ b/scripts/selective-tests/types.ts @@ -35,11 +35,11 @@ export interface AffectedTestsResult { unmappedFiles: string[]; } -/** Exit codes for scripts */ +/** Exit codes for scripts (2 = fallback so CI can distinguish from errors) */ export const EXIT_CODES = { SUCCESS: 0, - FALLBACK_TRIGGERED: 2, ERROR: 1, + FALLBACK_TRIGGERED: 2, } as const; /** Infrastructure files that should trigger all tests when changed */ @@ -66,11 +66,6 @@ export const INFRASTRUCTURE_PATTERNS = [ // Build configuration "vite.*.ts", - "electron-builder.yml", - - // CI configuration - ".github/workflows/**", - ".github/actions/**", // This selective test system itself "scripts/selective-tests/**", From f371a13c2b190647f2dd9e4915aa3de9e8a5d5ee Mon Sep 17 00:00:00 2001 From: Ammar Date: Fri, 5 Dec 2025 10:32:02 -0600 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=A4=96=20ci:=20consolidate=20workflow?= =?UTF-8?q?s=20into=20pr.yml=20with=20paths-filter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consolidate ci.yml and build.yml into a single pr.yml workflow with selective execution based on changed files. Changes: - Rename ci.yml → pr.yml, merge in build.yml jobs - Add dorny/paths-filter to detect docs-only PRs - Skip all tests/builds for docs-only changes - Consistent job naming: Test/*, Build/*, Smoke/* - Single 'required' job for branch protection Job structure: - changes: detect docs-only PRs - static-check: lint, typecheck, fmt - test-unit, test-integration, test-storybook, test-e2e - smoke-server, smoke-docker (merge_group only) - build-linux, build-macos, build-vscode - codex-comments - required: aggregates all job results Configure branch protection to require only 'PR / Required'. _Generated with `mux`_ --- .github/workflows/build.yml | 86 ---- .github/workflows/ci.yml | 365 ------------- .github/workflows/coverage-map.yml | 83 --- .github/workflows/pr.yml | 345 +++++++++++++ scripts/selective-tests/README.md | 98 ---- .../selective-tests/generate-coverage-map.ts | 245 --------- .../selective-tests/select-affected-tests.ts | 482 ------------------ scripts/selective-tests/types.ts | 72 --- 8 files changed, 345 insertions(+), 1431 deletions(-) delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/coverage-map.yml create mode 100644 .github/workflows/pr.yml delete mode 100644 scripts/selective-tests/README.md delete mode 100644 scripts/selective-tests/generate-coverage-map.ts delete mode 100644 scripts/selective-tests/select-affected-tests.ts delete mode 100644 scripts/selective-tests/types.ts diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 6b3a3e33d7..0000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Build - -on: - pull_request: - branches: ["**"] - merge_group: - workflow_dispatch: # Allow manual triggering - -jobs: - build-macos: - name: Build macOS - runs-on: ${{ github.repository_owner == 'coder' && 'depot-macos-15' || 'macos-latest' }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags - - - uses: ./.github/actions/setup-mux - - # No code signing - releases use release.yml (triggered by tag publish). - # Without CSC_LINK, x64 and arm64 builds run in parallel (~12 min faster). - - name: Package for macOS - run: make dist-mac - - - name: Upload macOS DMG (x64) - uses: actions/upload-artifact@v4 - with: - name: macos-dmg-x64 - path: release/*-x64.dmg - retention-days: 30 - if-no-files-found: error - - - name: Upload macOS DMG (arm64) - uses: actions/upload-artifact@v4 - with: - name: macos-dmg-arm64 - path: release/*-arm64.dmg - retention-days: 30 - if-no-files-found: error - - build-linux: - name: Build Linux - runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags - - - uses: ./.github/actions/setup-mux - - - name: Build application - run: bun run build - - - name: Package for Linux - run: make dist-linux - - - name: Upload Linux AppImage - uses: actions/upload-artifact@v4 - with: - name: linux-appimage - path: release/*.AppImage - retention-days: 30 - if-no-files-found: error - - build-vscode-extension: - name: Build VS Code Extension - runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags - - - uses: ./.github/actions/setup-mux - - - uses: ./.github/actions/build-vscode-extension - - - name: Upload VS Code extension artifact - uses: actions/upload-artifact@v4 - with: - name: vscode-extension - path: vscode/mux-*.vsix - retention-days: 30 - if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index df5b66ee5e..0000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,365 +0,0 @@ -name: CI - -on: - pull_request: - branches: ["**"] - merge_group: - workflow_dispatch: - inputs: - test_filter: - description: 'Optional test filter (e.g., "workspace", "tests/file.test.ts", or "-t pattern")' - required: false - type: string - # This filter is passed to unit tests, integration tests, e2e tests, and storybook tests - # to enable faster iteration when debugging specific test failures in CI - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - static-check: - name: Static Checks (lint + typecheck + fmt) - runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags - - - uses: ./.github/actions/setup-mux - - - name: Generate version file - run: ./scripts/generate-version.sh - - - name: Cache shfmt - id: cache-shfmt - uses: actions/cache@v4 - with: - path: ~/.local/bin/shfmt - # We install latest via webinstall; reflect that in the cache key to avoid pinning mismatches - key: ${{ runner.os }}-shfmt-latest - restore-keys: | - ${{ runner.os }}-shfmt- - - - name: Install and setup shfmt - run: | - # Install shfmt if not cached or if cached binary is broken - if [[ ! -f "$HOME/.local/bin/shfmt" ]] || ! "$HOME/.local/bin/shfmt" --version >/dev/null 2>&1; then - curl -sS https://webinstall.dev/shfmt | bash - fi - echo "$HOME/.local/bin" >> $GITHUB_PATH - # Verify shfmt is available - "$HOME/.local/bin/shfmt" --version - - - name: Install Nix - uses: cachix/install-nix-action@v27 - with: - extra_nix_config: | - experimental-features = nix-command flakes - - - name: Install uv - run: curl -LsSf https://astral.sh/uv/install.sh | sh - - - name: Add uv to PATH - run: echo "$HOME/.local/bin" >> $GITHUB_PATH - - - name: Run static checks - run: make -j3 static-check - - test: - name: Unit Tests - runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags - - - uses: ./.github/actions/setup-mux - - - name: Build worker files - run: make build-main - - - name: Run tests with coverage - run: bun test --coverage --coverage-reporter=lcov ${{ github.event.inputs.test_filter || 'src' }} - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage/lcov.info - flags: unit-tests - fail_ci_if_error: false - - integration-test: - name: Integration Tests - timeout-minutes: 10 - runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags and diff - - - uses: ./.github/actions/setup-mux - - - name: Build worker files - run: make build-main - - # Try to restore coverage map from cache for selective testing - - name: Restore coverage map - if: github.event.inputs.test_filter == '' - id: coverage-map-cache - uses: actions/cache/restore@v4 - with: - path: coverage-map.json - key: coverage-map-${{ github.sha }} - restore-keys: | - coverage-map-latest- - coverage-map- - - - name: Select affected tests - if: github.event.inputs.test_filter == '' && steps.coverage-map-cache.outputs.cache-hit != '' - id: select-tests - run: | - set +e - SELECTED=$(bun scripts/selective-tests/select-affected-tests.ts \ - --map coverage-map.json \ - --base origin/${{ github.base_ref || 'main' }} \ - --head ${{ github.sha }} \ - --output jest \ - --verbose 2>&1) - EXIT_CODE=$? - set -e - - # Extract just the test list (last line of output) - TEST_LIST=$(echo "$SELECTED" | tail -1) - - echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT - if [[ $EXIT_CODE -eq 0 ]]; then - echo "selected_tests=$TEST_LIST" >> $GITHUB_OUTPUT - echo "run_selective=true" >> $GITHUB_OUTPUT - else - echo "run_selective=false" >> $GITHUB_OUTPUT - fi - - - name: Run selective integration tests - if: steps.select-tests.outputs.run_selective == 'true' && steps.select-tests.outputs.selected_tests != '--testPathPattern=^$' - run: | - echo "Running selective tests: ${{ steps.select-tests.outputs.selected_tests }}" - TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent ${{ steps.select-tests.outputs.selected_tests }} - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - - - name: Skip tests (no affected tests) - if: steps.select-tests.outputs.run_selective == 'true' && steps.select-tests.outputs.selected_tests == '--testPathPattern=^$' - run: | - echo "::notice::No integration tests affected by changes - skipping" - echo "No integration tests were affected by the changes in this PR." >> $GITHUB_STEP_SUMMARY - - - name: Run all integration tests (fallback or manual filter) - if: steps.select-tests.outputs.run_selective != 'true' - # --silent suppresses per-test output (17+ test files × workers = overwhelming logs) - run: TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent ${{ github.event.inputs.test_filter || 'tests' }} - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: ./coverage/lcov.info - flags: integration-tests - fail_ci_if_error: false - - storybook-test: - name: Storybook Interaction Tests - runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} - if: github.event.inputs.test_filter == '' - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags - - - uses: ./.github/actions/setup-mux - - - uses: ./.github/actions/setup-playwright - - - name: Build Storybook - run: make storybook-build - - - name: Serve Storybook - run: | - bun x http-server storybook-static -p 6006 & - sleep 5 - - - name: Run Storybook tests - run: make test-storybook - - e2e-test: - name: E2E Tests (${{ matrix.os }}) - if: github.event.inputs.test_filter == '' - strategy: - fail-fast: false - matrix: - include: - # Linux: comprehensive E2E tests - - os: linux - runner: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} - test_scope: "all" - # macOS: window lifecycle and platform-dependent tests only - - os: macos - runner: ${{ github.repository_owner == 'coder' && 'depot-macos-latest' || 'macos-latest' }} - test_scope: "window-lifecycle" - runs-on: ${{ matrix.runner }} - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags - - - uses: ./.github/actions/setup-mux - - - name: Install xvfb (Linux) - if: matrix.os == 'linux' - run: | - sudo apt-get update - sudo apt-get install -y xvfb - - - uses: ./.github/actions/setup-playwright - - - name: Run comprehensive e2e tests (Linux) - if: matrix.os == 'linux' - run: xvfb-run -a make test-e2e - env: - ELECTRON_DISABLE_SANDBOX: 1 - - - name: Run window lifecycle e2e tests (macOS) - if: matrix.os == 'macos' - run: make test-e2e PLAYWRIGHT_ARGS="tests/e2e/scenarios/windowLifecycle.spec.ts" - - docker-smoke-test: - name: Docker Smoke Test - runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} - # Only run in merge queue (Docker builds are slow) - if: github.event_name == 'merge_group' - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build Docker image - uses: docker/build-push-action@v6 - with: - context: . - load: true - tags: mux-server:test - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Start container - run: | - docker run -d --name mux-test -p 3000:3000 mux-server:test - # Wait for server to be ready - for i in {1..30}; do - if curl -sf http://localhost:3000/health; then - echo "Server is ready" - break - fi - echo "Waiting for server... ($i/30)" - sleep 1 - done - - - name: Health check - run: | - response=$(curl -sf http://localhost:3000/health) - echo "Health response: $response" - if ! echo "$response" | grep -q '"status":"ok"'; then - echo "Health check failed" - exit 1 - fi - - - name: Version check - run: | - response=$(curl -sf http://localhost:3000/version) - echo "Version response: $response" - # Verify response contains expected fields - if ! echo "$response" | grep -q '"mode":"server"'; then - echo "Version check failed: missing mode=server" - exit 1 - fi - if ! echo "$response" | grep -q '"git_describe"'; then - echo "Version check failed: missing git_describe field" - exit 1 - fi - - - name: Show container logs on failure - if: failure() - run: docker logs mux-test - - - name: Cleanup - if: always() - run: docker rm -f mux-test || true - - mux-server-smoke-test: - name: Mux Server Smoke Test - runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} - # Tests oRPC/WebSocket flows that catch MockBrowserWindow bugs - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags - - - uses: ./.github/actions/setup-mux - - - name: Build application - run: make build - - - name: Pack npm package - run: npm pack - - - name: Run smoke test - env: - SERVER_PORT: 3001 - SERVER_HOST: localhost - STARTUP_TIMEOUT: 30 - run: | - # Find the actual tarball name (version may vary) - TARBALL=$(ls mux-*.tgz | head -1) - PACKAGE_TARBALL="$TARBALL" ./scripts/smoke-test.sh - - - name: Upload server logs on failure - if: failure() - uses: actions/upload-artifact@v4 - with: - name: mux-server-smoke-test-logs - path: /tmp/tmp.*/server.log - if-no-files-found: warn - retention-days: 7 - - check-codex-comments: - name: Check Codex Comments - runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} - if: github.event_name == 'pull_request' - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Required for git describe to find tags - - - name: Check for unresolved Codex comments - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - ./scripts/check_codex_comments.sh ${{ github.event.pull_request.number }} - -# Trigger CI run diff --git a/.github/workflows/coverage-map.yml b/.github/workflows/coverage-map.yml deleted file mode 100644 index 3660b8d2d1..0000000000 --- a/.github/workflows/coverage-map.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Coverage Map Generation - -on: - # Run daily at 2 AM UTC - schedule: - - cron: "0 2 * * *" - # Allow manual trigger - workflow_dispatch: - # Also regenerate when the selective test system changes - push: - branches: [main] - paths: - - "scripts/selective-tests/**" - - "tests/integration/**" - - ".github/workflows/coverage-map.yml" - -concurrency: - group: coverage-map-${{ github.ref }} - cancel-in-progress: true - -jobs: - generate-coverage-map: - name: Generate Coverage Map - runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} - timeout-minutes: 60 - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - uses: ./.github/actions/setup-mux - - - name: Build worker files - run: make build-main - - - name: Generate coverage map - run: bun scripts/selective-tests/generate-coverage-map.ts --output coverage-map.json - env: - TEST_INTEGRATION: "1" - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - - - name: Upload coverage map artifact - uses: actions/upload-artifact@v4 - with: - name: coverage-map - path: coverage-map.json - retention-days: 30 - - - name: Cache coverage map for CI - uses: actions/cache/save@v4 - with: - path: coverage-map.json - key: coverage-map-${{ github.sha }} - - # Also save with a "latest" key that PRs can restore from - - name: Cache coverage map (latest) - uses: actions/cache/save@v4 - with: - path: coverage-map.json - key: coverage-map-latest-${{ github.run_id }} - # This creates a new cache entry each run, but restore-keys in CI - # will match the most recent one - - - name: Summary - run: | - echo "## Coverage Map Generated" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- **Commit:** ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY - echo "- **Generated at:** $(date -u)" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Map Statistics" >> $GITHUB_STEP_SUMMARY - echo '```json' >> $GITHUB_STEP_SUMMARY - jq '{ - version, - generatedAt, - commitSha, - testCount: (.allTests | length), - filesWithCoverage: ([.fileToTests | to_entries[] | select(.value | length > 0)] | length), - totalMappings: ([.fileToTests | to_entries[] | .value | length] | add) - }' coverage-map.json >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000000..691845c141 --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,345 @@ +name: PR + +on: + pull_request: + branches: ["**"] + merge_group: + workflow_dispatch: + inputs: + test_filter: + description: 'Optional test filter (e.g., "workspace", "tests/file.test.ts", or "-t pattern")' + required: false + type: string + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + # Detect what files changed to skip jobs for docs-only or browser-only PRs + changes: + name: Detect Changes + runs-on: ubuntu-latest + outputs: + docs-only: ${{ steps.filter.outputs.docs == 'true' && steps.filter.outputs.src == 'false' && steps.filter.outputs.config == 'false' }} + # Browser-only: only browser/renderer code changed, no backend/node changes + browser-only: ${{ steps.filter.outputs.browser == 'true' && steps.filter.outputs.backend == 'false' && steps.filter.outputs.config == 'false' }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + docs: + - 'docs/**' + - '*.md' + - 'LICENSE' + - '.vscode/**' + src: + - 'src/**' + - 'tests/**' + - 'vscode/**' + browser: + - 'src/browser/**' + - 'src/styles/**' + - 'src/components/**' + - 'src/hooks/**' + - '**/*.stories.tsx' + - '.storybook/**' + backend: + - 'src/node/**' + - 'src/cli/**' + - 'src/desktop/**' + - 'src/common/**' + - 'tests/integration/**' + config: + - '.github/**' + - 'jest.config.cjs' + - 'babel.config.cjs' + - 'package.json' + - 'bun.lockb' + - 'tsconfig*.json' + - 'vite*.ts' + - 'Makefile' + - 'electron-builder.yml' + + static-check: + name: Static Checks + runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup-mux + - run: ./scripts/generate-version.sh + - uses: actions/cache@v4 + with: + path: ~/.local/bin/shfmt + key: ${{ runner.os }}-shfmt-latest + - name: Install shfmt + run: | + if [[ ! -f "$HOME/.local/bin/shfmt" ]] || ! "$HOME/.local/bin/shfmt" --version >/dev/null 2>&1; then + curl -sS https://webinstall.dev/shfmt | bash + fi + echo "$HOME/.local/bin" >> $GITHUB_PATH + - uses: cachix/install-nix-action@v27 + with: + extra_nix_config: | + experimental-features = nix-command flakes + - run: curl -LsSf https://astral.sh/uv/install.sh | sh + - run: echo "$HOME/.local/bin" >> $GITHUB_PATH + - run: make -j3 static-check + + test-unit: + name: Test / Unit + needs: [changes] + if: ${{ needs.changes.outputs.docs-only != 'true' }} + runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup-mux + - run: make build-main + - run: bun test --coverage --coverage-reporter=lcov ${{ github.event.inputs.test_filter || 'src' }} + - uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage/lcov.info + flags: unit-tests + fail_ci_if_error: false + + test-integration: + name: Test / Integration + needs: [changes] + # Skip for docs-only or browser-only (integration tests are backend IPC tests) + if: ${{ needs.changes.outputs.docs-only != 'true' && needs.changes.outputs.browser-only != 'true' }} + timeout-minutes: 10 + runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup-mux + - run: make build-main + - run: TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent ${{ github.event.inputs.test_filter || 'tests' }} + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + - uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage/lcov.info + flags: integration-tests + fail_ci_if_error: false + + test-storybook: + name: Test / Storybook + needs: [changes] + if: ${{ needs.changes.outputs.docs-only != 'true' && github.event.inputs.test_filter == '' }} + runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup-mux + - uses: ./.github/actions/setup-playwright + - run: make storybook-build + - run: | + bun x http-server storybook-static -p 6006 & + sleep 5 + - run: make test-storybook + + test-e2e: + name: Test / E2E (${{ matrix.os }}) + needs: [changes] + if: ${{ needs.changes.outputs.docs-only != 'true' && github.event.inputs.test_filter == '' }} + strategy: + fail-fast: false + matrix: + include: + - os: linux + runner: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} + - os: macos + runner: ${{ github.repository_owner == 'coder' && 'depot-macos-latest' || 'macos-latest' }} + runs-on: ${{ matrix.runner }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup-mux + - name: Install xvfb + if: matrix.os == 'linux' + run: | + sudo apt-get update + sudo apt-get install -y xvfb + - uses: ./.github/actions/setup-playwright + - name: Run tests + if: matrix.os == 'linux' + run: xvfb-run -a make test-e2e + env: + ELECTRON_DISABLE_SANDBOX: 1 + - name: Run tests + if: matrix.os == 'macos' + run: make test-e2e PLAYWRIGHT_ARGS="tests/e2e/scenarios/windowLifecycle.spec.ts" + + smoke-server: + name: Smoke / Server + needs: [changes] + if: ${{ needs.changes.outputs.docs-only != 'true' }} + runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup-mux + - run: make build + - run: npm pack + - env: + SERVER_PORT: 3001 + SERVER_HOST: localhost + STARTUP_TIMEOUT: 30 + run: | + TARBALL=$(ls mux-*.tgz | head -1) + PACKAGE_TARBALL="$TARBALL" ./scripts/smoke-test.sh + - uses: actions/upload-artifact@v4 + if: failure() + with: + name: smoke-server-logs + path: /tmp/tmp.*/server.log + if-no-files-found: warn + retention-days: 7 + + smoke-docker: + name: Smoke / Docker + if: github.event_name == 'merge_group' + runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: docker/setup-buildx-action@v3 + - uses: docker/build-push-action@v6 + with: + context: . + load: true + tags: mux-server:test + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Test container + run: | + docker run -d --name mux-test -p 3000:3000 mux-server:test + for i in {1..30}; do + curl -sf http://localhost:3000/health && break + echo "Waiting... ($i/30)" + sleep 1 + done + curl -sf http://localhost:3000/health | grep -q '"status":"ok"' + curl -sf http://localhost:3000/version | grep -q '"mode":"server"' + - if: failure() + run: docker logs mux-test + - if: always() + run: docker rm -f mux-test || true + + build-linux: + name: Build / Linux + needs: [changes] + if: ${{ needs.changes.outputs.docs-only != 'true' }} + runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup-mux + - run: bun run build + - run: make dist-linux + - uses: actions/upload-artifact@v4 + with: + name: build-linux + path: release/*.AppImage + retention-days: 30 + if-no-files-found: error + + build-macos: + name: Build / macOS + needs: [changes] + if: ${{ needs.changes.outputs.docs-only != 'true' }} + runs-on: ${{ github.repository_owner == 'coder' && 'depot-macos-15' || 'macos-latest' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup-mux + - run: make dist-mac + - uses: actions/upload-artifact@v4 + with: + name: build-macos-x64 + path: release/*-x64.dmg + retention-days: 30 + if-no-files-found: error + - uses: actions/upload-artifact@v4 + with: + name: build-macos-arm64 + path: release/*-arm64.dmg + retention-days: 30 + if-no-files-found: error + + build-vscode: + name: Build / VS Code + needs: [changes] + if: ${{ needs.changes.outputs.docs-only != 'true' }} + runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: ./.github/actions/setup-mux + - uses: ./.github/actions/build-vscode-extension + - uses: actions/upload-artifact@v4 + with: + name: build-vscode + path: vscode/mux-*.vsix + retention-days: 30 + if-no-files-found: error + + codex-comments: + name: Codex Comments + if: github.event_name == 'pull_request' + runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ./scripts/check_codex_comments.sh ${{ github.event.pull_request.number }} + + # Single required check - configure branch protection to require only this job + required: + name: Required + if: always() + needs: + - changes + - static-check + - test-unit + - test-integration + - test-storybook + - test-e2e + - smoke-server + - smoke-docker + - build-linux + - build-macos + - build-vscode + - codex-comments + runs-on: ubuntu-latest + steps: + - run: | + results="${{ join(needs.*.result, ' ') }}" + echo "Job results: $results" + for result in $results; do + if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then + echo "❌ CI failed" + exit 1 + fi + done + echo "✅ All checks passed" diff --git a/scripts/selective-tests/README.md b/scripts/selective-tests/README.md deleted file mode 100644 index 431aa33779..0000000000 --- a/scripts/selective-tests/README.md +++ /dev/null @@ -1,98 +0,0 @@ -# Selective Test Execution System - -This system reduces CI time by running only the integration tests affected by code changes, rather than the full test suite on every PR. - -## How It Works - -1. **Coverage Map Generation**: A daily workflow runs all integration tests individually with code coverage, building a reverse index from source files to tests. - -2. **Affected Test Selection**: When a PR runs, the system: - - Restores the coverage map from cache - - Identifies changed files via git diff - - Looks up which tests cover those files - - Runs only the affected tests - -3. **Safe Fallbacks**: The system falls back to running all tests when: - - Coverage map is missing or stale (>7 days) - - Infrastructure files change (jest.config, package.json, etc.) - - New test files are added - - Changed source files aren't in the coverage map - - Any error occurs during selection - -## Files - -- `types.ts` - Shared types and infrastructure file patterns -- `generate-coverage-map.ts` - Generates the coverage map by running tests with coverage -- `select-affected-tests.ts` - Selects tests based on changed files - -## Usage - -### Generate Coverage Map (local testing) - -```bash -bun scripts/selective-tests/generate-coverage-map.ts --output coverage-map.json -``` - -This takes ~30-60 minutes as it runs each test file individually. - -### Select Affected Tests - -```bash -# Using git diff -bun scripts/selective-tests/select-affected-tests.ts \ - --map coverage-map.json \ - --base origin/main \ - --head HEAD \ - --output jest - -# Using explicit file list -bun scripts/selective-tests/select-affected-tests.ts \ - --map coverage-map.json \ - --changed "src/node/services/workspaceService.ts,src/node/config.ts" \ - --output json -``` - -### Output Formats - -- `jest` - Space-separated test files for Jest CLI, or `tests` for all tests -- `json` - Full result object with reasoning -- `list` - Newline-separated test files, or `ALL` for all tests - -### Exit Codes - -- `0` - Selection successful (may be empty test list) -- `2` - Fallback triggered (run all tests) -- `1` - Error - -## CI Integration - -The system integrates with `.github/workflows/ci.yml`: - -1. Restores coverage map from cache -2. Runs selection script -3. Either runs selected tests or falls back to all tests -4. Skips tests entirely if no tests are affected - -The coverage map is regenerated daily by `.github/workflows/coverage-map.yml` and cached for PR use. - -## Infrastructure Files - -These files trigger a full test run when changed (see `INFRASTRUCTURE_PATTERNS` in `types.ts`): - -- Test configuration: `jest.config.cjs`, `babel.config.cjs` -- Build config: `tsconfig.json`, `package.json`, `bun.lockb` -- Test infrastructure: `tests/setup.ts`, `tests/integration/helpers.ts` -- Service container: `src/node/services/serviceContainer.ts` -- Shared types: `src/types/**`, `src/constants/**` - -## Debugging - -Use `--verbose` for detailed logging: - -```bash -bun scripts/selective-tests/select-affected-tests.ts \ - --map coverage-map.json \ - --changed "src/node/services/aiService.ts" \ - --output json \ - --verbose -``` diff --git a/scripts/selective-tests/generate-coverage-map.ts b/scripts/selective-tests/generate-coverage-map.ts deleted file mode 100644 index 6a8c8ed2a5..0000000000 --- a/scripts/selective-tests/generate-coverage-map.ts +++ /dev/null @@ -1,245 +0,0 @@ -#!/usr/bin/env bun -/** - * Generate a coverage map by running each integration test individually - * and recording which source files it covers. - * - * Usage: bun scripts/selective-tests/generate-coverage-map.ts [--output coverage-map.json] - * - * This script: - * 1. Discovers all integration test files - * 2. Runs each test individually with coverage enabled - * 3. Parses the coverage output to extract covered files - * 4. Builds a reverse index: source file → tests that cover it - * 5. Outputs a JSON coverage map - */ - -import { execSync, spawnSync } from "child_process"; -import * as fs from "fs"; -import * as path from "path"; -import { createHash } from "crypto"; -import type { CoverageMap } from "./types"; -import { INFRASTRUCTURE_PATTERNS } from "./types"; - -// Directories and defaults -const TESTS_DIR = "tests/integration"; -const COVERAGE_DIR = "coverage"; -const DEFAULT_OUTPUT = "coverage-map.json"; - -function log(message: string): void { - console.error(`[generate-coverage-map] ${message}`); -} - -function getGitCommitSha(): string { - try { - return execSync("git rev-parse HEAD", { encoding: "utf-8" }).trim(); - } catch { - return "unknown"; - } -} - -function hashFiles(files: string[]): string { - const hash = createHash("sha256"); - for (const file of files.sort()) { - if (fs.existsSync(file)) { - hash.update(file); - hash.update(fs.readFileSync(file)); - } - } - return hash.digest("hex").substring(0, 16); -} - -function discoverTestFiles(): string[] { - const testFiles: string[] = []; - const entries = fs.readdirSync(TESTS_DIR, { withFileTypes: true }); - - for (const entry of entries) { - if (entry.isFile() && entry.name.endsWith(".test.ts")) { - testFiles.push(path.join(TESTS_DIR, entry.name)); - } - } - - return testFiles.sort(); -} - -function discoverSourceFiles(): string[] { - const sourceFiles: string[] = []; - - function walkDir(dir: string): void { - const entries = fs.readdirSync(dir, { withFileTypes: true }); - for (const entry of entries) { - const fullPath = path.join(dir, entry.name); - if (entry.isDirectory() && !entry.name.startsWith(".")) { - walkDir(fullPath); - } else if (entry.isFile() && entry.name.endsWith(".ts")) { - sourceFiles.push(fullPath); - } - } - } - - walkDir("src"); - return sourceFiles.sort(); -} - -interface CoverageData { - [filePath: string]: { - s: Record; // statements - f?: Record; // functions - b?: Record; // branches - }; -} - -function extractCoveredFiles(coverageJsonPath: string): string[] { - if (!fs.existsSync(coverageJsonPath)) { - return []; - } - - const coverage: CoverageData = JSON.parse( - fs.readFileSync(coverageJsonPath, "utf-8") - ); - const coveredFiles: string[] = []; - - for (const [filePath, data] of Object.entries(coverage)) { - // Check if any statement was executed - const hasExecutedStatements = Object.values(data.s).some( - (count) => count > 0 - ); - if (hasExecutedStatements) { - // Convert absolute path to relative - const relativePath = path.relative(process.cwd(), filePath); - if (relativePath.startsWith("src/")) { - coveredFiles.push(relativePath); - } - } - } - - return coveredFiles; -} - -function runTestWithCoverage(testFile: string): string[] { - // Clean coverage directory first - if (fs.existsSync(COVERAGE_DIR)) { - fs.rmSync(COVERAGE_DIR, { recursive: true }); - } - - log(`Running: ${testFile}`); - - const result = spawnSync( - "bun", - [ - "x", - "jest", - "--coverage", - "--coverageReporters=json", - "--maxWorkers=1", - "--silent", - "--forceExit", - testFile, - ], - { - env: { - ...process.env, - TEST_INTEGRATION: "1", - // Disable color output for cleaner logs - FORCE_COLOR: "0", - }, - stdio: ["ignore", "pipe", "pipe"], - timeout: 5 * 60 * 1000, // 5 minute timeout per test - } - ); - - if (result.error) { - log(` Error running test: ${result.error.message}`); - return []; - } - - // Extract covered files from coverage-final.json - const coverageJsonPath = path.join(COVERAGE_DIR, "coverage-final.json"); - const coveredFiles = extractCoveredFiles(coverageJsonPath); - log(` Covered ${coveredFiles.length} source files`); - - return coveredFiles; -} - -function buildCoverageMap( - testFiles: string[], - sourceFiles: string[] -): CoverageMap { - const fileToTests: Record = {}; - const commitSha = getGitCommitSha(); - const sourceHash = hashFiles(sourceFiles); - - // Initialize all source files with empty arrays - for (const sourceFile of sourceFiles) { - fileToTests[sourceFile] = []; - } - - // Run each test and record coverage - for (let i = 0; i < testFiles.length; i++) { - const testFile = testFiles[i]; - log(`[${i + 1}/${testFiles.length}] Processing ${testFile}`); - - const coveredFiles = runTestWithCoverage(testFile); - - for (const sourceFile of coveredFiles) { - if (!fileToTests[sourceFile]) { - fileToTests[sourceFile] = []; - } - if (!fileToTests[sourceFile].includes(testFile)) { - fileToTests[sourceFile].push(testFile); - } - } - } - - return { - version: 1, - generatedAt: new Date().toISOString(), - commitSha, - sourceHash, - fileToTests, - allTests: testFiles, - infrastructureFiles: INFRASTRUCTURE_PATTERNS, - }; -} - -function main(): void { - const args = process.argv.slice(2); - let outputPath = DEFAULT_OUTPUT; - - for (let i = 0; i < args.length; i++) { - if (args[i] === "--output" && args[i + 1]) { - outputPath = args[i + 1]; - i++; - } - } - - log("Discovering test files..."); - const testFiles = discoverTestFiles(); - log(`Found ${testFiles.length} integration test files`); - - log("Discovering source files..."); - const sourceFiles = discoverSourceFiles(); - log(`Found ${sourceFiles.length} source files`); - - log("Building coverage map (this may take a while)..."); - const coverageMap = buildCoverageMap(testFiles, sourceFiles); - - // Write output - fs.writeFileSync(outputPath, JSON.stringify(coverageMap, null, 2)); - log(`Coverage map written to ${outputPath}`); - - // Print summary - const totalMappings = Object.values(coverageMap.fileToTests).reduce( - (sum, tests) => sum + tests.length, - 0 - ); - const filesWithCoverage = Object.values(coverageMap.fileToTests).filter( - (tests) => tests.length > 0 - ).length; - - log(`Summary:`); - log(` Total test files: ${coverageMap.allTests.length}`); - log(` Source files with coverage: ${filesWithCoverage}`); - log(` Total file→test mappings: ${totalMappings}`); -} - -main(); diff --git a/scripts/selective-tests/select-affected-tests.ts b/scripts/selective-tests/select-affected-tests.ts deleted file mode 100644 index 72b181a08f..0000000000 --- a/scripts/selective-tests/select-affected-tests.ts +++ /dev/null @@ -1,482 +0,0 @@ -#!/usr/bin/env bun -/** - * Select which integration tests to run based on changed files. - * - * Usage: - * bun scripts/selective-tests/select-affected-tests.ts [options] - * - * Options: - * --map Path to coverage map JSON (default: coverage-map.json) - * --base Git base ref for comparison (default: origin/main) - * --head Git head ref for comparison (default: HEAD) - * --changed Comma-separated list of changed files (overrides git diff) - * --output Output format: json, list, or jest (default: jest) - * --max-staleness Maximum map age in days (default: 7) - * --force-all Force running all tests (for debugging) - * --verbose Enable verbose logging - * - * Exit codes: - * 0 - Success, selective tests determined - * 2 - Fallback triggered, run all tests - * 1 - Error - * - * Output (stdout): - * - json: Full AffectedTestsResult as JSON - * - list: Newline-separated list of test files - * - jest: Space-separated test files suitable for jest CLI - */ - -import { execSync } from "child_process"; -import * as fs from "fs"; -import type { CoverageMap, AffectedTestsResult } from "./types"; -import { EXIT_CODES, INFRASTRUCTURE_PATTERNS } from "./types"; - -// minimatch is CommonJS, need to use require -// eslint-disable-next-line @typescript-eslint/no-require-imports -const minimatch = require("minimatch") as ( - file: string, - pattern: string, - options?: { matchBase?: boolean } -) => boolean; - -// Defaults -const DEFAULT_MAP_PATH = "coverage-map.json"; -const DEFAULT_BASE_REF = "origin/main"; -const DEFAULT_HEAD_REF = "HEAD"; -const DEFAULT_MAX_STALENESS_DAYS = 7; // Fallback if map older than this - -interface Options { - mapPath: string; - baseRef: string; - headRef: string; - changedFiles: string[] | null; - outputFormat: "json" | "list" | "jest"; - maxStalenessDays: number; - forceAll: boolean; - verbose: boolean; -} - -function log(message: string, verbose: boolean, force = false): void { - if (verbose || force) { - console.error(`[select-affected-tests] ${message}`); - } -} - -function parseArgs(): Options { - const args = process.argv.slice(2); - const options: Options = { - mapPath: DEFAULT_MAP_PATH, - baseRef: DEFAULT_BASE_REF, - headRef: DEFAULT_HEAD_REF, - changedFiles: null, - outputFormat: "jest", - maxStalenessDays: DEFAULT_MAX_STALENESS_DAYS, - forceAll: false, - verbose: false, - }; - - for (let i = 0; i < args.length; i++) { - switch (args[i]) { - case "--map": - options.mapPath = args[++i]; - break; - case "--base": - options.baseRef = args[++i]; - break; - case "--head": - options.headRef = args[++i]; - break; - case "--changed": - options.changedFiles = args[++i].split(",").filter(Boolean); - break; - case "--output": - options.outputFormat = args[++i] as "json" | "list" | "jest"; - break; - case "--max-staleness": - options.maxStalenessDays = parseInt(args[++i], 10); - break; - case "--force-all": - options.forceAll = true; - break; - case "--verbose": - options.verbose = true; - break; - } - } - - return options; -} - -function getChangedFiles(baseRef: string, headRef: string): string[] { - try { - // First, try to get the merge base for more accurate diffing - const mergeBase = execSync(`git merge-base ${baseRef} ${headRef}`, { - encoding: "utf-8", - }).trim(); - - const output = execSync(`git diff --name-only ${mergeBase} ${headRef}`, { - encoding: "utf-8", - }); - - return output - .split("\n") - .map((f) => f.trim()) - .filter(Boolean); - } catch { - // Fallback to direct diff if merge-base fails - try { - const output = execSync(`git diff --name-only ${baseRef} ${headRef}`, { - encoding: "utf-8", - }); - return output - .split("\n") - .map((f) => f.trim()) - .filter(Boolean); - } catch { - return []; - } - } -} - -function matchesPattern(file: string, pattern: string): boolean { - // Handle glob patterns - if (pattern.includes("*")) { - return minimatch(file, pattern, { matchBase: true }); - } - // Exact match or directory prefix - return file === pattern || file.startsWith(pattern + "/"); -} - -function isInfrastructureFile( - file: string, - patterns: string[] = INFRASTRUCTURE_PATTERNS -): boolean { - return patterns.some((pattern) => matchesPattern(file, pattern)); -} - -function isNewTestFile(file: string, coverageMap: CoverageMap): boolean { - return ( - file.startsWith("tests/integration/") && - file.endsWith(".test.ts") && - !coverageMap.allTests.includes(file) - ); -} - -function loadCoverageMap(mapPath: string): CoverageMap | null { - if (!fs.existsSync(mapPath)) { - return null; - } - - try { - const content = fs.readFileSync(mapPath, "utf-8"); - return JSON.parse(content) as CoverageMap; - } catch { - return null; - } -} - -function isMapStale(map: CoverageMap, maxDays: number): boolean { - const generatedAt = new Date(map.generatedAt); - const now = new Date(); - const ageMs = now.getTime() - generatedAt.getTime(); - const ageDays = ageMs / (1000 * 60 * 60 * 24); - return ageDays > maxDays; -} - -function selectAffectedTests( - options: Options -): AffectedTestsResult & { exitCode: number } { - const verbose = options.verbose; - - // Check for force-all flag - if (options.forceAll) { - log("Force-all flag set, running all tests", verbose, true); - return { - runAll: true, - reason: "Force-all flag set", - tests: [], - changedFiles: [], - unmappedFiles: [], - exitCode: EXIT_CODES.FALLBACK_TRIGGERED, - }; - } - - // Load coverage map - const coverageMap = loadCoverageMap(options.mapPath); - - if (!coverageMap) { - log(`Coverage map not found at ${options.mapPath}`, verbose, true); - return { - runAll: true, - reason: `Coverage map not found at ${options.mapPath}`, - tests: [], - changedFiles: [], - unmappedFiles: [], - exitCode: EXIT_CODES.FALLBACK_TRIGGERED, - }; - } - - log(`Loaded coverage map from ${options.mapPath}`, verbose); - log(` Generated: ${coverageMap.generatedAt}`, verbose); - log(` Commit: ${coverageMap.commitSha}`, verbose); - log(` Tests: ${coverageMap.allTests.length}`, verbose); - - // Check map staleness - if (isMapStale(coverageMap, options.maxStalenessDays)) { - log( - `Coverage map is stale (> ${options.maxStalenessDays} days old)`, - verbose, - true - ); - return { - runAll: true, - reason: `Coverage map is stale (> ${options.maxStalenessDays} days old)`, - tests: [], - changedFiles: [], - unmappedFiles: [], - exitCode: EXIT_CODES.FALLBACK_TRIGGERED, - }; - } - - // Get changed files - const changedFiles = - options.changedFiles ?? getChangedFiles(options.baseRef, options.headRef); - - if (changedFiles.length === 0) { - log("No changed files detected", verbose, true); - return { - runAll: false, - reason: "No changed files detected", - tests: [], - changedFiles: [], - unmappedFiles: [], - exitCode: EXIT_CODES.SUCCESS, - }; - } - - log(`Changed files (${changedFiles.length}):`, verbose); - for (const file of changedFiles) { - log(` ${file}`, verbose); - } - - // Check for infrastructure files - const infrastructurePatterns = - coverageMap.infrastructureFiles || INFRASTRUCTURE_PATTERNS; - const infrastructureChanges = changedFiles.filter((f) => - isInfrastructureFile(f, infrastructurePatterns) - ); - - if (infrastructureChanges.length > 0) { - log( - `Infrastructure files changed: ${infrastructureChanges.join(", ")}`, - verbose, - true - ); - return { - runAll: true, - reason: `Infrastructure files changed: ${infrastructureChanges.join(", ")}`, - tests: [], - changedFiles, - unmappedFiles: [], - exitCode: EXIT_CODES.FALLBACK_TRIGGERED, - }; - } - - // Check for new test files - const newTests = changedFiles.filter((f) => isNewTestFile(f, coverageMap)); - if (newTests.length > 0) { - log(`New test files detected: ${newTests.join(", ")}`, verbose, true); - return { - runAll: true, - reason: `New test files detected: ${newTests.join(", ")}`, - tests: [], - changedFiles, - unmappedFiles: [], - exitCode: EXIT_CODES.FALLBACK_TRIGGERED, - }; - } - - // Filter to source files only - const sourceChanges = changedFiles.filter( - (f) => - f.startsWith("src/") && - f.endsWith(".ts") && - !f.endsWith(".test.ts") && - !f.endsWith(".d.ts") - ); - - // Also include test file changes that are in the coverage map - const testChanges = changedFiles.filter( - (f) => - f.startsWith("tests/integration/") && - f.endsWith(".test.ts") && - coverageMap.allTests.includes(f) - ); - - // Non-source changes that we can safely ignore - const ignoredPatterns = [ - // Documentation - "docs/**", - "*.md", - "*.mdx", - // Editor/repo config - ".gitignore", - ".editorconfig", - ".vscode/**", - "LICENSE", - // Storybook (has its own tests) - "storybook/**", - ".storybook/**", - "*.stories.tsx", - // CI workflows (changes here don't affect test results) - ".github/**", - // E2E tests (separate from integration tests) - "tests/e2e/**", - // Build artifacts - "electron-builder.yml", - ]; - - const ignoredChanges = changedFiles.filter((f) => - ignoredPatterns.some((p) => matchesPattern(f, p)) - ); - - // Find unmapped source files - const unmappedSourceFiles = sourceChanges.filter( - (f) => !coverageMap.fileToTests[f] - ); - - // If we have unmapped source files that aren't in the map at all, be conservative - if (unmappedSourceFiles.length > 0) { - log( - `Unmapped source files found: ${unmappedSourceFiles.join(", ")}`, - verbose, - true - ); - return { - runAll: true, - reason: `Unmapped source files: ${unmappedSourceFiles.join(", ")}`, - tests: [], - changedFiles, - unmappedFiles: unmappedSourceFiles, - exitCode: EXIT_CODES.FALLBACK_TRIGGERED, - }; - } - - // Collect all affected tests - const affectedTests = new Set(); - - // Add tests for changed source files - for (const sourceFile of sourceChanges) { - const tests = coverageMap.fileToTests[sourceFile] || []; - for (const test of tests) { - affectedTests.add(test); - } - } - - // Add any changed test files directly - for (const testFile of testChanges) { - affectedTests.add(testFile); - } - - const testsList = Array.from(affectedTests).sort(); - - // Check if we have remaining changes that aren't accounted for - const accountedChanges = new Set([ - ...sourceChanges, - ...testChanges, - ...ignoredChanges, - ]); - const unaccountedChanges = changedFiles.filter( - (f) => !accountedChanges.has(f) - ); - - // If there are unaccounted changes that are TypeScript files, be conservative - const unaccountedTsChanges = unaccountedChanges.filter((f) => - f.endsWith(".ts") - ); - if (unaccountedTsChanges.length > 0) { - log( - `Unaccounted TypeScript changes: ${unaccountedTsChanges.join(", ")}`, - verbose, - true - ); - return { - runAll: true, - reason: `Unaccounted TypeScript changes: ${unaccountedTsChanges.join(", ")}`, - tests: [], - changedFiles, - unmappedFiles: unaccountedTsChanges, - exitCode: EXIT_CODES.FALLBACK_TRIGGERED, - }; - } - - log(`Affected tests (${testsList.length}):`, verbose); - for (const test of testsList) { - log(` ${test}`, verbose); - } - - return { - runAll: false, - reason: - testsList.length > 0 - ? `Selected ${testsList.length} tests for ${sourceChanges.length} changed source files` - : "No tests affected by changes", - tests: testsList, - changedFiles, - unmappedFiles: [], - exitCode: EXIT_CODES.SUCCESS, - }; -} - -function formatOutput( - result: AffectedTestsResult, - format: "json" | "list" | "jest" -): string { - switch (format) { - case "json": - return JSON.stringify(result, null, 2); - case "list": - if (result.runAll) { - return "ALL"; - } - return result.tests.join("\n"); - case "jest": - if (result.runAll) { - // Return 'tests' to run all integration tests - return "tests"; - } - if (result.tests.length === 0) { - // No tests to run - return a pattern that matches nothing - return "--testPathPattern=^$"; - } - // Return test files space-separated for jest CLI - return result.tests.join(" "); - } -} - -function main(): void { - const options = parseArgs(); - const result = selectAffectedTests(options); - - // Output the result - const output = formatOutput(result, options.outputFormat); - console.log(output); - - // Log summary - log(`Result: ${result.reason}`, options.verbose, true); - if (result.runAll) { - log("Fallback triggered: running all tests", options.verbose, true); - } else if (result.tests.length > 0) { - log( - `Running ${result.tests.length} selected tests`, - options.verbose, - true - ); - } else { - log("No tests need to run", options.verbose, true); - } - - process.exit(result.exitCode); -} - -main(); diff --git a/scripts/selective-tests/types.ts b/scripts/selective-tests/types.ts deleted file mode 100644 index 523e3d541a..0000000000 --- a/scripts/selective-tests/types.ts +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Types for the selective test filtering system. - * - * This system uses runtime coverage data to determine which integration tests - * need to run based on changed source files. - */ - -/** Coverage map: source file → list of test files that cover it */ -export interface CoverageMap { - version: 1; - generatedAt: string; - /** Git commit SHA when the map was generated */ - commitSha: string; - /** Hash of all source files for staleness detection */ - sourceHash: string; - /** Map from relative source file path to array of test file paths */ - fileToTests: Record; - /** List of all test files included in the map */ - allTests: string[]; - /** Files that are considered "infrastructure" - changes trigger all tests */ - infrastructureFiles: string[]; -} - -/** Result from the affected tests selection */ -export interface AffectedTestsResult { - /** Whether to run all tests (fallback triggered) */ - runAll: boolean; - /** Reason for the decision */ - reason: string; - /** List of test files to run (empty if runAll) */ - tests: string[]; - /** Changed files that triggered the selection */ - changedFiles: string[]; - /** Files that were not found in the coverage map */ - unmappedFiles: string[]; -} - -/** Exit codes for scripts (2 = fallback so CI can distinguish from errors) */ -export const EXIT_CODES = { - SUCCESS: 0, - ERROR: 1, - FALLBACK_TRIGGERED: 2, -} as const; - -/** Infrastructure files that should trigger all tests when changed */ -export const INFRASTRUCTURE_PATTERNS = [ - // Core configuration - "jest.config.cjs", - "babel.config.cjs", - "tsconfig.json", - "tsconfig.*.json", - "package.json", - "bun.lockb", - - // Test infrastructure - "tests/setup.ts", - "tests/integration/helpers.ts", - "tests/__mocks__/**", - - // Service container (imports all services) - "src/node/services/serviceContainer.ts", - - // Core shared types - "src/types/**", - "src/constants/**", - - // Build configuration - "vite.*.ts", - - // This selective test system itself - "scripts/selective-tests/**", -]; From 9fa5f7007e6691cfca3f97462dff66f7e44f753e Mon Sep 17 00:00:00 2001 From: Ammar Date: Fri, 5 Dec 2025 12:19:41 -0600 Subject: [PATCH 4/6] refactor: simplify tests folder structure - Flatten tests/integration/ipc/ to tests/ipc/ - Move tests/models/knownModels.test.ts to src/common/constants/ (colocate with implementation) - Delete orphan tests/worker-test.test.ts - Update CI to run tests/ipc + tests/runtime (fixes runtime tests not running) - Update paths-filter backend filter for new structure Tests philosophy: unit tests colocated in src/, integration tests in tests/ --- .github/workflows/pr.yml | 5 +++-- docs/system-prompt.md | 1 - .../common/constants}/knownModels.test.ts | 2 +- tests/{integration => ipc}/anthropic1MContext.test.ts | 0 .../anthropicCacheStrategy.test.ts | 0 tests/{integration => ipc}/createWorkspace.test.ts | 0 tests/{integration => ipc}/doubleRegister.test.ts | 0 tests/{integration => ipc}/executeBash.test.ts | 0 tests/{integration => ipc}/forkWorkspace.test.ts | 0 tests/{integration => ipc}/helpers.ts | 0 tests/{integration => ipc}/initWorkspace.test.ts | 0 tests/{integration => ipc}/modelNotFound.test.ts | 0 tests/{integration => ipc}/ollama.test.ts | 0 tests/{integration => ipc}/openai-web-search.test.ts | 0 tests/{integration => ipc}/orpcTestClient.ts | 0 tests/{integration => ipc}/projectCreate.test.ts | 0 tests/{integration => ipc}/projectRefactor.test.ts | 0 tests/{integration => ipc}/queuedMessages.test.ts | 0 tests/{integration => ipc}/removeWorkspace.test.ts | 0 tests/{integration => ipc}/renameWorkspace.test.ts | 0 tests/{integration => ipc}/resumeStream.test.ts | 0 tests/{integration => ipc}/runtimeExecuteBash.test.ts | 0 tests/{integration => ipc}/runtimeFileEditing.test.ts | 0 tests/{integration => ipc}/sendMessage.basic.test.ts | 0 .../{integration => ipc}/sendMessage.context.test.ts | 0 tests/{integration => ipc}/sendMessage.errors.test.ts | 0 tests/{integration => ipc}/sendMessage.heavy.test.ts | 0 tests/{integration => ipc}/sendMessage.images.test.ts | 0 .../sendMessage.reasoning.test.ts | 0 tests/{integration => ipc}/sendMessageTestHelpers.ts | 0 tests/{integration => ipc}/setup.ts | 0 tests/{integration => ipc}/streamCollector.ts | 0 .../{integration => ipc}/streamErrorRecovery.test.ts | 0 tests/{integration => ipc}/terminal.test.ts | 0 tests/{integration => ipc}/truncate.test.ts | 0 tests/{integration => ipc}/usageDelta.test.ts | 0 .../websocketHistoryReplay.test.ts | 0 tests/{integration => ipc}/windowTitle.test.ts | 0 tests/setup.ts | 2 +- tests/worker-test.test.ts | 11 ----------- 40 files changed, 5 insertions(+), 16 deletions(-) rename {tests/models => src/common/constants}/knownModels.test.ts (95%) rename tests/{integration => ipc}/anthropic1MContext.test.ts (100%) rename tests/{integration => ipc}/anthropicCacheStrategy.test.ts (100%) rename tests/{integration => ipc}/createWorkspace.test.ts (100%) rename tests/{integration => ipc}/doubleRegister.test.ts (100%) rename tests/{integration => ipc}/executeBash.test.ts (100%) rename tests/{integration => ipc}/forkWorkspace.test.ts (100%) rename tests/{integration => ipc}/helpers.ts (100%) rename tests/{integration => ipc}/initWorkspace.test.ts (100%) rename tests/{integration => ipc}/modelNotFound.test.ts (100%) rename tests/{integration => ipc}/ollama.test.ts (100%) rename tests/{integration => ipc}/openai-web-search.test.ts (100%) rename tests/{integration => ipc}/orpcTestClient.ts (100%) rename tests/{integration => ipc}/projectCreate.test.ts (100%) rename tests/{integration => ipc}/projectRefactor.test.ts (100%) rename tests/{integration => ipc}/queuedMessages.test.ts (100%) rename tests/{integration => ipc}/removeWorkspace.test.ts (100%) rename tests/{integration => ipc}/renameWorkspace.test.ts (100%) rename tests/{integration => ipc}/resumeStream.test.ts (100%) rename tests/{integration => ipc}/runtimeExecuteBash.test.ts (100%) rename tests/{integration => ipc}/runtimeFileEditing.test.ts (100%) rename tests/{integration => ipc}/sendMessage.basic.test.ts (100%) rename tests/{integration => ipc}/sendMessage.context.test.ts (100%) rename tests/{integration => ipc}/sendMessage.errors.test.ts (100%) rename tests/{integration => ipc}/sendMessage.heavy.test.ts (100%) rename tests/{integration => ipc}/sendMessage.images.test.ts (100%) rename tests/{integration => ipc}/sendMessage.reasoning.test.ts (100%) rename tests/{integration => ipc}/sendMessageTestHelpers.ts (100%) rename tests/{integration => ipc}/setup.ts (100%) rename tests/{integration => ipc}/streamCollector.ts (100%) rename tests/{integration => ipc}/streamErrorRecovery.test.ts (100%) rename tests/{integration => ipc}/terminal.test.ts (100%) rename tests/{integration => ipc}/truncate.test.ts (100%) rename tests/{integration => ipc}/usageDelta.test.ts (100%) rename tests/{integration => ipc}/websocketHistoryReplay.test.ts (100%) rename tests/{integration => ipc}/windowTitle.test.ts (100%) delete mode 100644 tests/worker-test.test.ts diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 691845c141..26bc4ccb94 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -51,7 +51,8 @@ jobs: - 'src/cli/**' - 'src/desktop/**' - 'src/common/**' - - 'tests/integration/**' + - 'tests/ipc/**' + - 'tests/runtime/**' config: - '.github/**' - 'jest.config.cjs' @@ -122,7 +123,7 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/setup-mux - run: make build-main - - run: TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent ${{ github.event.inputs.test_filter || 'tests' }} + - run: TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent ${{ github.event.inputs.test_filter || 'tests/ipc tests/runtime' }} env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/docs/system-prompt.md b/docs/system-prompt.md index fb5bb1305c..3b7de1ba03 100644 --- a/docs/system-prompt.md +++ b/docs/system-prompt.md @@ -62,5 +62,4 @@ You are in a git worktree at ${workspacePath} } ``` - {/* END SYSTEM_PROMPT_DOCS */} diff --git a/tests/models/knownModels.test.ts b/src/common/constants/knownModels.test.ts similarity index 95% rename from tests/models/knownModels.test.ts rename to src/common/constants/knownModels.test.ts index c880e7a784..d41fcd3447 100644 --- a/tests/models/knownModels.test.ts +++ b/src/common/constants/knownModels.test.ts @@ -41,7 +41,7 @@ describe("Known Models Integration", () => { const lookupKey = model.provider === "xai" ? `xai/${modelId}` : modelId; const modelData = (modelsJson[lookupKey as keyof typeof modelsJson] as Record) ?? - (modelsExtra[modelId as keyof typeof modelsExtra] as unknown as Record); + (modelsExtra[modelId] as Record); expect(modelData).toBeDefined(); // Check that basic metadata fields exist (not all models have all fields) diff --git a/tests/integration/anthropic1MContext.test.ts b/tests/ipc/anthropic1MContext.test.ts similarity index 100% rename from tests/integration/anthropic1MContext.test.ts rename to tests/ipc/anthropic1MContext.test.ts diff --git a/tests/integration/anthropicCacheStrategy.test.ts b/tests/ipc/anthropicCacheStrategy.test.ts similarity index 100% rename from tests/integration/anthropicCacheStrategy.test.ts rename to tests/ipc/anthropicCacheStrategy.test.ts diff --git a/tests/integration/createWorkspace.test.ts b/tests/ipc/createWorkspace.test.ts similarity index 100% rename from tests/integration/createWorkspace.test.ts rename to tests/ipc/createWorkspace.test.ts diff --git a/tests/integration/doubleRegister.test.ts b/tests/ipc/doubleRegister.test.ts similarity index 100% rename from tests/integration/doubleRegister.test.ts rename to tests/ipc/doubleRegister.test.ts diff --git a/tests/integration/executeBash.test.ts b/tests/ipc/executeBash.test.ts similarity index 100% rename from tests/integration/executeBash.test.ts rename to tests/ipc/executeBash.test.ts diff --git a/tests/integration/forkWorkspace.test.ts b/tests/ipc/forkWorkspace.test.ts similarity index 100% rename from tests/integration/forkWorkspace.test.ts rename to tests/ipc/forkWorkspace.test.ts diff --git a/tests/integration/helpers.ts b/tests/ipc/helpers.ts similarity index 100% rename from tests/integration/helpers.ts rename to tests/ipc/helpers.ts diff --git a/tests/integration/initWorkspace.test.ts b/tests/ipc/initWorkspace.test.ts similarity index 100% rename from tests/integration/initWorkspace.test.ts rename to tests/ipc/initWorkspace.test.ts diff --git a/tests/integration/modelNotFound.test.ts b/tests/ipc/modelNotFound.test.ts similarity index 100% rename from tests/integration/modelNotFound.test.ts rename to tests/ipc/modelNotFound.test.ts diff --git a/tests/integration/ollama.test.ts b/tests/ipc/ollama.test.ts similarity index 100% rename from tests/integration/ollama.test.ts rename to tests/ipc/ollama.test.ts diff --git a/tests/integration/openai-web-search.test.ts b/tests/ipc/openai-web-search.test.ts similarity index 100% rename from tests/integration/openai-web-search.test.ts rename to tests/ipc/openai-web-search.test.ts diff --git a/tests/integration/orpcTestClient.ts b/tests/ipc/orpcTestClient.ts similarity index 100% rename from tests/integration/orpcTestClient.ts rename to tests/ipc/orpcTestClient.ts diff --git a/tests/integration/projectCreate.test.ts b/tests/ipc/projectCreate.test.ts similarity index 100% rename from tests/integration/projectCreate.test.ts rename to tests/ipc/projectCreate.test.ts diff --git a/tests/integration/projectRefactor.test.ts b/tests/ipc/projectRefactor.test.ts similarity index 100% rename from tests/integration/projectRefactor.test.ts rename to tests/ipc/projectRefactor.test.ts diff --git a/tests/integration/queuedMessages.test.ts b/tests/ipc/queuedMessages.test.ts similarity index 100% rename from tests/integration/queuedMessages.test.ts rename to tests/ipc/queuedMessages.test.ts diff --git a/tests/integration/removeWorkspace.test.ts b/tests/ipc/removeWorkspace.test.ts similarity index 100% rename from tests/integration/removeWorkspace.test.ts rename to tests/ipc/removeWorkspace.test.ts diff --git a/tests/integration/renameWorkspace.test.ts b/tests/ipc/renameWorkspace.test.ts similarity index 100% rename from tests/integration/renameWorkspace.test.ts rename to tests/ipc/renameWorkspace.test.ts diff --git a/tests/integration/resumeStream.test.ts b/tests/ipc/resumeStream.test.ts similarity index 100% rename from tests/integration/resumeStream.test.ts rename to tests/ipc/resumeStream.test.ts diff --git a/tests/integration/runtimeExecuteBash.test.ts b/tests/ipc/runtimeExecuteBash.test.ts similarity index 100% rename from tests/integration/runtimeExecuteBash.test.ts rename to tests/ipc/runtimeExecuteBash.test.ts diff --git a/tests/integration/runtimeFileEditing.test.ts b/tests/ipc/runtimeFileEditing.test.ts similarity index 100% rename from tests/integration/runtimeFileEditing.test.ts rename to tests/ipc/runtimeFileEditing.test.ts diff --git a/tests/integration/sendMessage.basic.test.ts b/tests/ipc/sendMessage.basic.test.ts similarity index 100% rename from tests/integration/sendMessage.basic.test.ts rename to tests/ipc/sendMessage.basic.test.ts diff --git a/tests/integration/sendMessage.context.test.ts b/tests/ipc/sendMessage.context.test.ts similarity index 100% rename from tests/integration/sendMessage.context.test.ts rename to tests/ipc/sendMessage.context.test.ts diff --git a/tests/integration/sendMessage.errors.test.ts b/tests/ipc/sendMessage.errors.test.ts similarity index 100% rename from tests/integration/sendMessage.errors.test.ts rename to tests/ipc/sendMessage.errors.test.ts diff --git a/tests/integration/sendMessage.heavy.test.ts b/tests/ipc/sendMessage.heavy.test.ts similarity index 100% rename from tests/integration/sendMessage.heavy.test.ts rename to tests/ipc/sendMessage.heavy.test.ts diff --git a/tests/integration/sendMessage.images.test.ts b/tests/ipc/sendMessage.images.test.ts similarity index 100% rename from tests/integration/sendMessage.images.test.ts rename to tests/ipc/sendMessage.images.test.ts diff --git a/tests/integration/sendMessage.reasoning.test.ts b/tests/ipc/sendMessage.reasoning.test.ts similarity index 100% rename from tests/integration/sendMessage.reasoning.test.ts rename to tests/ipc/sendMessage.reasoning.test.ts diff --git a/tests/integration/sendMessageTestHelpers.ts b/tests/ipc/sendMessageTestHelpers.ts similarity index 100% rename from tests/integration/sendMessageTestHelpers.ts rename to tests/ipc/sendMessageTestHelpers.ts diff --git a/tests/integration/setup.ts b/tests/ipc/setup.ts similarity index 100% rename from tests/integration/setup.ts rename to tests/ipc/setup.ts diff --git a/tests/integration/streamCollector.ts b/tests/ipc/streamCollector.ts similarity index 100% rename from tests/integration/streamCollector.ts rename to tests/ipc/streamCollector.ts diff --git a/tests/integration/streamErrorRecovery.test.ts b/tests/ipc/streamErrorRecovery.test.ts similarity index 100% rename from tests/integration/streamErrorRecovery.test.ts rename to tests/ipc/streamErrorRecovery.test.ts diff --git a/tests/integration/terminal.test.ts b/tests/ipc/terminal.test.ts similarity index 100% rename from tests/integration/terminal.test.ts rename to tests/ipc/terminal.test.ts diff --git a/tests/integration/truncate.test.ts b/tests/ipc/truncate.test.ts similarity index 100% rename from tests/integration/truncate.test.ts rename to tests/ipc/truncate.test.ts diff --git a/tests/integration/usageDelta.test.ts b/tests/ipc/usageDelta.test.ts similarity index 100% rename from tests/integration/usageDelta.test.ts rename to tests/ipc/usageDelta.test.ts diff --git a/tests/integration/websocketHistoryReplay.test.ts b/tests/ipc/websocketHistoryReplay.test.ts similarity index 100% rename from tests/integration/websocketHistoryReplay.test.ts rename to tests/ipc/websocketHistoryReplay.test.ts diff --git a/tests/integration/windowTitle.test.ts b/tests/ipc/windowTitle.test.ts similarity index 100% rename from tests/integration/windowTitle.test.ts rename to tests/ipc/windowTitle.test.ts diff --git a/tests/setup.ts b/tests/setup.ts index df6f47bc0e..c334234346 100644 --- a/tests/setup.ts +++ b/tests/setup.ts @@ -28,7 +28,7 @@ if (typeof globalThis.File === "undefined") { if (process.env.TEST_INTEGRATION === "1") { // Store promise globally to ensure it blocks subsequent test execution (globalThis as any).__muxPreloadPromise = (async () => { - const { preloadTestModules } = await import("./integration/setup"); + const { preloadTestModules } = await import("./ipc/setup"); await preloadTestModules(); })(); diff --git a/tests/worker-test.test.ts b/tests/worker-test.test.ts deleted file mode 100644 index 34395a9e47..0000000000 --- a/tests/worker-test.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -describe("Worker test", () => { - it("should preload tokenizers", async () => { - console.log("Test starting..."); - const start = Date.now(); - const { loadTokenizerModules } = await import("../src/node/utils/main/tokenizer"); - console.log("Import done in", Date.now() - start, "ms"); - const result = await loadTokenizerModules(["anthropic:claude-sonnet-4-5"]); - console.log("Result:", result, "in", Date.now() - start, "ms"); - expect(result).toHaveLength(1); - }, 30000); -}); From e509e3ff2107b2c283ede0b482521f830dd69eb0 Mon Sep 17 00:00:00 2001 From: Ammar Date: Fri, 5 Dec 2025 12:24:55 -0600 Subject: [PATCH 5/6] feat: dynamic integration test filtering based on changed paths Instead of skipping the entire integration job, compute which test directories to run based on paths-filter outputs: - ipc filter: src/node/services, orpc, config, git, utils, common, cli, desktop, tests/ipc - runtime filter: src/node/runtime, tests/runtime - config changes: run all integration tests - manual override: workflow_dispatch test_filter still works The job always runs (unless docs-only), but tests are scoped to relevant changes. --- .github/workflows/pr.yml | 71 ++++++++++++++++++------ src/common/constants/knownModels.test.ts | 4 +- 2 files changed, 56 insertions(+), 19 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 26bc4ccb94..1b87df08e0 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -16,14 +16,15 @@ concurrency: cancel-in-progress: true jobs: - # Detect what files changed to skip jobs for docs-only or browser-only PRs + # Detect what files changed to determine which jobs/tests to run changes: name: Detect Changes runs-on: ubuntu-latest outputs: docs-only: ${{ steps.filter.outputs.docs == 'true' && steps.filter.outputs.src == 'false' && steps.filter.outputs.config == 'false' }} - # Browser-only: only browser/renderer code changed, no backend/node changes - browser-only: ${{ steps.filter.outputs.browser == 'true' && steps.filter.outputs.backend == 'false' && steps.filter.outputs.config == 'false' }} + config: ${{ steps.filter.outputs.config }} + ipc: ${{ steps.filter.outputs.ipc }} + runtime: ${{ steps.filter.outputs.runtime }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 @@ -39,19 +40,20 @@ jobs: - 'src/**' - 'tests/**' - 'vscode/**' - browser: - - 'src/browser/**' - - 'src/styles/**' - - 'src/components/**' - - 'src/hooks/**' - - '**/*.stories.tsx' - - '.storybook/**' - backend: - - 'src/node/**' + # IPC tests: backend services, ORPC, config, common types + ipc: + - 'src/node/services/**' + - 'src/node/orpc/**' + - 'src/node/config/**' + - 'src/node/git/**' + - 'src/node/utils/**' + - 'src/common/**' - 'src/cli/**' - 'src/desktop/**' - - 'src/common/**' - 'tests/ipc/**' + # Runtime tests: runtime implementations (SSH, worktree) + runtime: + - 'src/node/runtime/**' - 'tests/runtime/**' config: - '.github/**' @@ -113,8 +115,7 @@ jobs: test-integration: name: Test / Integration needs: [changes] - # Skip for docs-only or browser-only (integration tests are backend IPC tests) - if: ${{ needs.changes.outputs.docs-only != 'true' && needs.changes.outputs.browser-only != 'true' }} + if: ${{ needs.changes.outputs.docs-only != 'true' }} timeout-minutes: 10 runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-16' || 'ubuntu-latest' }} steps: @@ -123,11 +124,49 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/setup-mux - run: make build-main - - run: TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent ${{ github.event.inputs.test_filter || 'tests/ipc tests/runtime' }} + - name: Compute test filter + id: test-filter + run: | + # Manual override takes precedence + if [[ -n "${{ github.event.inputs.test_filter }}" ]]; then + echo "filter=${{ github.event.inputs.test_filter }}" >> $GITHUB_OUTPUT + exit 0 + fi + + # Config changes = run everything + if [[ "${{ needs.changes.outputs.config }}" == "true" ]]; then + echo "filter=tests/ipc tests/runtime" >> $GITHUB_OUTPUT + exit 0 + fi + + # Build filter from changed paths + paths="" + if [[ "${{ needs.changes.outputs.ipc }}" == "true" ]]; then + paths="$paths tests/ipc" + fi + if [[ "${{ needs.changes.outputs.runtime }}" == "true" ]]; then + paths="$paths tests/runtime" + fi + paths=$(echo $paths | xargs) # trim whitespace + + if [[ -z "$paths" ]]; then + echo "No integration tests needed for changed paths" + echo "filter=" >> $GITHUB_OUTPUT + else + echo "Running: $paths" + echo "filter=$paths" >> $GITHUB_OUTPUT + fi + - name: Run integration tests + if: ${{ steps.test-filter.outputs.filter != '' }} + run: TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent ${{ steps.test-filter.outputs.filter }} env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + - name: Skip integration tests + if: ${{ steps.test-filter.outputs.filter == '' }} + run: echo "Skipping integration tests - no relevant paths changed" - uses: codecov/codecov-action@v5 + if: ${{ steps.test-filter.outputs.filter != '' }} with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage/lcov.info diff --git a/src/common/constants/knownModels.test.ts b/src/common/constants/knownModels.test.ts index d41fcd3447..ad8f9882c9 100644 --- a/src/common/constants/knownModels.test.ts +++ b/src/common/constants/knownModels.test.ts @@ -39,9 +39,7 @@ describe("Known Models Integration", () => { const modelId = model.providerModelId; // xAI models are prefixed with "xai/" in models.json const lookupKey = model.provider === "xai" ? `xai/${modelId}` : modelId; - const modelData = - (modelsJson[lookupKey as keyof typeof modelsJson] as Record) ?? - (modelsExtra[modelId] as Record); + const modelData = modelsJson[lookupKey as keyof typeof modelsJson] ?? modelsExtra[modelId]; expect(modelData).toBeDefined(); // Check that basic metadata fields exist (not all models have all fields) From e6ca02eaad185dcb841882ca5c24f0702f88ea9b Mon Sep 17 00:00:00 2001 From: Ammar Date: Fri, 5 Dec 2025 12:28:10 -0600 Subject: [PATCH 6/6] simplify: integration test filtering (future-proof) - docs-only: skip entire integration job (job-level if) - browser-only: skip tests/ipc, run all other tests under tests/ - otherwise: run ALL tests under tests/ (new directories auto-included) Uses --testPathIgnorePatterns for browser-only exclusion. Jest's testMatch only picks up *.test.ts, so e2e/*.spec.ts is already excluded. --- .github/workflows/pr.yml | 66 ++++++++++++--------------------------- tests/worker-test.test.ts | 11 +++++++ 2 files changed, 31 insertions(+), 46 deletions(-) create mode 100644 tests/worker-test.test.ts diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1b87df08e0..ea8fb2e73d 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -22,9 +22,7 @@ jobs: runs-on: ubuntu-latest outputs: docs-only: ${{ steps.filter.outputs.docs == 'true' && steps.filter.outputs.src == 'false' && steps.filter.outputs.config == 'false' }} - config: ${{ steps.filter.outputs.config }} - ipc: ${{ steps.filter.outputs.ipc }} - runtime: ${{ steps.filter.outputs.runtime }} + browser-only: ${{ steps.filter.outputs.browser == 'true' && steps.filter.outputs.backend == 'false' && steps.filter.outputs.config == 'false' }} steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@v3 @@ -40,21 +38,18 @@ jobs: - 'src/**' - 'tests/**' - 'vscode/**' - # IPC tests: backend services, ORPC, config, common types - ipc: - - 'src/node/services/**' - - 'src/node/orpc/**' - - 'src/node/config/**' - - 'src/node/git/**' - - 'src/node/utils/**' - - 'src/common/**' + browser: + - 'src/browser/**' + - 'src/styles/**' + - '**/*.stories.tsx' + - '.storybook/**' + backend: + - 'src/node/**' - 'src/cli/**' - 'src/desktop/**' - - 'tests/ipc/**' - # Runtime tests: runtime implementations (SSH, worktree) - runtime: - - 'src/node/runtime/**' - - 'tests/runtime/**' + - 'src/common/**' + - 'tests/**' + - '!tests/e2e/**' config: - '.github/**' - 'jest.config.cjs' @@ -124,47 +119,26 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/setup-mux - run: make build-main - - name: Compute test filter - id: test-filter + - name: Run integration tests run: | - # Manual override takes precedence + # Manual override if [[ -n "${{ github.event.inputs.test_filter }}" ]]; then - echo "filter=${{ github.event.inputs.test_filter }}" >> $GITHUB_OUTPUT + TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent ${{ github.event.inputs.test_filter }} exit 0 fi - # Config changes = run everything - if [[ "${{ needs.changes.outputs.config }}" == "true" ]]; then - echo "filter=tests/ipc tests/runtime" >> $GITHUB_OUTPUT + # Browser-only PRs skip IPC tests but run all other integration tests + if [[ "${{ needs.changes.outputs.browser-only }}" == "true" ]]; then + echo "Browser-only PR - skipping tests/ipc" + TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent --testPathIgnorePatterns='tests/ipc' tests/ exit 0 fi - # Build filter from changed paths - paths="" - if [[ "${{ needs.changes.outputs.ipc }}" == "true" ]]; then - paths="$paths tests/ipc" - fi - if [[ "${{ needs.changes.outputs.runtime }}" == "true" ]]; then - paths="$paths tests/runtime" - fi - paths=$(echo $paths | xargs) # trim whitespace - - if [[ -z "$paths" ]]; then - echo "No integration tests needed for changed paths" - echo "filter=" >> $GITHUB_OUTPUT - else - echo "Running: $paths" - echo "filter=$paths" >> $GITHUB_OUTPUT - fi - - name: Run integration tests - if: ${{ steps.test-filter.outputs.filter != '' }} - run: TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent ${{ steps.test-filter.outputs.filter }} + # Default: run ALL integration tests (future-proof for new test directories) + TEST_INTEGRATION=1 bun x jest --coverage --maxWorkers=100% --silent tests/ env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - - name: Skip integration tests - if: ${{ steps.test-filter.outputs.filter == '' }} - run: echo "Skipping integration tests - no relevant paths changed" - uses: codecov/codecov-action@v5 if: ${{ steps.test-filter.outputs.filter != '' }} with: diff --git a/tests/worker-test.test.ts b/tests/worker-test.test.ts new file mode 100644 index 0000000000..34395a9e47 --- /dev/null +++ b/tests/worker-test.test.ts @@ -0,0 +1,11 @@ +describe("Worker test", () => { + it("should preload tokenizers", async () => { + console.log("Test starting..."); + const start = Date.now(); + const { loadTokenizerModules } = await import("../src/node/utils/main/tokenizer"); + console.log("Import done in", Date.now() - start, "ms"); + const result = await loadTokenizerModules(["anthropic:claude-sonnet-4-5"]); + console.log("Result:", result, "in", Date.now() - start, "ms"); + expect(result).toHaveLength(1); + }, 30000); +});