From 4994ac165c6c7cd8a59e4528846fd56c604a7f6d Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Wed, 16 Jul 2025 16:08:35 +0100 Subject: [PATCH 1/8] Add Windows debugging scripts for flaky stdio test Add PowerShell scripts to help debug test_stdio_context_manager_exiting flakiness on Windows with Python 3.11/3.12: - check-tee-command.ps1: Verify tee command availability - test-stdio-flakiness-200-runs.ps1: Run test 200 times for statistics - test-stdio-flakiness-until-failure.ps1: Run until failure occurs - test-stdio-verbose-debug.ps1: Single run with maximum debug output - README.md: Documentation for using the scripts --- scripts/windows-debug/README.md | 55 ++++++++++ scripts/windows-debug/check-tee-command.ps1 | 101 ++++++++++++++++++ .../test-stdio-flakiness-200-runs.ps1 | 44 ++++++++ .../test-stdio-flakiness-until-failure.ps1 | 42 ++++++++ .../test-stdio-verbose-debug.ps1 | 40 +++++++ 5 files changed, 282 insertions(+) create mode 100644 scripts/windows-debug/README.md create mode 100644 scripts/windows-debug/check-tee-command.ps1 create mode 100644 scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 create mode 100644 scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 create mode 100644 scripts/windows-debug/test-stdio-verbose-debug.ps1 diff --git a/scripts/windows-debug/README.md b/scripts/windows-debug/README.md new file mode 100644 index 0000000000..378b7d0e8f --- /dev/null +++ b/scripts/windows-debug/README.md @@ -0,0 +1,55 @@ +# Windows Test Debugging Scripts + +This folder contains PowerShell scripts to help debug the flaky `test_stdio_context_manager_exiting` test on Windows with Python 3.11/3.12. + +## Prerequisites + +- Windows PowerShell or PowerShell Core +- Python 3.11 or 3.12 +- `uv` installed and configured +- Virtual environment activated + +## Scripts + +### 1. `check-tee-command.ps1` +Checks if the `tee` command is available on your Windows system. The test requires `tee` to run. + +```powershell +.\check-tee-command.ps1 +``` + +### 2. `test-stdio-flakiness-200-runs.ps1` +Runs the test 200 times and reports the failure rate. + +```powershell +.\test-stdio-flakiness-200-runs.ps1 +``` + +### 3. `test-stdio-flakiness-until-failure.ps1` +Runs the test in a loop until it fails, useful for catching intermittent failures. + +```powershell +.\test-stdio-flakiness-until-failure.ps1 +``` + +### 4. `test-stdio-verbose-debug.ps1` +Runs the test once with maximum debugging output enabled. + +```powershell +.\test-stdio-verbose-debug.ps1 +``` + +## Usage Notes + +1. Make sure your virtual environment is activated before running these scripts +2. Run scripts from the repository root directory +3. If scripts fail with execution policy errors, run: + ```powershell + Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + ``` + +## Interpreting Results + +- If `tee` is not available, the test will be skipped +- Flaky failures typically show up as timeout errors or process cleanup issues +- Look for patterns in failure output, especially around process termination timing \ No newline at end of file diff --git a/scripts/windows-debug/check-tee-command.ps1 b/scripts/windows-debug/check-tee-command.ps1 new file mode 100644 index 0000000000..68d83bb225 --- /dev/null +++ b/scripts/windows-debug/check-tee-command.ps1 @@ -0,0 +1,101 @@ +#!/usr/bin/env pwsh +# Script to check if tee command is available on Windows +# Usage: .\check-tee-command.ps1 + +Write-Host "Checking for 'tee' command availability on Windows..." -ForegroundColor Cyan +Write-Host "" + +# Method 1: Using where.exe +Write-Host "Method 1: Using where.exe" -ForegroundColor Yellow +try { + $whereResult = where.exe tee 2>&1 + if ($LASTEXITCODE -eq 0) { + Write-Host " Found tee at: $whereResult" -ForegroundColor Green + } else { + Write-Host " tee not found via where.exe" -ForegroundColor Red + } +} catch { + Write-Host " Error checking with where.exe: $_" -ForegroundColor Red +} + +Write-Host "" + +# Method 2: Using Get-Command +Write-Host "Method 2: Using Get-Command" -ForegroundColor Yellow +try { + $getCommandResult = Get-Command tee -ErrorAction Stop + Write-Host " Found tee:" -ForegroundColor Green + Write-Host " Name: $($getCommandResult.Name)" + Write-Host " CommandType: $($getCommandResult.CommandType)" + Write-Host " Source: $($getCommandResult.Source)" + Write-Host " Version: $($getCommandResult.Version)" +} catch { + Write-Host " tee not found via Get-Command" -ForegroundColor Red +} + +Write-Host "" + +# Method 3: Check common locations +Write-Host "Method 3: Checking common locations" -ForegroundColor Yellow +$commonPaths = @( + "C:\Program Files\Git\usr\bin\tee.exe", + "C:\Program Files (x86)\Git\usr\bin\tee.exe", + "C:\tools\msys64\usr\bin\tee.exe", + "C:\msys64\usr\bin\tee.exe", + "C:\cygwin64\bin\tee.exe", + "C:\cygwin\bin\tee.exe" +) + +$found = $false +foreach ($path in $commonPaths) { + if (Test-Path $path) { + Write-Host " Found at: $path" -ForegroundColor Green + $found = $true + } +} + +if (-not $found) { + Write-Host " tee not found in common locations" -ForegroundColor Red +} + +Write-Host "" + +# Method 4: Check if it's a PowerShell alias +Write-Host "Method 4: Checking PowerShell aliases" -ForegroundColor Yellow +$alias = Get-Alias tee -ErrorAction SilentlyContinue +if ($alias) { + Write-Host " Found PowerShell alias:" -ForegroundColor Green + Write-Host " Name: $($alias.Name)" + Write-Host " Definition: $($alias.Definition)" +} else { + Write-Host " No PowerShell alias for tee" -ForegroundColor Yellow +} + +Write-Host "" + +# Method 5: Python check (what the test uses) +Write-Host "Method 5: Python shutil.which() check" -ForegroundColor Yellow +$pythonCheck = python -c "import shutil; print(shutil.which('tee'))" +if ($pythonCheck -and $pythonCheck -ne "None") { + Write-Host " Python found tee at: $pythonCheck" -ForegroundColor Green +} else { + Write-Host " Python shutil.which() did not find tee" -ForegroundColor Red +} + +Write-Host "" +Write-Host "========== SUMMARY ==========" -ForegroundColor Cyan +if ($whereResult -or $getCommandResult -or $found -or ($pythonCheck -and $pythonCheck -ne "None")) { + Write-Host "tee command is available" -ForegroundColor Green + Write-Host "" + Write-Host "The test_stdio_context_manager_exiting test should run." -ForegroundColor Green +} else { + Write-Host "tee command is NOT available" -ForegroundColor Red + Write-Host "" + Write-Host "The test_stdio_context_manager_exiting test will be SKIPPED." -ForegroundColor Yellow + Write-Host "" + Write-Host "To install tee on Windows, you can:" -ForegroundColor Cyan + Write-Host " 1. Install Git for Windows (includes tee in Git Bash)" + Write-Host " 2. Install WSL (Windows Subsystem for Linux)" + Write-Host " 3. Install MSYS2 or Cygwin" + Write-Host " 4. Use PowerShell's Tee-Object cmdlet (different syntax)" +} \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 b/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 new file mode 100644 index 0000000000..691f4aac28 --- /dev/null +++ b/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 @@ -0,0 +1,44 @@ +#!/usr/bin/env pwsh +# Script to run test_stdio_context_manager_exiting 200 times to detect flakiness +# Usage: .\test-stdio-flakiness-200-runs.ps1 + +Write-Host "Running test_stdio_context_manager_exiting 200 times to detect flakiness..." -ForegroundColor Cyan +Write-Host "Test: tests/client/test_stdio.py::test_stdio_context_manager_exiting" -ForegroundColor Yellow +Write-Host "" + +$startTime = Get-Date +$count = 0 +$failures = 0 +$failedRuns = @() + +for ($i = 1; $i -le 200; $i++) { + Write-Host "Run $i of 200..." -NoNewline + + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs 2>&1 + $exitCode = $LASTEXITCODE + + if ($exitCode -ne 0) { + $failures++ + $failedRuns += $i + Write-Host " FAILED" -ForegroundColor Red + Write-Host "Failure output:" -ForegroundColor Red + Write-Host $output + Write-Host "" + } else { + Write-Host " PASSED" -ForegroundColor Green + } +} + +$endTime = Get-Date +$duration = $endTime - $startTime + +Write-Host "" +Write-Host "========== SUMMARY ==========" -ForegroundColor Cyan +Write-Host "Total runs: 200" +Write-Host "Successful runs: $($count - $failures)" -ForegroundColor Green +Write-Host "Failed runs: $failures" -ForegroundColor Red +if ($failures -gt 0) { + Write-Host "Failed on runs: $($failedRuns -join ', ')" -ForegroundColor Red +} +Write-Host "Duration: $($duration.ToString())" +Write-Host "Failure rate: $([math]::Round(($failures / 200) * 100, 2))%" \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 b/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 new file mode 100644 index 0000000000..66b705aa4d --- /dev/null +++ b/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 @@ -0,0 +1,42 @@ +#!/usr/bin/env pwsh +# Script to run test_stdio_context_manager_exiting until it fails +# Usage: .\test-stdio-flakiness-until-failure.ps1 + +Write-Host "Running test_stdio_context_manager_exiting until failure..." -ForegroundColor Cyan +Write-Host "Test: tests/client/test_stdio.py::test_stdio_context_manager_exiting" -ForegroundColor Yellow +Write-Host "Press Ctrl+C to stop" -ForegroundColor Gray +Write-Host "" + +$startTime = Get-Date +$i = 0 + +while ($true) { + $i++ + Write-Host "Run $i..." -NoNewline + + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs 2>&1 + $exitCode = $LASTEXITCODE + + if ($exitCode -ne 0) { + $endTime = Get-Date + $duration = $endTime - $startTime + + Write-Host " FAILED!" -ForegroundColor Red + Write-Host "" + Write-Host "========== FAILURE DETECTED ==========" -ForegroundColor Red + Write-Host "Failed on run: $i" -ForegroundColor Red + Write-Host "Time until failure: $($duration.ToString())" -ForegroundColor Yellow + Write-Host "" + Write-Host "Failure output:" -ForegroundColor Red + Write-Host $output + break + } else { + Write-Host " PASSED" -ForegroundColor Green + } + + # Small delay to prevent overwhelming the system + Start-Sleep -Milliseconds 100 +} + +Write-Host "" +Write-Host "Exiting after failure detection." -ForegroundColor Cyan \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-verbose-debug.ps1 b/scripts/windows-debug/test-stdio-verbose-debug.ps1 new file mode 100644 index 0000000000..0e5622779c --- /dev/null +++ b/scripts/windows-debug/test-stdio-verbose-debug.ps1 @@ -0,0 +1,40 @@ +#!/usr/bin/env pwsh +# Script to run test_stdio_context_manager_exiting with maximum debugging output +# Usage: .\test-stdio-verbose-debug.ps1 + +Write-Host "Running test_stdio_context_manager_exiting with verbose debug output..." -ForegroundColor Cyan +Write-Host "" + +# Set environment variables for debugging +$env:PYTHONFAULTHANDLER = "1" +$env:PYTEST_CURRENT_TEST = "1" + +Write-Host "Environment variables set:" -ForegroundColor Yellow +Write-Host " PYTHONFAULTHANDLER = 1 (enables Python fault handler)" +Write-Host "" + +Write-Host "Running test with maximum verbosity..." -ForegroundColor Cyan +Write-Host "" + +# Run the test with all debugging options +uv run --frozen pytest ` + tests/client/test_stdio.py::test_stdio_context_manager_exiting ` + -xvs ` + --log-cli-level=DEBUG ` + --log-cli-format="%(asctime)s [%(levelname)s] %(name)s: %(message)s" ` + --capture=no ` + --tb=long ` + --full-trace + +$exitCode = $LASTEXITCODE + +Write-Host "" +if ($exitCode -eq 0) { + Write-Host "Test PASSED" -ForegroundColor Green +} else { + Write-Host "Test FAILED with exit code: $exitCode" -ForegroundColor Red +} + +# Clean up environment variables +Remove-Item Env:PYTHONFAULTHANDLER -ErrorAction SilentlyContinue +Remove-Item Env:PYTEST_CURRENT_TEST -ErrorAction SilentlyContinue \ No newline at end of file From 154ef9c6bca527c4ee51ad1bfa5890e95a351c55 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Wed, 16 Jul 2025 16:15:17 +0100 Subject: [PATCH 2/8] Fix pytest-xdist issue in Windows debug scripts Add -n 0 flag to disable xdist parallel execution which was causing test collection failures. Also fix successful run count calculation. --- scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 | 4 ++-- scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 | 2 +- scripts/windows-debug/test-stdio-verbose-debug.ps1 | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 b/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 index 691f4aac28..5d30c2e070 100644 --- a/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 +++ b/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 @@ -14,7 +14,7 @@ $failedRuns = @() for ($i = 1; $i -le 200; $i++) { Write-Host "Run $i of 200..." -NoNewline - $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs 2>&1 + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -n 0 2>&1 $exitCode = $LASTEXITCODE if ($exitCode -ne 0) { @@ -35,7 +35,7 @@ $duration = $endTime - $startTime Write-Host "" Write-Host "========== SUMMARY ==========" -ForegroundColor Cyan Write-Host "Total runs: 200" -Write-Host "Successful runs: $($count - $failures)" -ForegroundColor Green +Write-Host "Successful runs: $(200 - $failures)" -ForegroundColor Green Write-Host "Failed runs: $failures" -ForegroundColor Red if ($failures -gt 0) { Write-Host "Failed on runs: $($failedRuns -join ', ')" -ForegroundColor Red diff --git a/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 b/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 index 66b705aa4d..8724502b37 100644 --- a/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 +++ b/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 @@ -14,7 +14,7 @@ while ($true) { $i++ Write-Host "Run $i..." -NoNewline - $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs 2>&1 + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -n 0 2>&1 $exitCode = $LASTEXITCODE if ($exitCode -ne 0) { diff --git a/scripts/windows-debug/test-stdio-verbose-debug.ps1 b/scripts/windows-debug/test-stdio-verbose-debug.ps1 index 0e5622779c..29f47ff7da 100644 --- a/scripts/windows-debug/test-stdio-verbose-debug.ps1 +++ b/scripts/windows-debug/test-stdio-verbose-debug.ps1 @@ -20,6 +20,7 @@ Write-Host "" uv run --frozen pytest ` tests/client/test_stdio.py::test_stdio_context_manager_exiting ` -xvs ` + -n 0 ` --log-cli-level=DEBUG ` --log-cli-format="%(asctime)s [%(levelname)s] %(name)s: %(message)s" ` --capture=no ` From 781367c52c60edf8ba4b13e3e0d1a28906b6fc92 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Wed, 16 Jul 2025 16:17:14 +0100 Subject: [PATCH 3/8] Fix xdist issue with multiple approaches - Use PYTEST_DISABLE_PLUGIN_AUTOLOAD environment variable (recommended in CLAUDE.md) - Use -p no:xdist flag to explicitly disable the plugin - Add --no-cov to avoid coverage plugin issues - Add test-stdio-simple.ps1 to try different methods of disabling xdist --- .../test-stdio-flakiness-200-runs.ps1 | 13 ++++- .../test-stdio-flakiness-until-failure.ps1 | 13 ++++- scripts/windows-debug/test-stdio-simple.ps1 | 54 +++++++++++++++++++ .../test-stdio-verbose-debug.ps1 | 9 +++- 4 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 scripts/windows-debug/test-stdio-simple.ps1 diff --git a/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 b/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 index 5d30c2e070..54df0c9a12 100644 --- a/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 +++ b/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 @@ -6,6 +6,9 @@ Write-Host "Running test_stdio_context_manager_exiting 200 times to detect flaki Write-Host "Test: tests/client/test_stdio.py::test_stdio_context_manager_exiting" -ForegroundColor Yellow Write-Host "" +# Disable pytest plugin autoload to avoid xdist issues +$env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" + $startTime = Get-Date $count = 0 $failures = 0 @@ -14,7 +17,10 @@ $failedRuns = @() for ($i = 1; $i -le 200; $i++) { Write-Host "Run $i of 200..." -NoNewline - $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -n 0 2>&1 + $output = & { + $env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" + uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs --no-cov -p no:xdist 2>&1 + } $exitCode = $LASTEXITCODE if ($exitCode -ne 0) { @@ -41,4 +47,7 @@ if ($failures -gt 0) { Write-Host "Failed on runs: $($failedRuns -join ', ')" -ForegroundColor Red } Write-Host "Duration: $($duration.ToString())" -Write-Host "Failure rate: $([math]::Round(($failures / 200) * 100, 2))%" \ No newline at end of file +Write-Host "Failure rate: $([math]::Round(($failures / 200) * 100, 2))%" + +# Clean up environment variable +Remove-Item Env:PYTEST_DISABLE_PLUGIN_AUTOLOAD -ErrorAction SilentlyContinue \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 b/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 index 8724502b37..4c0e047414 100644 --- a/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 +++ b/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 @@ -7,6 +7,9 @@ Write-Host "Test: tests/client/test_stdio.py::test_stdio_context_manager_exiting Write-Host "Press Ctrl+C to stop" -ForegroundColor Gray Write-Host "" +# Disable pytest plugin autoload to avoid xdist issues +$env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" + $startTime = Get-Date $i = 0 @@ -14,7 +17,10 @@ while ($true) { $i++ Write-Host "Run $i..." -NoNewline - $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -n 0 2>&1 + $output = & { + $env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" + uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs --no-cov -p no:xdist 2>&1 + } $exitCode = $LASTEXITCODE if ($exitCode -ne 0) { @@ -39,4 +45,7 @@ while ($true) { } Write-Host "" -Write-Host "Exiting after failure detection." -ForegroundColor Cyan \ No newline at end of file +Write-Host "Exiting after failure detection." -ForegroundColor Cyan + +# Clean up environment variable +Remove-Item Env:PYTEST_DISABLE_PLUGIN_AUTOLOAD -ErrorAction SilentlyContinue \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-simple.ps1 b/scripts/windows-debug/test-stdio-simple.ps1 new file mode 100644 index 0000000000..d249703c42 --- /dev/null +++ b/scripts/windows-debug/test-stdio-simple.ps1 @@ -0,0 +1,54 @@ +#!/usr/bin/env pwsh +# Simple script to run the test without xdist +# Usage: .\test-stdio-simple.ps1 + +Write-Host "Running test_stdio_context_manager_exiting with xdist disabled..." -ForegroundColor Cyan +Write-Host "" + +# Method 1: Using environment variable (recommended in CLAUDE.md) +Write-Host "Method 1: Using PYTEST_DISABLE_PLUGIN_AUTOLOAD environment variable" -ForegroundColor Yellow +$env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" +uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs +$exitCode1 = $LASTEXITCODE +Remove-Item Env:PYTEST_DISABLE_PLUGIN_AUTOLOAD -ErrorAction SilentlyContinue + +if ($exitCode1 -eq 0) { + Write-Host "Method 1: PASSED" -ForegroundColor Green +} else { + Write-Host "Method 1: FAILED with exit code $exitCode1" -ForegroundColor Red +} + +Write-Host "" + +# Method 2: Using -p no:xdist to disable the plugin +Write-Host "Method 2: Using -p no:xdist flag" -ForegroundColor Yellow +uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -p no:xdist +$exitCode2 = $LASTEXITCODE + +if ($exitCode2 -eq 0) { + Write-Host "Method 2: PASSED" -ForegroundColor Green +} else { + Write-Host "Method 2: FAILED with exit code $exitCode2" -ForegroundColor Red +} + +Write-Host "" + +# Method 3: Override addopts from pyproject.toml +Write-Host "Method 3: Overriding pytest addopts" -ForegroundColor Yellow +uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -o addopts="" +$exitCode3 = $LASTEXITCODE + +if ($exitCode3 -eq 0) { + Write-Host "Method 3: PASSED" -ForegroundColor Green +} else { + Write-Host "Method 3: FAILED with exit code $exitCode3" -ForegroundColor Red +} + +Write-Host "" +Write-Host "========== SUMMARY ==========" -ForegroundColor Cyan +if ($exitCode1 -eq 0 -or $exitCode2 -eq 0 -or $exitCode3 -eq 0) { + Write-Host "At least one method succeeded!" -ForegroundColor Green + Write-Host "Use the successful method in your testing." +} else { + Write-Host "All methods failed. The test may have a different issue." -ForegroundColor Red +} \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-verbose-debug.ps1 b/scripts/windows-debug/test-stdio-verbose-debug.ps1 index 29f47ff7da..9696af81eb 100644 --- a/scripts/windows-debug/test-stdio-verbose-debug.ps1 +++ b/scripts/windows-debug/test-stdio-verbose-debug.ps1 @@ -8,19 +8,23 @@ Write-Host "" # Set environment variables for debugging $env:PYTHONFAULTHANDLER = "1" $env:PYTEST_CURRENT_TEST = "1" +$env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" Write-Host "Environment variables set:" -ForegroundColor Yellow Write-Host " PYTHONFAULTHANDLER = 1 (enables Python fault handler)" +Write-Host " PYTEST_DISABLE_PLUGIN_AUTOLOAD = '' (disables pytest plugin autoload)" Write-Host "" Write-Host "Running test with maximum verbosity..." -ForegroundColor Cyan Write-Host "" # Run the test with all debugging options +$env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" uv run --frozen pytest ` tests/client/test_stdio.py::test_stdio_context_manager_exiting ` -xvs ` - -n 0 ` + --no-cov ` + -p no:xdist ` --log-cli-level=DEBUG ` --log-cli-format="%(asctime)s [%(levelname)s] %(name)s: %(message)s" ` --capture=no ` @@ -38,4 +42,5 @@ if ($exitCode -eq 0) { # Clean up environment variables Remove-Item Env:PYTHONFAULTHANDLER -ErrorAction SilentlyContinue -Remove-Item Env:PYTEST_CURRENT_TEST -ErrorAction SilentlyContinue \ No newline at end of file +Remove-Item Env:PYTEST_CURRENT_TEST -ErrorAction SilentlyContinue +Remove-Item Env:PYTEST_DISABLE_PLUGIN_AUTOLOAD -ErrorAction SilentlyContinue \ No newline at end of file From b0edf56111814a443b5b02edb542d175e8228112 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Wed, 16 Jul 2025 16:25:58 +0100 Subject: [PATCH 4/8] Simplify Windows debug scripts based on working solution - Add setup-environment.ps1 to configure Git for Windows tee in PATH - Simplify all scripts to use working -o addopts="" approach - Add tee availability checks to all scripts - Update README with clear setup instructions and troubleshooting - Remove complex xdist workarounds in favor of simple override --- scripts/windows-debug/README.md | 64 +++++++++++++++++-- scripts/windows-debug/check-tee-command.ps1 | 45 ++++++++++++- scripts/windows-debug/setup-environment.ps1 | 59 +++++++++++++++++ .../test-stdio-flakiness-200-runs.ps1 | 22 ++++--- .../test-stdio-flakiness-until-failure.ps1 | 22 ++++--- scripts/windows-debug/test-stdio-simple.ps1 | 62 +++++++----------- .../test-stdio-verbose-debug.ps1 | 20 ++++-- 7 files changed, 220 insertions(+), 74 deletions(-) create mode 100644 scripts/windows-debug/setup-environment.ps1 diff --git a/scripts/windows-debug/README.md b/scripts/windows-debug/README.md index 378b7d0e8f..9f33eed175 100644 --- a/scripts/windows-debug/README.md +++ b/scripts/windows-debug/README.md @@ -8,31 +8,64 @@ This folder contains PowerShell scripts to help debug the flaky `test_stdio_cont - Python 3.11 or 3.12 - `uv` installed and configured - Virtual environment activated +- Git for Windows installed (provides the `tee` command) + +## Quick Start + +1. First, set up your environment to make `tee` available: + ```powershell + . .\setup-environment.ps1 + ``` + **Note:** The dot (.) at the beginning is important - it sources the script. + +2. Verify the test runs: + ```powershell + .\test-stdio-simple.ps1 + ``` + +3. Run the flakiness tests: + ```powershell + .\test-stdio-flakiness-200-runs.ps1 + ``` ## Scripts -### 1. `check-tee-command.ps1` +### 1. `setup-environment.ps1` +Sets up the environment by adding Git for Windows tools to PATH. **Must be dot-sourced.** + +```powershell +. .\setup-environment.ps1 +``` + +### 2. `check-tee-command.ps1` Checks if the `tee` command is available on your Windows system. The test requires `tee` to run. ```powershell .\check-tee-command.ps1 ``` -### 2. `test-stdio-flakiness-200-runs.ps1` +### 3. `test-stdio-simple.ps1` +Runs the test once to verify it works. + +```powershell +.\test-stdio-simple.ps1 +``` + +### 4. `test-stdio-flakiness-200-runs.ps1` Runs the test 200 times and reports the failure rate. ```powershell .\test-stdio-flakiness-200-runs.ps1 ``` -### 3. `test-stdio-flakiness-until-failure.ps1` +### 5. `test-stdio-flakiness-until-failure.ps1` Runs the test in a loop until it fails, useful for catching intermittent failures. ```powershell .\test-stdio-flakiness-until-failure.ps1 ``` -### 4. `test-stdio-verbose-debug.ps1` +### 6. `test-stdio-verbose-debug.ps1` Runs the test once with maximum debugging output enabled. ```powershell @@ -42,8 +75,9 @@ Runs the test once with maximum debugging output enabled. ## Usage Notes 1. Make sure your virtual environment is activated before running these scripts -2. Run scripts from the repository root directory -3. If scripts fail with execution policy errors, run: +2. Always run `setup-environment.ps1` first in each new PowerShell session +3. Run scripts from the `scripts\windows-debug` directory +4. If scripts fail with execution policy errors, run: ```powershell Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser ``` @@ -52,4 +86,20 @@ Runs the test once with maximum debugging output enabled. - If `tee` is not available, the test will be skipped - Flaky failures typically show up as timeout errors or process cleanup issues -- Look for patterns in failure output, especially around process termination timing \ No newline at end of file +- Look for patterns in failure output, especially around process termination timing + +## Troubleshooting + +### "tee command not found" +1. Install Git for Windows from https://gitforwindows.org/ +2. Run `. .\setup-environment.ps1` to add Git tools to PATH +3. Verify with `python -c "import shutil; print(shutil.which('tee'))"` + +### "no tests ran" with xdist +The scripts now use `-o addopts=""` to override the pytest configuration that enables xdist by default. + +### Making PATH changes permanent +Add this line to your PowerShell profile: +```powershell +$env:PATH = "C:\Program Files\Git\usr\bin;$env:PATH" +``` \ No newline at end of file diff --git a/scripts/windows-debug/check-tee-command.ps1 b/scripts/windows-debug/check-tee-command.ps1 index 68d83bb225..46fa7f093e 100644 --- a/scripts/windows-debug/check-tee-command.ps1 +++ b/scripts/windows-debug/check-tee-command.ps1 @@ -5,6 +5,9 @@ Write-Host "Checking for 'tee' command availability on Windows..." -ForegroundColor Cyan Write-Host "" +# Store original PATH +$originalPath = $env:PATH + # Method 1: Using where.exe Write-Host "Method 1: Using where.exe" -ForegroundColor Yellow try { @@ -82,12 +85,47 @@ if ($pythonCheck -and $pythonCheck -ne "None") { Write-Host " Python shutil.which() did not find tee" -ForegroundColor Red } +Write-Host "" + +# Method 6: Try adding Git for Windows to PATH if it exists +Write-Host "Method 6: Adding Git for Windows to PATH temporarily" -ForegroundColor Yellow +$gitPaths = @( + "C:\Program Files\Git\usr\bin", + "C:\Program Files (x86)\Git\usr\bin" +) + +$addedToPath = $false +foreach ($gitPath in $gitPaths) { + if (Test-Path $gitPath) { + Write-Host " Found Git directory: $gitPath" -ForegroundColor Green + $env:PATH = "$gitPath;$env:PATH" + $teeCheck = python -c "import shutil; print(shutil.which('tee'))" + if ($teeCheck -and $teeCheck -ne "None") { + Write-Host " tee is now available at: $teeCheck" -ForegroundColor Green + $addedToPath = $true + break + } + } +} + +if (-not $addedToPath) { + # Restore original PATH if we didn't find tee + $env:PATH = $originalPath + Write-Host " Could not add Git for Windows tee to PATH" -ForegroundColor Red +} + Write-Host "" Write-Host "========== SUMMARY ==========" -ForegroundColor Cyan -if ($whereResult -or $getCommandResult -or $found -or ($pythonCheck -and $pythonCheck -ne "None")) { +if ($whereResult -or $getCommandResult -or $found -or ($pythonCheck -and $pythonCheck -ne "None") -or $addedToPath) { Write-Host "tee command is available" -ForegroundColor Green Write-Host "" Write-Host "The test_stdio_context_manager_exiting test should run." -ForegroundColor Green + if ($addedToPath) { + Write-Host "" + Write-Host "Note: Git for Windows tee was added to PATH for this session." -ForegroundColor Yellow + Write-Host "To make this permanent, add this to your PowerShell profile:" -ForegroundColor Yellow + Write-Host " `$env:PATH = `"C:\Program Files\Git\usr\bin;`$env:PATH`"" -ForegroundColor Cyan + } } else { Write-Host "tee command is NOT available" -ForegroundColor Red Write-Host "" @@ -98,4 +136,7 @@ if ($whereResult -or $getCommandResult -or $found -or ($pythonCheck -and $python Write-Host " 2. Install WSL (Windows Subsystem for Linux)" Write-Host " 3. Install MSYS2 or Cygwin" Write-Host " 4. Use PowerShell's Tee-Object cmdlet (different syntax)" -} \ No newline at end of file +} + +# Restore original PATH +$env:PATH = $originalPath \ No newline at end of file diff --git a/scripts/windows-debug/setup-environment.ps1 b/scripts/windows-debug/setup-environment.ps1 new file mode 100644 index 0000000000..25dd44e27c --- /dev/null +++ b/scripts/windows-debug/setup-environment.ps1 @@ -0,0 +1,59 @@ +#!/usr/bin/env pwsh +# Script to set up the environment for running stdio tests on Windows +# This adds Git for Windows tools to PATH if available +# Usage: . .\setup-environment.ps1 (note the dot-sourcing) + +Write-Host "Setting up environment for stdio tests..." -ForegroundColor Cyan +Write-Host "" + +# Check if Git for Windows is installed +$gitPaths = @( + "C:\Program Files\Git\usr\bin", + "C:\Program Files (x86)\Git\usr\bin" +) + +$gitFound = $false +$gitPath = "" + +foreach ($path in $gitPaths) { + if (Test-Path $path) { + $gitPath = $path + $gitFound = $true + break + } +} + +if ($gitFound) { + Write-Host "Found Git for Windows at: $gitPath" -ForegroundColor Green + + # Add to PATH + $env:PATH = "$gitPath;$env:PATH" + + # Verify tee is available + $teeCheck = python -c "import shutil; print(shutil.which('tee'))" + if ($teeCheck -and $teeCheck -ne "None") { + Write-Host "Successfully added tee to PATH: $teeCheck" -ForegroundColor Green + Write-Host "" + Write-Host "Environment is ready for stdio tests!" -ForegroundColor Green + Write-Host "" + Write-Host "You can now run the test scripts or individual tests:" -ForegroundColor Cyan + Write-Host " .\test-stdio-flakiness-200-runs.ps1" + Write-Host " .\test-stdio-flakiness-until-failure.ps1" + Write-Host " .\test-stdio-verbose-debug.ps1" + Write-Host "" + Write-Host "Or run individual tests with:" -ForegroundColor Cyan + Write-Host " uv run pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -v -o addopts=""""" + } else { + Write-Host "WARNING: Git path was added but tee is still not available" -ForegroundColor Yellow + } +} else { + Write-Host "Git for Windows not found in standard locations." -ForegroundColor Red + Write-Host "" + Write-Host "Please install Git for Windows from: https://gitforwindows.org/" -ForegroundColor Yellow + Write-Host "Or manually add the Git usr\bin directory to your PATH." -ForegroundColor Yellow +} + +Write-Host "" +Write-Host "Note: This only affects the current PowerShell session." -ForegroundColor Gray +Write-Host "To make changes permanent, add to your PowerShell profile:" -ForegroundColor Gray +Write-Host ' $env:PATH = "C:\Program Files\Git\usr\bin;$env:PATH"' -ForegroundColor Cyan \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 b/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 index 54df0c9a12..26bdddd6be 100644 --- a/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 +++ b/scripts/windows-debug/test-stdio-flakiness-200-runs.ps1 @@ -1,13 +1,21 @@ #!/usr/bin/env pwsh # Script to run test_stdio_context_manager_exiting 200 times to detect flakiness # Usage: .\test-stdio-flakiness-200-runs.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available Write-Host "Running test_stdio_context_manager_exiting 200 times to detect flakiness..." -ForegroundColor Cyan Write-Host "Test: tests/client/test_stdio.py::test_stdio_context_manager_exiting" -ForegroundColor Yellow Write-Host "" -# Disable pytest plugin autoload to avoid xdist issues -$env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 +} $startTime = Get-Date $count = 0 @@ -17,10 +25,7 @@ $failedRuns = @() for ($i = 1; $i -le 200; $i++) { Write-Host "Run $i of 200..." -NoNewline - $output = & { - $env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" - uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs --no-cov -p no:xdist 2>&1 - } + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -o addopts="" 2>&1 $exitCode = $LASTEXITCODE if ($exitCode -ne 0) { @@ -47,7 +52,4 @@ if ($failures -gt 0) { Write-Host "Failed on runs: $($failedRuns -join ', ')" -ForegroundColor Red } Write-Host "Duration: $($duration.ToString())" -Write-Host "Failure rate: $([math]::Round(($failures / 200) * 100, 2))%" - -# Clean up environment variable -Remove-Item Env:PYTEST_DISABLE_PLUGIN_AUTOLOAD -ErrorAction SilentlyContinue \ No newline at end of file +Write-Host "Failure rate: $([math]::Round(($failures / 200) * 100, 2))%" \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 b/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 index 4c0e047414..1a9df0e403 100644 --- a/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 +++ b/scripts/windows-debug/test-stdio-flakiness-until-failure.ps1 @@ -1,14 +1,22 @@ #!/usr/bin/env pwsh # Script to run test_stdio_context_manager_exiting until it fails # Usage: .\test-stdio-flakiness-until-failure.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available Write-Host "Running test_stdio_context_manager_exiting until failure..." -ForegroundColor Cyan Write-Host "Test: tests/client/test_stdio.py::test_stdio_context_manager_exiting" -ForegroundColor Yellow Write-Host "Press Ctrl+C to stop" -ForegroundColor Gray Write-Host "" -# Disable pytest plugin autoload to avoid xdist issues -$env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 +} $startTime = Get-Date $i = 0 @@ -17,10 +25,7 @@ while ($true) { $i++ Write-Host "Run $i..." -NoNewline - $output = & { - $env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" - uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs --no-cov -p no:xdist 2>&1 - } + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -o addopts="" 2>&1 $exitCode = $LASTEXITCODE if ($exitCode -ne 0) { @@ -45,7 +50,4 @@ while ($true) { } Write-Host "" -Write-Host "Exiting after failure detection." -ForegroundColor Cyan - -# Clean up environment variable -Remove-Item Env:PYTEST_DISABLE_PLUGIN_AUTOLOAD -ErrorAction SilentlyContinue \ No newline at end of file +Write-Host "Exiting after failure detection." -ForegroundColor Cyan \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-simple.ps1 b/scripts/windows-debug/test-stdio-simple.ps1 index d249703c42..3cf877ca05 100644 --- a/scripts/windows-debug/test-stdio-simple.ps1 +++ b/scripts/windows-debug/test-stdio-simple.ps1 @@ -1,54 +1,40 @@ #!/usr/bin/env pwsh # Simple script to run the test without xdist # Usage: .\test-stdio-simple.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available Write-Host "Running test_stdio_context_manager_exiting with xdist disabled..." -ForegroundColor Cyan Write-Host "" -# Method 1: Using environment variable (recommended in CLAUDE.md) -Write-Host "Method 1: Using PYTEST_DISABLE_PLUGIN_AUTOLOAD environment variable" -ForegroundColor Yellow -$env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" -uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -$exitCode1 = $LASTEXITCODE -Remove-Item Env:PYTEST_DISABLE_PLUGIN_AUTOLOAD -ErrorAction SilentlyContinue - -if ($exitCode1 -eq 0) { - Write-Host "Method 1: PASSED" -ForegroundColor Green -} else { - Write-Host "Method 1: FAILED with exit code $exitCode1" -ForegroundColor Red +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 } +Write-Host "tee found at: $teeCheck" -ForegroundColor Green Write-Host "" -# Method 2: Using -p no:xdist to disable the plugin -Write-Host "Method 2: Using -p no:xdist flag" -ForegroundColor Yellow -uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -p no:xdist -$exitCode2 = $LASTEXITCODE - -if ($exitCode2 -eq 0) { - Write-Host "Method 2: PASSED" -ForegroundColor Green -} else { - Write-Host "Method 2: FAILED with exit code $exitCode2" -ForegroundColor Red -} - -Write-Host "" - -# Method 3: Override addopts from pyproject.toml -Write-Host "Method 3: Overriding pytest addopts" -ForegroundColor Yellow +# Run the test with the working method +Write-Host "Running test with -o addopts='' to override xdist..." -ForegroundColor Yellow uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -xvs -o addopts="" -$exitCode3 = $LASTEXITCODE - -if ($exitCode3 -eq 0) { - Write-Host "Method 3: PASSED" -ForegroundColor Green -} else { - Write-Host "Method 3: FAILED with exit code $exitCode3" -ForegroundColor Red -} +$exitCode = $LASTEXITCODE Write-Host "" -Write-Host "========== SUMMARY ==========" -ForegroundColor Cyan -if ($exitCode1 -eq 0 -or $exitCode2 -eq 0 -or $exitCode3 -eq 0) { - Write-Host "At least one method succeeded!" -ForegroundColor Green - Write-Host "Use the successful method in your testing." +if ($exitCode -eq 0) { + Write-Host "Test PASSED!" -ForegroundColor Green + Write-Host "" + Write-Host "You can now use the other scripts to test for flakiness:" -ForegroundColor Cyan + Write-Host " .\test-stdio-flakiness-200-runs.ps1" + Write-Host " .\test-stdio-flakiness-until-failure.ps1" + Write-Host " .\test-stdio-verbose-debug.ps1" } else { - Write-Host "All methods failed. The test may have a different issue." -ForegroundColor Red + Write-Host "Test FAILED with exit code $exitCode" -ForegroundColor Red + Write-Host "" + Write-Host "Try running with verbose debug:" -ForegroundColor Yellow + Write-Host " .\test-stdio-verbose-debug.ps1" } \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-verbose-debug.ps1 b/scripts/windows-debug/test-stdio-verbose-debug.ps1 index 9696af81eb..593c216cf6 100644 --- a/scripts/windows-debug/test-stdio-verbose-debug.ps1 +++ b/scripts/windows-debug/test-stdio-verbose-debug.ps1 @@ -1,30 +1,37 @@ #!/usr/bin/env pwsh # Script to run test_stdio_context_manager_exiting with maximum debugging output # Usage: .\test-stdio-verbose-debug.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available Write-Host "Running test_stdio_context_manager_exiting with verbose debug output..." -ForegroundColor Cyan Write-Host "" +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 +} + # Set environment variables for debugging $env:PYTHONFAULTHANDLER = "1" $env:PYTEST_CURRENT_TEST = "1" -$env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" Write-Host "Environment variables set:" -ForegroundColor Yellow Write-Host " PYTHONFAULTHANDLER = 1 (enables Python fault handler)" -Write-Host " PYTEST_DISABLE_PLUGIN_AUTOLOAD = '' (disables pytest plugin autoload)" Write-Host "" Write-Host "Running test with maximum verbosity..." -ForegroundColor Cyan Write-Host "" # Run the test with all debugging options -$env:PYTEST_DISABLE_PLUGIN_AUTOLOAD = "" uv run --frozen pytest ` tests/client/test_stdio.py::test_stdio_context_manager_exiting ` -xvs ` - --no-cov ` - -p no:xdist ` + -o addopts="" ` --log-cli-level=DEBUG ` --log-cli-format="%(asctime)s [%(levelname)s] %(name)s: %(message)s" ` --capture=no ` @@ -42,5 +49,4 @@ if ($exitCode -eq 0) { # Clean up environment variables Remove-Item Env:PYTHONFAULTHANDLER -ErrorAction SilentlyContinue -Remove-Item Env:PYTEST_CURRENT_TEST -ErrorAction SilentlyContinue -Remove-Item Env:PYTEST_DISABLE_PLUGIN_AUTOLOAD -ErrorAction SilentlyContinue \ No newline at end of file +Remove-Item Env:PYTEST_CURRENT_TEST -ErrorAction SilentlyContinue \ No newline at end of file From 4eb6657f96ade1ad59fc8111314f158316ed22b1 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Wed, 16 Jul 2025 16:32:45 +0100 Subject: [PATCH 5/8] Add parallel and stress test scripts for Windows stdio flakiness Based on deep analysis, the flakiness likely occurs due to: 1. Job Object handle race conditions when tests run in parallel 2. Windows handle inheritance between test processes 3. Timing sensitivity in the minimal test that just opens/closes immediately 4. Python 3.11/3.12 specific subprocess handling changes These scripts help reproduce the CI environment conditions: - test-stdio-parallel-flakiness.ps1: Runs with xdist parallel workers - test-stdio-stress-race.ps1: Rapidly creates processes to expose races --- .../test-stdio-parallel-flakiness.ps1 | 94 +++++++++++++ .../windows-debug/test-stdio-stress-race.ps1 | 125 ++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 scripts/windows-debug/test-stdio-parallel-flakiness.ps1 create mode 100644 scripts/windows-debug/test-stdio-stress-race.ps1 diff --git a/scripts/windows-debug/test-stdio-parallel-flakiness.ps1 b/scripts/windows-debug/test-stdio-parallel-flakiness.ps1 new file mode 100644 index 0000000000..af32801754 --- /dev/null +++ b/scripts/windows-debug/test-stdio-parallel-flakiness.ps1 @@ -0,0 +1,94 @@ +#!/usr/bin/env pwsh +# Script to test for flakiness when running tests in parallel (like CI does) +# This simulates the xdist environment where the issue occurs +# Usage: .\test-stdio-parallel-flakiness.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available + +Write-Host "Testing stdio with parallel execution to simulate CI environment..." -ForegroundColor Cyan +Write-Host "" + +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 +} + +Write-Host "Running tests with different parallel configurations..." -ForegroundColor Yellow +Write-Host "" + +# Test 1: Run with 4 workers (default CI behavior) +Write-Host "Test 1: Running with 4 parallel workers (CI default)..." -ForegroundColor Cyan +$failures1 = 0 +for ($i = 1; $i -le 20; $i++) { + Write-Host " Run $i..." -NoNewline + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -v -n 4 2>&1 + if ($LASTEXITCODE -ne 0) { + $failures1++ + Write-Host " FAILED" -ForegroundColor Red + } else { + Write-Host " PASSED" -ForegroundColor Green + } +} +Write-Host " Result: $failures1 failures out of 20 runs" -ForegroundColor $(if ($failures1 -eq 0) { "Green" } else { "Red" }) +Write-Host "" + +# Test 2: Run with 2 workers +Write-Host "Test 2: Running with 2 parallel workers..." -ForegroundColor Cyan +$failures2 = 0 +for ($i = 1; $i -le 20; $i++) { + Write-Host " Run $i..." -NoNewline + $output = uv run --frozen pytest tests/client/test_stdio.py::test_stdio_context_manager_exiting -v -n 2 2>&1 + if ($LASTEXITCODE -ne 0) { + $failures2++ + Write-Host " FAILED" -ForegroundColor Red + } else { + Write-Host " PASSED" -ForegroundColor Green + } +} +Write-Host " Result: $failures2 failures out of 20 runs" -ForegroundColor $(if ($failures2 -eq 0) { "Green" } else { "Red" }) +Write-Host "" + +# Test 3: Run all stdio tests in parallel (simulates real CI) +Write-Host "Test 3: Running ALL stdio tests with 4 workers (full CI simulation)..." -ForegroundColor Cyan +$failures3 = 0 +for ($i = 1; $i -le 10; $i++) { + Write-Host " Run $i..." -NoNewline + $output = uv run --frozen pytest tests/client/test_stdio.py -v -n 4 2>&1 + if ($LASTEXITCODE -ne 0) { + $failures3++ + Write-Host " FAILED" -ForegroundColor Red + # Show which test failed + $failedTest = $output | Select-String "FAILED tests/client/test_stdio.py::" | Select-Object -First 1 + if ($failedTest) { + Write-Host " Failed: $failedTest" -ForegroundColor Red + } + } else { + Write-Host " PASSED" -ForegroundColor Green + } +} +Write-Host " Result: $failures3 failures out of 10 runs" -ForegroundColor $(if ($failures3 -eq 0) { "Green" } else { "Red" }) +Write-Host "" + +# Summary +Write-Host "========== SUMMARY ==========" -ForegroundColor Cyan +Write-Host "4 workers (single test): $failures1/20 failures" +Write-Host "2 workers (single test): $failures2/20 failures" +Write-Host "4 workers (all tests): $failures3/10 failures" +Write-Host "" + +if ($failures1 -gt 0 -or $failures2 -gt 0 -or $failures3 -gt 0) { + Write-Host "FLAKINESS DETECTED with parallel execution!" -ForegroundColor Red + Write-Host "" + Write-Host "This confirms the issue is related to parallel test execution." -ForegroundColor Yellow + Write-Host "The race condition likely involves:" -ForegroundColor Yellow + Write-Host " - Windows Job Object handle management" -ForegroundColor Gray + Write-Host " - Process cleanup timing with multiple workers" -ForegroundColor Gray + Write-Host " - Handle inheritance between test processes" -ForegroundColor Gray +} else { + Write-Host "No flakiness detected in this run." -ForegroundColor Green + Write-Host "The issue might require specific timing conditions to reproduce." -ForegroundColor Yellow +} \ No newline at end of file diff --git a/scripts/windows-debug/test-stdio-stress-race.ps1 b/scripts/windows-debug/test-stdio-stress-race.ps1 new file mode 100644 index 0000000000..633fa7b202 --- /dev/null +++ b/scripts/windows-debug/test-stdio-stress-race.ps1 @@ -0,0 +1,125 @@ +#!/usr/bin/env pwsh +# Script to stress test the specific race condition in stdio cleanup +# This creates many processes rapidly to expose handle/job object races +# Usage: .\test-stdio-stress-race.ps1 +# +# Prerequisites: Run . .\setup-environment.ps1 first to ensure tee is available + +Write-Host "Stress testing stdio cleanup race conditions..." -ForegroundColor Cyan +Write-Host "This test creates many processes rapidly to expose timing issues." -ForegroundColor Yellow +Write-Host "" + +# Check if tee is available +$teeCheck = python -c "import shutil; print(shutil.which('tee'))" +if (-not $teeCheck -or $teeCheck -eq "None") { + Write-Host "ERROR: tee command not found!" -ForegroundColor Red + Write-Host "Please run: . .\setup-environment.ps1" -ForegroundColor Yellow + Write-Host "(Note the dot at the beginning to source the script)" -ForegroundColor Yellow + exit 1 +} + +# Create a Python script that runs the test many times in quick succession +$stressScript = @' +import asyncio +import sys +import time +from pathlib import Path + +# Add parent directories to path +sys.path.insert(0, str(Path(__file__).parent.parent.parent)) + +from mcp.client.stdio import stdio_client, StdioServerParameters + +async def rapid_test(test_id: int): + """Run a single test iteration""" + try: + async with stdio_client(StdioServerParameters(command="tee")) as (_, _): + pass + return True, None + except Exception as e: + return False, str(e) + +async def stress_test(iterations: int, concurrent: int): + """Run many tests concurrently""" + print(f"Running {iterations} tests with {concurrent} concurrent...") + + failures = 0 + errors = [] + start_time = time.time() + + # Run in batches + for batch in range(0, iterations, concurrent): + batch_size = min(concurrent, iterations - batch) + tasks = [rapid_test(batch + i) for i in range(batch_size)] + results = await asyncio.gather(*tasks) + + for success, error in results: + if not success: + failures += 1 + if error and error not in errors: + errors.append(error) + + # Progress indicator + if (batch + batch_size) % 100 == 0: + print(f" Completed {batch + batch_size}/{iterations} tests...") + + duration = time.time() - start_time + return failures, errors, duration + +async def main(): + # Test different concurrency levels + configs = [ + (100, 1), # Sequential + (100, 2), # Low concurrency + (100, 5), # Medium concurrency + (100, 10), # High concurrency + ] + + for iterations, concurrent in configs: + print(f"\nTest: {iterations} iterations, {concurrent} concurrent") + failures, errors, duration = await stress_test(iterations, concurrent) + + print(f" Duration: {duration:.2f}s") + print(f" Failures: {failures}/{iterations}") + if errors: + print(f" Unique errors: {len(errors)}") + for error in errors[:3]: # Show first 3 errors + print(f" - {error}") + + if failures > 0: + print(" RACE CONDITION DETECTED!" if concurrent > 1 else " FAILURE DETECTED!") + +if __name__ == "__main__": + asyncio.run(main()) +'@ + +# Save the stress test script +$scriptPath = Join-Path $PSScriptRoot "stress_test.py" +$stressScript | Out-File -FilePath $scriptPath -Encoding UTF8 + +Write-Host "Running stress tests..." -ForegroundColor Cyan +Write-Host "" + +# Run the stress test +uv run python $scriptPath + +$exitCode = $LASTEXITCODE + +# Clean up +Remove-Item $scriptPath -ErrorAction SilentlyContinue + +Write-Host "" +Write-Host "========== ANALYSIS ==========" -ForegroundColor Cyan + +if ($exitCode -ne 0) { + Write-Host "Stress test failed to complete." -ForegroundColor Red +} else { + Write-Host "Stress test completed." -ForegroundColor Green + Write-Host "" + Write-Host "If failures increased with concurrency, it indicates:" -ForegroundColor Yellow + Write-Host " - Race condition in process cleanup" -ForegroundColor Gray + Write-Host " - Job Object handle conflicts" -ForegroundColor Gray + Write-Host " - Windows handle inheritance issues" -ForegroundColor Gray + Write-Host "" + Write-Host "This matches the CI flakiness pattern where parallel tests fail." -ForegroundColor Yellow +} \ No newline at end of file From 79e5ce71345b7128aa521aa7eae30cc4fa4ba197 Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Wed, 16 Jul 2025 16:37:29 +0100 Subject: [PATCH 6/8] remove readme --- scripts/windows-debug/README.md | 105 -------------------------------- 1 file changed, 105 deletions(-) delete mode 100644 scripts/windows-debug/README.md diff --git a/scripts/windows-debug/README.md b/scripts/windows-debug/README.md deleted file mode 100644 index 9f33eed175..0000000000 --- a/scripts/windows-debug/README.md +++ /dev/null @@ -1,105 +0,0 @@ -# Windows Test Debugging Scripts - -This folder contains PowerShell scripts to help debug the flaky `test_stdio_context_manager_exiting` test on Windows with Python 3.11/3.12. - -## Prerequisites - -- Windows PowerShell or PowerShell Core -- Python 3.11 or 3.12 -- `uv` installed and configured -- Virtual environment activated -- Git for Windows installed (provides the `tee` command) - -## Quick Start - -1. First, set up your environment to make `tee` available: - ```powershell - . .\setup-environment.ps1 - ``` - **Note:** The dot (.) at the beginning is important - it sources the script. - -2. Verify the test runs: - ```powershell - .\test-stdio-simple.ps1 - ``` - -3. Run the flakiness tests: - ```powershell - .\test-stdio-flakiness-200-runs.ps1 - ``` - -## Scripts - -### 1. `setup-environment.ps1` -Sets up the environment by adding Git for Windows tools to PATH. **Must be dot-sourced.** - -```powershell -. .\setup-environment.ps1 -``` - -### 2. `check-tee-command.ps1` -Checks if the `tee` command is available on your Windows system. The test requires `tee` to run. - -```powershell -.\check-tee-command.ps1 -``` - -### 3. `test-stdio-simple.ps1` -Runs the test once to verify it works. - -```powershell -.\test-stdio-simple.ps1 -``` - -### 4. `test-stdio-flakiness-200-runs.ps1` -Runs the test 200 times and reports the failure rate. - -```powershell -.\test-stdio-flakiness-200-runs.ps1 -``` - -### 5. `test-stdio-flakiness-until-failure.ps1` -Runs the test in a loop until it fails, useful for catching intermittent failures. - -```powershell -.\test-stdio-flakiness-until-failure.ps1 -``` - -### 6. `test-stdio-verbose-debug.ps1` -Runs the test once with maximum debugging output enabled. - -```powershell -.\test-stdio-verbose-debug.ps1 -``` - -## Usage Notes - -1. Make sure your virtual environment is activated before running these scripts -2. Always run `setup-environment.ps1` first in each new PowerShell session -3. Run scripts from the `scripts\windows-debug` directory -4. If scripts fail with execution policy errors, run: - ```powershell - Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - ``` - -## Interpreting Results - -- If `tee` is not available, the test will be skipped -- Flaky failures typically show up as timeout errors or process cleanup issues -- Look for patterns in failure output, especially around process termination timing - -## Troubleshooting - -### "tee command not found" -1. Install Git for Windows from https://gitforwindows.org/ -2. Run `. .\setup-environment.ps1` to add Git tools to PATH -3. Verify with `python -c "import shutil; print(shutil.which('tee'))"` - -### "no tests ran" with xdist -The scripts now use `-o addopts=""` to override the pytest configuration that enables xdist by default. - -### Making PATH changes permanent -Add this line to your PowerShell profile: -```powershell -$env:PATH = "C:\Program Files\Git\usr\bin;$env:PATH" -``` \ No newline at end of file From 4cab85c3ca09696a893fe949f2400aac7f153f1b Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Wed, 16 Jul 2025 16:39:38 +0100 Subject: [PATCH 7/8] Add environment diagnostic script for Windows CI differences This script collects detailed information about: - Windows version and build - Python version and subprocess configuration - Job Objects support - Security software - Process creation timing - Handle inheritance settings This will help identify differences between local dev environments (where the test passes) and CI environments (where it flakes). --- .../windows-debug/diagnose-environment.ps1 | 152 ++++++++++++++++++ scripts/windows-debug/test.txt | 0 2 files changed, 152 insertions(+) create mode 100644 scripts/windows-debug/diagnose-environment.ps1 create mode 100644 scripts/windows-debug/test.txt diff --git a/scripts/windows-debug/diagnose-environment.ps1 b/scripts/windows-debug/diagnose-environment.ps1 new file mode 100644 index 0000000000..5d49480b23 --- /dev/null +++ b/scripts/windows-debug/diagnose-environment.ps1 @@ -0,0 +1,152 @@ +#!/usr/bin/env pwsh +# Script to diagnose environment differences that might cause CI flakiness +# Usage: .\diagnose-environment.ps1 + +Write-Host "Diagnosing Windows environment for stdio test issues..." -ForegroundColor Cyan +Write-Host "" + +# System Information +Write-Host "=== SYSTEM INFORMATION ===" -ForegroundColor Yellow +Write-Host "Windows Version:" +(Get-CimInstance Win32_OperatingSystem).Version +Write-Host "Windows Build:" +(Get-ItemProperty "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion").DisplayVersion +Write-Host "" + +# Python Information +Write-Host "=== PYTHON INFORMATION ===" -ForegroundColor Yellow +Write-Host "Python Version:" +python --version +Write-Host "" +Write-Host "Python Build Info:" +python -c "import sys; print(f'Version: {sys.version}')" +python -c "import sys; print(f'Platform: {sys.platform}')" +python -c "import sys; print(f'Windows Version: {sys.getwindowsversion()}')" +Write-Host "" + +# Check subprocess configuration +Write-Host "=== SUBPROCESS CONFIGURATION ===" -ForegroundColor Yellow +python -c @" +import subprocess +import sys +print(f'CREATE_NO_WINDOW available: {hasattr(subprocess, "CREATE_NO_WINDOW")}') +print(f'CREATE_NEW_PROCESS_GROUP available: {hasattr(subprocess, "CREATE_NEW_PROCESS_GROUP")}') +print(f'Windows subprocess startup info: {hasattr(subprocess, "STARTUPINFO")}') + +# Check handle inheritance defaults +import os +print(f'\nHandle inheritance (os.O_NOINHERIT): {hasattr(os, "O_NOINHERIT")}') + +# Check asyncio event loop +import asyncio +try: + loop = asyncio.get_event_loop_policy() + print(f'Event loop policy: {type(loop).__name__}') +except: + print('Event loop policy: Unable to determine') +"@ +Write-Host "" + +# Check for Job Objects support +Write-Host "=== JOB OBJECTS SUPPORT ===" -ForegroundColor Yellow +python -c @" +try: + import win32job + import win32api + print('pywin32 available: Yes') + print(f'win32job version: {win32job.__file__}') + + # Try to create a job object + try: + job = win32job.CreateJobObject(None, '') + win32api.CloseHandle(job) + print('Job Object creation: Success') + except Exception as e: + print(f'Job Object creation: Failed - {e}') +except ImportError: + print('pywin32 available: No (Job Objects not available)') +"@ +Write-Host "" + +# Check process limits +Write-Host "=== PROCESS LIMITS ===" -ForegroundColor Yellow +python -c @" +import os +import psutil +proc = psutil.Process(os.getpid()) +print(f'Open handles: {proc.num_handles()}') +print(f'Open files: {len(proc.open_files())}') + +# Check system-wide limits +print(f'Total processes: {len(psutil.pids())}') +"@ +Write-Host "" + +# Check security software +Write-Host "=== SECURITY SOFTWARE ===" -ForegroundColor Yellow +Get-CimInstance -Namespace "root\SecurityCenter2" -ClassName AntiVirusProduct -ErrorAction SilentlyContinue | + Select-Object displayName, productState | Format-Table +Write-Host "" + +# Test rapid process creation +Write-Host "=== RAPID PROCESS CREATION TEST ===" -ForegroundColor Yellow +Write-Host "Testing rapid tee process creation/destruction..." +$testScript = @' +import time +import subprocess +import sys + +failures = 0 +times = [] + +for i in range(20): + start = time.time() + try: + # Create process with same flags as stdio client + proc = subprocess.Popen( + ['tee'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0) + ) + proc.stdin.close() + proc.wait(timeout=0.5) + elapsed = time.time() - start + times.append(elapsed) + except Exception as e: + failures += 1 + print(f" Iteration {i+1}: FAILED - {e}") + + if (i+1) % 5 == 0: + avg_time = sum(times) / len(times) if times else 0 + print(f" Completed {i+1}/20 (avg: {avg_time*1000:.1f}ms)") + +print(f"\nFailures: {failures}/20") +if times: + print(f"Average time: {sum(times)/len(times)*1000:.1f}ms") + print(f"Max time: {max(times)*1000:.1f}ms") +'@ + +python -c $testScript +Write-Host "" + +# Environment variables that might affect subprocess +Write-Host "=== RELEVANT ENVIRONMENT VARIABLES ===" -ForegroundColor Yellow +@("COMSPEC", "PATH", "PYTHONPATH", "PYTHONASYNCIODEBUG") | ForEach-Object { + $value = [Environment]::GetEnvironmentVariable($_) + if ($value) { + Write-Host "$_`:" + Write-Host " $value" -ForegroundColor Gray + } +} +Write-Host "" + +Write-Host "=== DIAGNOSIS COMPLETE ===" -ForegroundColor Cyan +Write-Host "" +Write-Host "Share this output when reporting the flakiness issue." -ForegroundColor Yellow +Write-Host "Key differences to look for between local and CI:" -ForegroundColor Yellow +Write-Host " - Windows version/build" -ForegroundColor Gray +Write-Host " - Python build details" -ForegroundColor Gray +Write-Host " - Security software presence" -ForegroundColor Gray +Write-Host " - Process creation timing" -ForegroundColor Gray \ No newline at end of file diff --git a/scripts/windows-debug/test.txt b/scripts/windows-debug/test.txt new file mode 100644 index 0000000000..e69de29bb2 From 6be585d008e232f62cdf7c19a881cf605a385d6a Mon Sep 17 00:00:00 2001 From: Felix Weinberger Date: Wed, 16 Jul 2025 17:14:40 +0100 Subject: [PATCH 8/8] trigger ci --- test.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 test.txt diff --git a/test.txt b/test.txt new file mode 100644 index 0000000000..5ca2944976 --- /dev/null +++ b/test.txt @@ -0,0 +1,6 @@ +hello +test +test +test +test +test