Skip to content

Commit 91e2327

Browse files
committed
test
1 parent aab7fa4 commit 91e2327

File tree

3 files changed

+156
-0
lines changed

3 files changed

+156
-0
lines changed

.github/scripts/analyze_profile.py

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/usr/bin/env python3
2+
"""Analyze CPU profile using GitHub Models API for performance insights."""
3+
4+
import os
5+
import sys
6+
from pathlib import Path
7+
8+
9+
def analyze_profile_with_github_models(profile_path: str) -> str:
10+
"""Analyze a CPU profile using GitHub Models API."""
11+
try:
12+
import requests
13+
except ImportError:
14+
return "⚠️ requests package not available for LLM analysis"
15+
16+
github_token = os.getenv("GITHUB_TOKEN")
17+
if not github_token:
18+
return "⚠️ GITHUB_TOKEN not set"
19+
20+
# Read profile (limit to first 50KB to avoid token limits)
21+
profile_content = Path(profile_path).read_text(encoding="utf-8")
22+
if len(profile_content) > 50_000:
23+
profile_content = profile_content[:50_000] + "\n... (truncated)"
24+
25+
prompt = f"""Analyze this CPU profiling data from py-spy and provide actionable performance insights.
26+
27+
The profile is in "collapsed stack trace" format where each line shows:
28+
- A semicolon-separated call stack (deepest function last)
29+
- Followed by a space and sample count
30+
31+
Focus on:
32+
1. **Hotspots**: Which functions consume the most CPU time?
33+
2. **Patterns**: Are there inefficiencies like excessive I/O, loops, or imports?
34+
3. **Recommendations**: Specific, actionable suggestions to improve performance
35+
36+
Profile data:
37+
```
38+
{profile_content}
39+
```
40+
41+
Provide a concise analysis (2-3 paragraphs max) with the most important findings."""
42+
43+
# Use GitHub Models API (available in GitHub Actions)
44+
# Reference: https://docs.github.com/en/github-models
45+
endpoint = "https://models.inference.ai.azure.com/chat/completions"
46+
47+
headers = {
48+
"Content-Type": "application/json",
49+
"Authorization": f"Bearer {github_token}",
50+
}
51+
52+
payload = {
53+
"model": "gpt-4o", # GitHub Models supports gpt-4o, gpt-4o-mini, etc.
54+
"messages": [
55+
{"role": "user", "content": prompt}
56+
],
57+
"max_tokens": 1024,
58+
"temperature": 0.7,
59+
}
60+
61+
try:
62+
response = requests.post(endpoint, headers=headers, json=payload, timeout=30)
63+
response.raise_for_status()
64+
result = response.json()
65+
return result["choices"][0]["message"]["content"]
66+
except requests.exceptions.RequestException as e:
67+
return f"⚠️ Failed to analyze with GitHub Models: {e}"
68+
except (KeyError, IndexError) as e:
69+
return f"⚠️ Unexpected response format from GitHub Models: {e}"
70+
71+
72+
def main():
73+
if len(sys.argv) != 2:
74+
print("Usage: analyze_profile.py <profile.txt>")
75+
sys.exit(1)
76+
77+
profile_path = sys.argv[1]
78+
if not Path(profile_path).exists():
79+
print(f"Error: {profile_path} not found")
80+
sys.exit(1)
81+
82+
analysis = analyze_profile_with_github_models(profile_path)
83+
print(analysis)
84+
85+
86+
if __name__ == "__main__":
87+
main()

.github/workflows/integration_tests.yml

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,69 @@ jobs:
8888
path: testcases/${{ matrix.testcase }}/artifacts/
8989
if-no-files-found: ignore
9090

91+
- name: Analyze performance profile with LLM
92+
if: matrix.testcase == 'performance-testcase' && github.event_name == 'pull_request'
93+
env:
94+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
95+
run: |
96+
if [ -f testcases/${{ matrix.testcase }}/artifacts/cpu_profile.txt ]; then
97+
echo "## 🔍 Performance Profile Analysis - ${{ matrix.environment }}" > analysis.md
98+
echo "" >> analysis.md
99+
100+
# Run LLM analysis using GitHub Models (powered by GitHub Copilot)
101+
echo "### 🤖 AI Analysis (GitHub Copilot)" >> analysis.md
102+
echo "" >> analysis.md
103+
uv run --with requests python .github/scripts/analyze_profile.py testcases/${{ matrix.testcase }}/artifacts/cpu_profile.txt >> analysis.md || echo "_Analysis failed - GitHub Models may not be available in this repository_" >> analysis.md
104+
echo "" >> analysis.md
105+
106+
# Add artifact links
107+
echo "### 📦 Artifacts" >> analysis.md
108+
echo "" >> analysis.md
109+
echo "Download profiling artifacts from this run:" >> analysis.md
110+
echo "" >> analysis.md
111+
echo "**CPU Profiling (py-spy):**" >> analysis.md
112+
echo "- 🔥 **Flamegraph SVG** - Visual representation (open in browser)" >> analysis.md
113+
echo "- 📊 **Speedscope JSON** - Interactive timeline viewer ([speedscope.app](https://speedscope.app))" >> analysis.md
114+
echo "- 📝 **Raw Profile** - Text format for custom analysis" >> analysis.md
115+
echo "" >> analysis.md
116+
echo "**Memory Profiling (memray):**" >> analysis.md
117+
echo "- 💾 **Memory Flamegraph HTML** - Memory allocation visualization" >> analysis.md
118+
echo "- 🔬 **Memory Profile Binary** - Raw memray data for custom analysis" >> analysis.md
119+
echo "" >> analysis.md
120+
echo "[View all artifacts →](../../actions/runs/${{ github.run_id }})" >> analysis.md
121+
echo "" >> analysis.md
122+
123+
# Add sample stack traces for quick preview
124+
echo "<details>" >> analysis.md
125+
echo "<summary>Top 20 Stack Traces (click to expand)</summary>" >> analysis.md
126+
echo "" >> analysis.md
127+
echo '```' >> analysis.md
128+
head -20 testcases/${{ matrix.testcase }}/artifacts/cpu_profile.txt >> analysis.md
129+
echo '```' >> analysis.md
130+
echo "</details>" >> analysis.md
131+
132+
echo "analysis_file=analysis.md" >> $GITHUB_OUTPUT
133+
fi
134+
135+
- name: Comment PR with performance analysis
136+
if: matrix.testcase == 'performance-testcase' && github.event_name == 'pull_request'
137+
uses: actions/github-script@v7
138+
with:
139+
script: |
140+
const fs = require('fs');
141+
const analysisFile = 'analysis.md';
142+
143+
if (fs.existsSync(analysisFile)) {
144+
const analysis = fs.readFileSync(analysisFile, 'utf8');
145+
146+
github.rest.issues.createComment({
147+
issue_number: context.issue.number,
148+
owner: context.repo.owner,
149+
repo: context.repo.repo,
150+
body: analysis
151+
});
152+
}
153+
91154
summarize-results:
92155
needs: [integration-tests]
93156
runs-on: ubuntu-latest

testcases/performance-testcase/run.sh

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,9 @@ uv run py-spy record --subprocesses -f flamegraph -o artifacts/cpu_profile.svg -
2626

2727
echo "Run agent with py-spy profiling (speedscope JSON for interactive viewing)"
2828
uv run py-spy record --subprocesses -f speedscope -o artifacts/cpu_profile_speedscope.json -- uv run uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}'
29+
30+
echo "Run agent with memray memory profiling"
31+
uv run memray run -o artifacts/memory_profile.bin --force -m uipath run main '{"message": "abc", "repeat": 2, "prefix": "xyz"}'
32+
33+
echo "Generating memory flamegraph from memray profile"
34+
uv run memray flamegraph artifacts/memory_profile.bin -o artifacts/memory_flamegraph.html --force || echo "Failed to generate memory flamegraph"

0 commit comments

Comments
 (0)