Skip to content

Commit b5a1ce0

Browse files
jeremyederclaude
jeremyeder
and
claude
authored
feat: convert AgentReady assessment to on-demand workflow (#213)
Implements comment-based on-demand assessment with enhanced UX. Changes: - Trigger on `/agentready assess [URL]` comments - Authorization: Only @jeremyeder can trigger - Dual mode: Assess current repo or external public repos - Enhanced comment format: - Summary with score, certification level, emoji - Top 5 failing attributes (sorted by impact/tier) - Links to artifacts and workflow - Collapsed details section with full report - Security: URL validation, public repo enforcement, command injection protection - Friendly unauthorized user response with CLI instructions - Keeps workflow_dispatch for manual admin access Shellcheck fixes: - Use bash parameter expansion instead of sed - Proper variable quoting in all shell scripts - Brace grouping for GITHUB_OUTPUT redirects Fixes #191 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
1 parent 2d3bfe5 commit b5a1ce0

File tree

1 file changed

+268
-25
lines changed

1 file changed

+268
-25
lines changed

‎.github/workflows/agentready-assessment.yml‎

Lines changed: 268 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,306 @@
1-
name: AgentReady Assessment
1+
name: AgentReady Assessment (On-Demand)
22

33
on:
4-
pull_request:
5-
types: [opened, synchronize, reopened]
6-
push:
7-
branches: [main, master]
4+
issue_comment:
5+
types: [created]
6+
pull_request_review_comment:
7+
types: [created]
88
workflow_dispatch:
99

1010
jobs:
11+
unauthorized:
12+
# Respond to unauthorized users with helpful message
13+
if: |
14+
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '/agentready assess') && github.event.comment.user.login != 'jeremyeder') ||
15+
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/agentready assess') && github.event.comment.user.login != 'jeremyeder')
16+
17+
runs-on: ubuntu-latest
18+
permissions:
19+
issues: write
20+
pull-requests: write
21+
22+
steps:
23+
- name: Post unauthorized message
24+
uses: actions/github-script@v7
25+
with:
26+
script: |
27+
const user = context.payload.comment.user.login;
28+
const body = `👋 Hi @${user}! Thanks for your interest in AgentReady.\n\n` +
29+
`The \`/agentready assess\` command is currently restricted to repository maintainers.\n\n` +
30+
`**To assess your own repository:**\n` +
31+
`\`\`\`bash\n` +
32+
`pip install agentready\n` +
33+
`agentready assess .\n` +
34+
`\`\`\`\n\n` +
35+
`See [AgentReady documentation](https://github.com/ambient-code/agentready) for more information.`;
36+
37+
await github.rest.issues.createComment({
38+
owner: context.repo.owner,
39+
repo: context.repo.repo,
40+
issue_number: context.issue.number,
41+
body: body
42+
});
43+
1144
assess:
45+
# Only run on /agentready assess command (from @jeremyeder only) or manual trigger
46+
if: |
47+
(github.event_name == 'issue_comment' && contains(github.event.comment.body, '/agentready assess') && github.event.comment.user.login == 'jeremyeder') ||
48+
(github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '/agentready assess') && github.event.comment.user.login == 'jeremyeder') ||
49+
github.event_name == 'workflow_dispatch'
50+
1251
runs-on: ubuntu-latest
1352
permissions:
1453
contents: read
1554
pull-requests: write
55+
issues: write
1656

1757
steps:
18-
- name: Checkout code
19-
uses: actions/checkout@v6
58+
- name: Parse command and extract repository URL
59+
id: parse
60+
env:
61+
COMMENT_BODY: ${{ github.event.comment.body || '' }}
62+
run: |
63+
# SAFE: Using environment variable to avoid command injection
64+
# Extract repository URL (case-insensitive)
65+
REPO_URL=$(echo "$COMMENT_BODY" | grep -ioP '/agentready\s+assess\s+\Khttps://github\.com/[^\s]+' || echo "")
66+
67+
# Default to current repo if no URL
68+
if [ -z "$REPO_URL" ]; then
69+
REPO_URL="https://github.com/${{ github.repository }}"
70+
ASSESS_CURRENT_REPO="true"
71+
echo "::notice::No repository URL provided, assessing current repository"
72+
else
73+
ASSESS_CURRENT_REPO="false"
74+
echo "::notice::Assessing external repository: $REPO_URL"
75+
fi
76+
77+
# Clean URL and extract org/repo (using bash parameter expansion)
78+
REPO_URL="${REPO_URL%.git}"
79+
ORG_REPO="${REPO_URL#https://github.com/}"
80+
81+
# Output all variables in one block
82+
{
83+
echo "repo_url=$REPO_URL"
84+
echo "org_repo=$ORG_REPO"
85+
echo "assess_current=$ASSESS_CURRENT_REPO"
86+
} >> "$GITHUB_OUTPUT"
87+
88+
- name: Validate repository URL and access
89+
id: validate
90+
env:
91+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
92+
REPO_URL: ${{ steps.parse.outputs.repo_url }}
93+
ORG_REPO: ${{ steps.parse.outputs.org_repo }}
94+
ASSESS_CURRENT: ${{ steps.parse.outputs.assess_current }}
95+
run: |
96+
# Skip validation for current repo
97+
if [ "$ASSESS_CURRENT" == "true" ]; then
98+
echo "✅ Assessing current repository (validation skipped)"
99+
echo "is_valid=true" >> "$GITHUB_OUTPUT"
100+
exit 0
101+
fi
102+
103+
# Validate URL format
104+
if ! echo "$REPO_URL" | grep -qE '^https://github\.com/[a-zA-Z0-9_-]+/[a-zA-Z0-9._-]+$'; then
105+
echo "::error::Invalid repository URL format: $REPO_URL"
106+
echo "is_valid=false" >> "$GITHUB_OUTPUT"
107+
exit 1
108+
fi
109+
110+
# Verify repository is public
111+
if ! gh repo view "$ORG_REPO" --json isPrivate -q '.isPrivate' > /tmp/is_private.txt 2>&1; then
112+
echo "::error::Repository not found: $ORG_REPO"
113+
echo "is_valid=false" >> "$GITHUB_OUTPUT"
114+
exit 1
115+
fi
116+
117+
IS_PRIVATE=$(cat /tmp/is_private.txt)
118+
if [ "$IS_PRIVATE" == "true" ]; then
119+
echo "::error::Repository $ORG_REPO is private. Only public repos allowed."
120+
echo "is_valid=false" >> "$GITHUB_OUTPUT"
121+
exit 1
122+
fi
123+
124+
echo "✅ Repository $ORG_REPO is public and accessible"
125+
echo "is_valid=true" >> "$GITHUB_OUTPUT"
126+
127+
- name: Checkout current repository
128+
if: steps.parse.outputs.assess_current == 'true'
129+
uses: actions/checkout@v4
130+
with:
131+
ref: ${{ github.event.pull_request.head.ref || github.ref }}
132+
133+
- name: Clone external repository
134+
if: steps.parse.outputs.assess_current == 'false' && steps.validate.outputs.is_valid == 'true'
135+
env:
136+
REPO_URL: ${{ steps.parse.outputs.repo_url }}
137+
run: |
138+
echo "Cloning $REPO_URL..."
139+
git clone "$REPO_URL" /tmp/repo-to-assess
140+
141+
if [ ! -d "/tmp/repo-to-assess/.git" ]; then
142+
echo "::error::Failed to clone repository"
143+
exit 1
144+
fi
145+
146+
echo "✅ Repository cloned successfully"
20147
21148
- name: Set up Python
22-
uses: actions/setup-python@v6
149+
uses: actions/setup-python@v5
23150
with:
24151
python-version: '3.12'
25152

26153
- name: Install AgentReady
27-
run: |
28-
pip install -e .
154+
run: pip install -e .
29155

30156
- name: Run AgentReady Assessment
157+
id: assessment
158+
env:
159+
ASSESS_CURRENT: ${{ steps.parse.outputs.assess_current }}
31160
run: |
32-
agentready assess . --verbose
161+
# Set paths based on mode
162+
if [ "$ASSESS_CURRENT" == "true" ]; then
163+
REPO_PATH="."
164+
OUTPUT_DIR=".agentready"
165+
else
166+
REPO_PATH="/tmp/repo-to-assess"
167+
OUTPUT_DIR="/tmp/assessment-output"
168+
mkdir -p "$OUTPUT_DIR"
169+
fi
170+
171+
echo "Assessing: $REPO_PATH"
172+
agentready assess "$REPO_PATH" --verbose --output-dir "$OUTPUT_DIR"
173+
174+
# Verify report generated
175+
if [ ! -f "$OUTPUT_DIR/report-latest.md" ]; then
176+
echo "::error::Assessment failed - no report"
177+
exit 1
178+
fi
179+
180+
echo "output_dir=$OUTPUT_DIR" >> "$GITHUB_OUTPUT"
181+
182+
- name: Extract assessment summary
183+
if: steps.assessment.outcome == 'success'
184+
id: summary
185+
env:
186+
OUTPUT_DIR: ${{ steps.assessment.outputs.output_dir }}
187+
run: |
188+
# Extract key metrics from JSON
189+
SCORE=$(jq -r '.overall_score' "$OUTPUT_DIR/assessment-latest.json")
190+
CERT_LEVEL=$(jq -r '.certification_level' "$OUTPUT_DIR/assessment-latest.json")
191+
192+
# Extract top 5 failing attributes sorted by tier (tier 1 = highest impact)
193+
# Format: tier,name for sorting, then extract just names
194+
FAILING=$(jq -r '.findings[] | select(.status == "fail") | "\(.attribute.tier),\(.attribute.name)"' "$OUTPUT_DIR/assessment-latest.json" | \
195+
sort -n | \
196+
cut -d',' -f2 | \
197+
head -5 | \
198+
paste -sd "," -)
199+
200+
# Output all variables in one block
201+
{
202+
echo "score=$SCORE"
203+
echo "cert_level=$CERT_LEVEL"
204+
echo "failing=$FAILING"
205+
} >> "$GITHUB_OUTPUT"
33206
34207
- name: Upload Assessment Reports
35-
uses: actions/upload-artifact@v5
208+
uses: actions/upload-artifact@v4
36209
if: always()
37210
with:
38-
name: agentready-reports
39-
path: .agentready/
211+
name: agentready-reports-${{ github.run_id }}
212+
path: ${{ steps.assessment.outputs.output_dir }}
40213
retention-days: 30
41214

42-
- name: Comment on PR
43-
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository
44-
uses: actions/github-script@v8
215+
- name: Post assessment results
216+
if: always()
217+
uses: actions/github-script@v7
218+
env:
219+
OUTPUT_DIR: ${{ steps.assessment.outputs.output_dir }}
220+
REPO_URL: ${{ steps.parse.outputs.repo_url }}
221+
ORG_REPO: ${{ steps.parse.outputs.org_repo }}
222+
ASSESS_CURRENT: ${{ steps.parse.outputs.assess_current }}
223+
VALIDATION_FAILED: ${{ steps.validate.outputs.is_valid == 'false' }}
224+
ASSESSMENT_FAILED: ${{ steps.assessment.outcome == 'failure' }}
225+
SCORE: ${{ steps.summary.outputs.score }}
226+
CERT_LEVEL: ${{ steps.summary.outputs.cert_level }}
227+
FAILING: ${{ steps.summary.outputs.failing }}
45228
with:
46229
script: |
47230
const fs = require('fs');
48-
const reportPath = '.agentready/report-latest.md';
231+
let body;
232+
233+
const validationFailed = process.env.VALIDATION_FAILED === 'true';
234+
const assessmentFailed = process.env.ASSESSMENT_FAILED === 'true';
235+
const orgRepo = process.env.ORG_REPO;
236+
const outputDir = process.env.OUTPUT_DIR;
237+
const assessCurrent = process.env.ASSESS_CURRENT === 'true';
238+
const repoUrl = process.env.REPO_URL;
239+
const score = process.env.SCORE;
240+
const certLevel = process.env.CERT_LEVEL;
241+
const failing = process.env.FAILING;
49242
50-
if (!fs.existsSync(reportPath)) {
51-
console.log('No report found');
52-
return;
243+
// Validation failure
244+
if (validationFailed) {
245+
body = `## ❌ AgentReady Assessment Failed\n\n` +
246+
`Could not assess **${orgRepo}**\n\n` +
247+
`**Possible reasons:**\n` +
248+
`- Repository does not exist\n` +
249+
`- Repository is private (only public repos)\n` +
250+
`- Invalid URL format\n\n` +
251+
`**Expected:** \`/agentready assess https://github.com/owner/repo\``;
53252
}
253+
// Assessment failure
254+
else if (assessmentFailed) {
255+
body = `## ❌ AgentReady Assessment Failed\n\n` +
256+
`Assessment of **${orgRepo}** failed.\n\n` +
257+
`[View logs](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`;
258+
}
259+
// Success - Structured format with summary and collapsed details
260+
else {
261+
const reportPath = `${outputDir}/report-latest.md`;
262+
const report = fs.readFileSync(reportPath, 'utf8');
263+
const repoContext = assessCurrent ? '(Current Repository)' : `([${orgRepo}](${repoUrl}))`;
264+
265+
// Certificate emoji
266+
const certEmoji = {
267+
'Platinum': '🏆',
268+
'Gold': '🥇',
269+
'Silver': '🥈',
270+
'Bronze': '🥉',
271+
'Needs Improvement': '📋'
272+
}[certLevel] || '📊';
273+
274+
// Build summary section
275+
let summary = `## ${certEmoji} AgentReady Assessment ${repoContext}\n\n`;
276+
summary += `**Score:** ${score}/100 | **Level:** ${certLevel}\n\n`;
54277
55-
const report = fs.readFileSync(reportPath, 'utf8');
278+
// Next steps - top 5 by impact
279+
summary += `### 🎯 Next Steps (by impact)\n\n`;
280+
if (failing && failing.length > 0) {
281+
const failingList = failing.split(',').map((f, i) => `${i + 1}. Fix: ${f}`).join('\n');
282+
summary += `${failingList}\n\n`;
283+
} else {
284+
summary += `✅ All critical attributes passing! Consider improving optional attributes.\n\n`;
285+
}
56286
57-
// Post comment with assessment results
287+
// Links section
288+
const artifactUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
289+
summary += `📦 [Download Full Reports](${artifactUrl}) • `;
290+
summary += `🔗 [View Workflow](${artifactUrl})\n\n`;
291+
292+
// Collapsed verbose output
293+
summary += `<details>\n<summary>📄 Full Assessment Report (click to expand)</summary>\n\n`;
294+
summary += `${report}\n\n`;
295+
summary += `</details>`;
296+
297+
body = summary;
298+
}
299+
300+
// Post comment
58301
await github.rest.issues.createComment({
59-
issue_number: context.issue.number,
60302
owner: context.repo.owner,
61303
repo: context.repo.repo,
62-
body: report
304+
issue_number: context.issue.number,
305+
body: body
63306
});

0 commit comments

Comments
 (0)