Add comprehensive evaluation infrastructure for spec and plan templates

.github/workflows/eval.yml

@@ -0,0 +1,176 @@
+name: AI Evals
+
+on:
+  workflow_dispatch:  # Manual trigger only
+    inputs:
+      model:
+        description: 'Model to use for evaluation'
+        required: false
+        default: 'claude-sonnet-4-5-20250929'
+        type: string
+
+jobs:
+  eval:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write  # For posting PR comments
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          # No requirements.txt needed for check_eval_scores.py (uses stdlib only)
+
+      - name: Run Evaluations
+        env:
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          LLM_AUTH_TOKEN: ${{ secrets.LLM_AUTH_TOKEN }}
+          LLM_MODEL: ${{ github.event.inputs.model || 'claude-sonnet-4-5-20250929' }}
+        run: |
+          chmod +x ./evals/scripts/run-promptfoo-eval.sh
+          ./evals/scripts/run-promptfoo-eval.sh --json
+
+      - name: Check Quality Thresholds
+        id: check_thresholds
+        run: |
+          python3 evals/scripts/check_eval_scores.py \
+            --results eval-results.json \
+            --min-score 0.70 \
+            --min-pass-rate 0.70 \
+            --verbose || echo "threshold_failed=true" >> $GITHUB_OUTPUT
+
+      - name: Generate Summary
+        if: always()
+        id: summary
+        run: |
+          if [ -f eval-results.json ]; then
+            python3 << 'EOF'
+          import json
+          import os
+
+          with open('eval-results.json', 'r') as f:
+              data = json.load(f)
+
+          results = data.get('results', {})
+          stats = results.get('stats', {})
+
+          total = stats.get('successes', 0) + stats.get('failures', 0)
+          passed = stats.get('successes', 0)
+          failed = stats.get('failures', 0)
+          pass_rate = (passed / total * 100) if total > 0 else 0
+
+          # Token usage
+          tokens = stats.get('tokenUsage', {})
+          total_tokens = tokens.get('total', 0)
+          cached_tokens = tokens.get('cached', 0)
+
+          summary = f"""## 📊 Eval Results
+
+          **Overall:** {passed}/{total} tests passed ({pass_rate:.0f}%)
+
+          | Metric | Value |
+          |--------|-------|
+          | ✅ Passed | {passed} |
+          | ❌ Failed | {failed} |
+          | 📈 Pass Rate | {pass_rate:.0f}% |
+          | 🪙 Total Tokens | {total_tokens:,} |
+          | 💾 Cached Tokens | {cached_tokens:,} |
+
+          """
+
+          # List failed tests
+          if failed > 0:
+              summary += "\n### ❌ Failed Tests\n\n"
+              for result in results.get('results', []):
+                  if not result.get('success', False):
+                      test_name = result.get('description', 'Unknown')
+                      score = result.get('score', 0)
+                      summary += f"- {test_name} (score: {score:.2f})\n"
+
+          # Success message
+          if pass_rate >= 70:
+              summary += "\n✅ **Quality thresholds met!**"
+          else:
+              summary += "\n⚠️ **Quality thresholds not met.** Please review failures."
+
+          # Write to output file for PR comment
+          with open('eval_summary.txt', 'w') as f:
+              f.write(summary)
+
+          print(summary)
+          EOF
+          else
+            echo "⚠️ No evaluation results found" > eval_summary.txt
+          fi
+
+      - name: Comment PR with Results
+        if: github.event_name == 'pull_request' && always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+
+            let summary = '## 📊 Eval Results\n\n⚠️ Evaluation failed to complete.';
+
+            if (fs.existsSync('eval_summary.txt')) {
+              summary = fs.readFileSync('eval_summary.txt', 'utf8');
+            }
+
+            // Find existing comment
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+
+            const botComment = comments.find(comment =>
+              comment.user.type === 'Bot' &&
+              comment.body.includes('📊 Eval Results')
+            );
+
+            if (botComment) {
+              // Update existing comment
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+                body: summary
+              });
+            } else {
+              // Create new comment
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: context.issue.number,
+                body: summary
+              });
+            }
+
+      - name: Upload Results Artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-results
+          path: |
+            eval-results*.json
+            eval_summary.txt
+          retention-days: 30
+
+      - name: Fail if thresholds not met
+        if: steps.check_thresholds.outputs.threshold_failed == 'true'
+        run: |
+          echo "❌ Quality thresholds not met"
+          exit 1

.github/workflows/scripts/create-github-release.sh

@@ -12,43 +12,27 @@ fi
 
 VERSION="$1"
 
-# Remove 'v' prefix from version for release title
+# Tag prefix must remain in sync with get-next-version
+TAG_PREFIX="agentic-sdlc-v"
+
+# Remove prefix from version for release title
 VERSION_NO_V=${VERSION#v}
 
-gh release create "$VERSION" \
-  .genreleases/spec-kit-template-copilot-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-copilot-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-claude-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-claude-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-gemini-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-gemini-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-cursor-agent-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-cursor-agent-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-opencode-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-opencode-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-qwen-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-qwen-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-windsurf-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-windsurf-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-codex-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-codex-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-kilocode-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-kilocode-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-auggie-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-auggie-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-roo-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-roo-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-codebuddy-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-codebuddy-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-qoder-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-qoder-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-amp-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-amp-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-shai-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-shai-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-q-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-q-ps-"$VERSION".zip \
-  .genreleases/spec-kit-template-bob-sh-"$VERSION".zip \
-  .genreleases/spec-kit-template-bob-ps-"$VERSION".zip \
-  --title "Spec Kit Templates - $VERSION_NO_V" \
+ASSETS=()
+AGENTS=(copilot claude gemini cursor-agent opencode qwen windsurf codex kilocode auggie roo codebuddy qoder amp shai q bob)
+SCRIPTS=(sh ps)
+
+for agent in "${AGENTS[@]}"; do
+  for script in "${SCRIPTS[@]}"; do
+    asset_path=".genreleases/agentic-sdlc-spec-kit-template-${agent}-${script}-${VERSION}.zip"
+    if [[ ! -f "$asset_path" ]]; then
+      echo "Error: missing release asset $asset_path" >&2
+      exit 1
+    fi
+    ASSETS+=("$asset_path")
+  done
+done
+
+gh release create "$VERSION" "${ASSETS[@]}" \
+  --title "Agentic SDLC Spec Kit Templates - $VERSION_NO_V" \
   --notes-file release_notes.md

.github/workflows/scripts/create-release-packages.sh

@@ -18,8 +18,10 @@ if [[ $# -ne 1 ]]; then
   exit 1
 fi
 NEW_VERSION="$1"
-if [[ ! $NEW_VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-  echo "Version must look like v0.0.0" >&2
+TAG_PREFIX="agentic-sdlc-v"
+
+if [[ ! $NEW_VERSION =~ ^${TAG_PREFIX}[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  echo "Version must look like ${TAG_PREFIX}0.0.0" >&2
   exit 1
 fi
 
@@ -218,8 +220,8 @@ build_variant() {
       mkdir -p "$base_dir/.bob/commands"
       generate_commands bob md "\$ARGUMENTS" "$base_dir/.bob/commands" "$script" ;;
   esac
-  ( cd "$base_dir" && zip -r "../spec-kit-template-${agent}-${script}-${NEW_VERSION}.zip" . )
-  echo "Created $GENRELEASES_DIR/spec-kit-template-${agent}-${script}-${NEW_VERSION}.zip"
+  ( cd "$base_dir" && zip -r "../agentic-sdlc-spec-kit-template-${agent}-${script}-${NEW_VERSION}.zip" . )
+  echo "Created $GENRELEASES_DIR/agentic-sdlc-spec-kit-template-${agent}-${script}-${NEW_VERSION}.zip"
 }
 
 # Determine agent list
@@ -269,5 +271,4 @@ for agent in "${AGENT_LIST[@]}"; do
 done
 
 echo "Archives in $GENRELEASES_DIR:"
-ls -1 "$GENRELEASES_DIR"/spec-kit-template-*-"${NEW_VERSION}".zip
-
+ls -1 "$GENRELEASES_DIR"/agentic-sdlc-spec-kit-template-*-"${NEW_VERSION}".zip

.github/workflows/scripts/generate-release-notes.sh

@@ -10,11 +10,12 @@ if [[ $# -ne 2 ]]; then
   exit 1
 fi
 
+TAG_PREFIX="agentic-sdlc-v"
 NEW_VERSION="$1"
 LAST_TAG="$2"
 
 # Get commits since last tag
-if [ "$LAST_TAG" = "v0.0.0" ]; then
+if [ "$LAST_TAG" = "${TAG_PREFIX}0.0.0" ]; then
   # Check how many commits we have and use that as the limit
   COMMIT_COUNT=$(git rev-list --count HEAD)
   if [ "$COMMIT_COUNT" -gt 10 ]; then

.github/workflows/scripts/get-next-version.sh

@@ -5,20 +5,26 @@ set -euo pipefail
 # Calculate the next version based on the latest git tag and output GitHub Actions variables
 # Usage: get-next-version.sh
 
-# Get the latest tag, or use v0.0.0 if no tags exist
-LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0")
+# Prefix all fork-specific tags to avoid upstream conflicts
+TAG_PREFIX="agentic-sdlc-v"
+
+# Get the latest prefixed tag, or fall back to the prefixed zero version
+LATEST_TAG=$(git tag --list "${TAG_PREFIX}*" --sort=-v:refname | head -n 1)
+if [[ -z "${LATEST_TAG}" ]]; then
+  LATEST_TAG="${TAG_PREFIX}0.0.0"
+fi
 echo "latest_tag=$LATEST_TAG" >> $GITHUB_OUTPUT
 
 # Extract version number and increment
-VERSION=$(echo $LATEST_TAG | sed 's/v//')
+VERSION=${LATEST_TAG#${TAG_PREFIX}}
 IFS='.' read -ra VERSION_PARTS <<< "$VERSION"
 MAJOR=${VERSION_PARTS[0]:-0}
 MINOR=${VERSION_PARTS[1]:-0}
 PATCH=${VERSION_PARTS[2]:-0}
 
 # Increment patch version
 PATCH=$((PATCH + 1))
-NEW_VERSION="v$MAJOR.$MINOR.$PATCH"
+NEW_VERSION="${TAG_PREFIX}$MAJOR.$MINOR.$PATCH"
 
 echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT
 echo "New version will be: $NEW_VERSION"

.github/workflows/scripts/update-version.sh

@@ -12,8 +12,10 @@ fi
 
 VERSION="$1"
 
-# Remove 'v' prefix for Python versioning
-PYTHON_VERSION=${VERSION#v}
+TAG_PREFIX="agentic-sdlc-v"
+
+# Remove custom prefix for Python versioning
+PYTHON_VERSION=${VERSION#${TAG_PREFIX}}
 
 if [ -f "pyproject.toml" ]; then
   sed -i "s/version = \".*\"/version = \"$PYTHON_VERSION\"/" pyproject.toml

.gitignore

@@ -34,6 +34,9 @@ env/
 .DS_Store
 *.tmp
 
+# Github
+.github/workflows/.secrets
+
 # Project specific
 *.log
 .env
@@ -43,4 +46,26 @@ env/
 # Spec Kit-specific files
 .genreleases/
 *.zip
-sdd-*/
+sdd-*
+
+# Evaluation artifacts
+eval-results*.json
+*.backup
+.promptfoo/
+evals/.venv/
+evals/notebooks/.venv/
+evals/notebooks/.ipynb_checkpoints/
+evals/datasets/analysis-results/*.csv
+evals/datasets/analysis-results/*.txt
+evals/datasets/analysis-results/automated-analysis-*.csv
+evals/datasets/analysis-results/plan-analysis-*.csv
+evals/datasets/analysis-results/plan-eval-analysis-*.txt
+evals/datasets/analysis-results/plan-summary-*.txt
+evals/datasets/analysis-results/promptfoo-analysis-*.csv
+evals/datasets/analysis-results/summary-*.txt
+
+# Annotation tool artifacts
+evals/annotation-tool/.venv/
+evals/annotation-tool/annotations.json
+evals/annotation-tool/annotations_export_*.json
+evals/annotation-tool/.sesskey

.markdownlint-cli2.jsonc

@@ -22,6 +22,7 @@
     }
   },
   "ignores": [
-    ".genreleases/"
+    ".genreleases/",
+    "evals/docs"
   ]
-}
+}

.markdownlintignore

@@ -0,0 +1 @@
+evals/docs