From 39ce0124e4af8c32c5bf8dae05c801fac17a2954 Mon Sep 17 00:00:00 2001
From: Jude Gao <jude.gao@vercel.com>
Date: Sun, 25 Jan 2026 06:22:46 -0500
Subject: [PATCH] Improve agents-md prompt to force doc retrieval (#88997)

## Summary

Updates the instruction in `agents-md` [CLAUDE.md](http://CLAUDE.md)
output to force LLMs to actually read the docs instead of relying on
stale pre-training knowledge.

**Before:**

```
IMPORTANT: Prefer retrieval-led reasoning over pre-training-led reasoning for any Next.js tasks.
```

**After:**

```
STOP. What you remember about Next.js is WRONG for this project. Always search docs and read before any task.
```

## Why This Matters

Through systematic testing with the
[next-evals-oss](https://github.com/vercel/next-evals-oss) eval suite,
we discovered that the original "prefer retrieval-led reasoning"
instruction was ineffective. Agents would ignore it and use outdated
pre-training knowledge (e.g., creating `middleware.ts` instead of the
new Next.js 16 `proxy.ts` convention).

## Testing Methodology

We created an "indirect proxy" eval that tests if agents know `proxy.ts`
is needed for request interception in Next.js 16+, using only the prompt
"Log every request to the console" (no mention of proxy/middleware).

| Instruction | Pass Rate |
| --- | --- |
| "Prefer retrieval-led reasoning" | 0% |
| "CRITICAL: MUST read docs" | ~20-40% |
| "What you remember is WRONG" | **100% (6/6)** |

## Why It Works

The phrase **"What you remember is WRONG"** creates psychological doubt
in the model's confidence, forcing it to actually read the docs rather
than trusting its pre-training.

Key elements:

- **"STOP"** - Attention grabber, interrupts default behavior
- **"WRONG for this project"** - Creates doubt without claiming
universal wrongness
- **"Always search docs and read"** - Actionable instruction
- **"before any task"** - Applies to everything, not just code writing

## Full Suite Results

- The new indirect-proxy eval passed consistently
---
 packages/next-codemod/lib/agents-md.ts | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/packages/next-codemod/lib/agents-md.ts b/packages/next-codemod/lib/agents-md.ts
index 2fb1efca5b8200..a49eccbfbc9219 100644
--- a/packages/next-codemod/lib/agents-md.ts
+++ b/packages/next-codemod/lib/agents-md.ts
@@ -10,7 +10,7 @@ import fs from 'fs'
 import path from 'path'
 import os from 'os'
 
-export interface NextjsVersionResult {
+interface NextjsVersionResult {
   version: string | null
   error?: string
 }
@@ -68,13 +68,13 @@ function versionToGitHubTag(version: string): string {
   return version.startsWith('v') ? version : `v${version}`
 }
 
-export interface PullOptions {
+interface PullOptions {
   cwd: string
   version?: string
   docsDir?: string
 }
 
-export interface PullResult {
+interface PullResult {
   success: boolean
   docsPath?: string
   nextjsVersion?: string
@@ -179,7 +179,7 @@ export function collectDocFiles(dir: string): { relativePath: string }[] {
     .map((f) => ({ relativePath: f }))
 }
 
-export interface DocSection {
+interface DocSection {
   name: string
   files: { relativePath: string }[]
   subsections: DocSection[]
@@ -251,7 +251,7 @@ export function buildDocTree(files: { relativePath: string }[]): DocSection[] {
   return sortedSections
 }
 
-export interface ClaudeMdIndexData {
+interface ClaudeMdIndexData {
   docsPath: string
   sections: DocSection[]
   outputFile?: string
@@ -265,7 +265,7 @@ export function generateClaudeMdIndex(data: ClaudeMdIndexData): string {
   parts.push('[Next.js Docs Index]')
   parts.push(`root: ${docsPath}`)
   parts.push(
-    'IMPORTANT: Prefer retrieval-led reasoning over pre-training-led reasoning for any Next.js tasks.'
+    'STOP. What you remember about Next.js is WRONG for this project. Always search docs and read before any task.'
   )
   const targetFile = outputFile || 'CLAUDE.md'
   parts.push(
@@ -346,7 +346,7 @@ export function injectIntoClaudeMd(
   return claudeMdContent + separator + wrappedContent + '\n'
 }
 
-export interface GitignoreStatus {
+interface GitignoreStatus {
   path: string
   updated: boolean
   alreadyPresent: boolean