From 009d8862b9e03e0d8c96454c3d19ab51f9034d8f Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 17 Feb 2026 11:25:29 -0800 Subject: [PATCH 1/9] feat: improve LLM markdown exports with doctree-based titles and navigation Load public/doctree.json to replace raw slug names with proper titles in generated .md files. Adds a sectionOverrides registry for clean per-page customization of appended navigation sections. Changes: - Root index.md: platforms by title, frameworks grouped by platform, doc sections by title - platforms.md: structured Platforms + Frameworks sections - Platform pages (e.g. platforms/javascript.md): "Frameworks" with proper titles, sorted "Topics" - Guide pages (e.g. platforms/javascript/guides/nextjs.md): "Other Frameworks" + "Topics" - All other section pages: titles and sidebar_order from doctree - Graceful fallback to slug-based output if doctree unavailable Closes #16413 Co-Authored-By: Claude --- scripts/generate-md-exports.mjs | 472 +++++++++++++++++++++++++++----- 1 file changed, 407 insertions(+), 65 deletions(-) diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index 547709bf8e219..0c8e09d15536c 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -38,6 +38,41 @@ const R2_BUCKET = process.env.NEXT_PUBLIC_DEVELOPER_DOCS const accessKeyId = process.env.R2_ACCESS_KEY_ID; const secretAccessKey = process.env.R2_SECRET_ACCESS_KEY; +// --- Doc tree helpers for generating structured navigation --- + +function findNode(node, pathParts) { + for (const part of pathParts) { + node = node.children?.find(c => c.slug === part); + if (!node) { + return null; + } + } + return node; +} + +function getTitle(node) { + return node.frontmatter?.sidebar_title || node.frontmatter?.title || node.slug; +} + +function isVisible(node) { + return ( + !node.frontmatter?.sidebar_hidden && + !node.frontmatter?.draft && + !node.path?.includes('__v') && + (node.frontmatter?.title || node.frontmatter?.sidebar_title) + ); +} + +function getVisibleChildren(node) { + return (node.children || []) + .filter(isVisible) + .sort((a, b) => { + const orderDiff = + (a.frontmatter?.sidebar_order ?? 99) - (b.frontmatter?.sidebar_order ?? 99); + return orderDiff !== 0 ? orderDiff : getTitle(a).localeCompare(getTitle(b)); + }); +} + function getS3Client() { return new S3Client({ endpoint: 'https://773afa1f62ff86c80db4f24f7ff1e9c8.r2.cloudflarestorage.com', @@ -61,6 +96,349 @@ async function uploadToCFR2(s3Client, relativePath, data) { return; } +// --- Index and child section builders --- + +function titleForPath(docTree, mdPath) { + const parts = mdPath.replace(/\.md$/, '').split('/'); + const node = findNode(docTree, parts); + return node ? getTitle(node) : parts[parts.length - 1]; +} + +function buildDocTreeRootIndex(docTree, topLevelPaths) { + const platformsNode = findNode(docTree, ['platforms']); + let platformsSection = ''; + let frameworksSection = ''; + + if (platformsNode) { + const platforms = getVisibleChildren(platformsNode); + platformsSection = `## Platforms\n\n${platforms + .map(p => `- [${getTitle(p)}](${DOCS_ORIGIN}/platforms/${p.slug}.md)`) + .join('\n')}\n`; + + const frameworkLines = []; + for (const platform of platforms) { + const guidesNode = findNode(platform, ['guides']); + if (!guidesNode) { + continue; + } + const guides = getVisibleChildren(guidesNode); + if (guides.length === 0) { + continue; + } + frameworkLines.push(`\n### ${getTitle(platform)}\n`); + for (const guide of guides) { + frameworkLines.push( + `- [${getTitle(guide)}](${DOCS_ORIGIN}/platforms/${platform.slug}/guides/${guide.slug}.md)` + ); + } + } + if (frameworkLines.length > 0) { + frameworksSection = `## Frameworks\n${frameworkLines.join('\n')}\n`; + } + } + + // Build documentation sections from top-level pages (excluding platforms) + const docSections = topLevelPaths + .filter(p => p !== '_not-found.md' && p !== 'platforms.md') + .map(p => { + const title = titleForPath(docTree, p); + return `- [${title}](${DOCS_ORIGIN}/${p})`; + }) + .join('\n'); + + return `# Sentry Documentation + +Sentry is a developer-first application monitoring platform that helps you identify and fix issues in real-time. It provides error tracking, performance monitoring, session replay, and more across all major platforms and frameworks. + +## Key Features + +- **Error Monitoring**: Capture and diagnose errors with full stack traces, breadcrumbs, and context +- **Tracing**: Track requests across services to identify performance bottlenecks +- **Session Replay**: Watch real user sessions to understand what led to errors +- **Profiling**: Identify slow functions and optimize application performance +- **Crons**: Monitor scheduled jobs and detect failures +- **Logs**: Collect and analyze application logs in context + +${platformsSection} +${frameworksSection} +## Documentation + +${docSections} + +## Quick Links + +- [Platform SDKs](${DOCS_ORIGIN}/platforms.md) - Install Sentry for your language/framework +- [API Reference](${DOCS_ORIGIN}/api.md) - Programmatic access to Sentry +- [CLI](${DOCS_ORIGIN}/cli.md) - Command-line interface for Sentry operations +`; +} + +function buildDocTreeDevRootIndex(docTree, topLevelPaths) { + const docSections = topLevelPaths + .filter(p => p !== '_not-found.md') + .map(p => { + const title = titleForPath(docTree, p); + return `- [${title}](${DOCS_ORIGIN}/${p})`; + }) + .join('\n'); + + return `# Sentry Developer Documentation + +${docSections} + +## Quick Links + +- [Backend Development](${DOCS_ORIGIN}/backend.md) - Backend service architecture +- [Frontend Development](${DOCS_ORIGIN}/frontend.md) - Frontend development guide +- [SDK Development](${DOCS_ORIGIN}/sdk.md) - SDK development documentation +`; +} + +function buildFallbackRootIndex(topLevelPaths) { + return `# Sentry Documentation + +Sentry is a developer-first application monitoring platform that helps you identify and fix issues in real-time. It provides error tracking, performance monitoring, session replay, and more across all major platforms and frameworks. + +## Key Features + +- **Error Monitoring**: Capture and diagnose errors with full stack traces, breadcrumbs, and context +- **Tracing**: Track requests across services to identify performance bottlenecks +- **Session Replay**: Watch real user sessions to understand what led to errors +- **Profiling**: Identify slow functions and optimize application performance +- **Crons**: Monitor scheduled jobs and detect failures +- **Logs**: Collect and analyze application logs in context + +## Documentation Sections + +${topLevelPaths + .filter(p => p !== '_not-found.md') + .map(p => `- [${p.replace(/\.md$/, '')}](${DOCS_ORIGIN}/${p})`) + .join('\n')} + +${ + process.env.NEXT_PUBLIC_DEVELOPER_DOCS + ? `## Quick Links + +- [Backend Development](${DOCS_ORIGIN}/backend.md) - Backend service architecture +- [Frontend Development](${DOCS_ORIGIN}/frontend.md) - Frontend development guide +- [SDK Development](${DOCS_ORIGIN}/sdk.md) - SDK development documentation +` + : `## Quick Links + +- [Platform SDKs](${DOCS_ORIGIN}/platforms.md) - Install Sentry for your language/framework +- [API Reference](${DOCS_ORIGIN}/api.md) - Programmatic access to Sentry +- [CLI](${DOCS_ORIGIN}/cli.md) - Command-line interface for Sentry operations +` +}`; +} + +// Registry of custom section builders for specific page patterns. +// Each entry has a `match` function and a `build` function. +// First match wins; unmatched pages fall through to the generic builder. +const sectionOverrides = [ + { + match: (_parts, parentPath) => parentPath === 'platforms.md', + build: (docTree, _parentParts, _parentNode, _children) => + buildPlatformsIndexSection(docTree), + }, + { + // Platform index pages (e.g., platforms/javascript.md) + match: (parts, parentPath) => + parentPath.startsWith('platforms/') && parts.length === 2, + build: (docTree, parentParts, parentNode, children) => + buildPlatformPageSection(docTree, parentParts, parentNode, children), + }, + { + // Guide index pages (e.g., platforms/javascript/guides/nextjs.md) + match: (parts, _parentPath) => + parts.length === 4 && parts[0] === 'platforms' && parts[2] === 'guides', + build: (docTree, parentParts, _parentNode, children) => + buildGuidePageSection(docTree, parentParts, _parentNode, children), + }, +]; + +function buildDocTreeChildSection(docTree, parentPath, children) { + const parentParts = parentPath.replace(/\.md$/, '').split('/'); + const parentNode = findNode(docTree, parentParts); + + for (const override of sectionOverrides) { + if (override.match(parentParts, parentPath)) { + return override.build(docTree, parentParts, parentNode, children); + } + } + + // Default: generic section with doctree titles + return buildGenericSection(docTree, parentPath, children); +} + +function buildPlatformsIndexSection(docTree) { + const platformsNode = findNode(docTree, ['platforms']); + if (!platformsNode) { + return ''; + } + + const platforms = getVisibleChildren(platformsNode); + let section = `\n## Platforms\n\n`; + section += platforms + .map(p => `- [${getTitle(p)}](${DOCS_ORIGIN}/platforms/${p.slug}.md)`) + .join('\n'); + + const frameworkLines = []; + for (const platform of platforms) { + const guidesNode = findNode(platform, ['guides']); + if (!guidesNode) { + continue; + } + const guides = getVisibleChildren(guidesNode); + if (guides.length === 0) { + continue; + } + frameworkLines.push(`\n### ${getTitle(platform)}\n`); + for (const guide of guides) { + frameworkLines.push( + `- [${getTitle(guide)}](${DOCS_ORIGIN}/platforms/${platform.slug}/guides/${guide.slug}.md)` + ); + } + } + if (frameworkLines.length > 0) { + section += `\n\n## Frameworks\n${frameworkLines.join('\n')}\n`; + } + + return section + '\n'; +} + +function buildPlatformPageSection(docTree, parentParts, parentNode, children) { + const platformSlug = parentParts[1]; + let section = ''; + + // Frameworks (guides) + const guides = children.filter(p => p.includes('/guides/')); + if (guides.length > 0 && parentNode) { + const guidesNode = findNode(parentNode, ['guides']); + if (guidesNode) { + const sortedGuides = getVisibleChildren(guidesNode); + section += `\n## Frameworks\n\n`; + section += sortedGuides + .map( + g => + `- [${getTitle(g)}](${DOCS_ORIGIN}/platforms/${platformSlug}/guides/${g.slug}.md)` + ) + .join('\n'); + section += '\n'; + } else { + section += buildFallbackGuidesList(guides, 'Frameworks'); + } + } + + // Topics (non-guide children) + const topics = children.filter(p => !p.includes('/guides/')); + if (topics.length > 0) { + section += buildSortedTopicsSection(docTree, topics, 'Topics'); + } + + return section; +} + +function buildGuidePageSection(docTree, parentParts, _parentNode, children) { + const platformSlug = parentParts[1]; + const guideSlug = parentParts[3]; + let section = ''; + + // Other frameworks for this platform + const platformNode = findNode(docTree, ['platforms', platformSlug]); + if (platformNode) { + const guidesNode = findNode(platformNode, ['guides']); + if (guidesNode) { + const siblingGuides = getVisibleChildren(guidesNode).filter( + g => g.slug !== guideSlug + ); + if (siblingGuides.length > 0) { + const platformTitle = getTitle(platformNode); + section += `\n## Other ${platformTitle} Frameworks\n\n`; + section += siblingGuides + .map( + g => + `- [${getTitle(g)}](${DOCS_ORIGIN}/platforms/${platformSlug}/guides/${g.slug}.md)` + ) + .join('\n'); + section += '\n'; + } + } + } + + // Topics (child pages of this guide) + if (children.length > 0) { + section += buildSortedTopicsSection(docTree, children, 'Topics'); + } + + return section; +} + +function buildSortedTopicsSection(docTree, pages, heading) { + // Sort using doctree order when available + const pagesWithMeta = pages.map(p => { + const parts = p.replace(/\.md$/, '').split('/'); + const node = findNode(docTree, parts); + return { + path: p, + title: node ? getTitle(node) : parts[parts.length - 1], + order: node?.frontmatter?.sidebar_order ?? 99, + }; + }); + + pagesWithMeta.sort((a, b) => { + const orderDiff = a.order - b.order; + return orderDiff !== 0 ? orderDiff : a.title.localeCompare(b.title); + }); + + let section = `\n## ${heading}\n\n`; + section += pagesWithMeta + .map(p => `- [${p.title}](${DOCS_ORIGIN}/${p.path})`) + .join('\n'); + section += '\n'; + return section; +} + +function buildGenericSection(docTree, _parentPath, children) { + // Use doctree titles for all children + return buildSortedTopicsSection(docTree, children, 'Pages in this section'); +} + +function buildFallbackGuidesList(guides, heading) { + const guideList = guides + .map(p => { + const name = p.replace(/\.md$/, '').split('/').pop(); + return `- [${name}](${DOCS_ORIGIN}/${p})`; + }) + .join('\n'); + return `\n## ${heading}\n\n${guideList}\n`; +} + +function buildFallbackChildSection(parentPath, children) { + const isPlatformIndex = + parentPath.startsWith('platforms/') && parentPath.split('/').length === 2; + + const guides = isPlatformIndex ? children.filter(p => p.includes('/guides/')) : []; + const otherPages = isPlatformIndex + ? children.filter(p => !p.includes('/guides/')) + : children; + + let childSection = ''; + if (guides.length > 0) { + childSection += buildFallbackGuidesList(guides, 'Guides'); + } + if (otherPages.length > 0) { + const pageList = otherPages + .map(p => { + const name = p.replace(/\.md$/, '').split('/').pop(); + return `- [${name}](${DOCS_ORIGIN}/${p})`; + }) + .join('\n'); + childSection += `\n## Pages in this section\n\n${pageList}\n`; + } + return childSection; +} + // Global set to track which cache files are used across all workers let globalUsedCacheFiles = null; @@ -99,6 +477,20 @@ async function createWork() { console.log(`🚀 Starting markdown generation from: ${INPUT_DIR}`); console.log(`📁 Output directory: ${OUTPUT_DIR}`); + // Load doctree for structured navigation + const doctreeFilename = process.env.NEXT_PUBLIC_DEVELOPER_DOCS + ? 'doctree-dev.json' + : 'doctree.json'; + const doctreePath = path.join(root, 'public', doctreeFilename); + let docTree = null; + try { + docTree = JSON.parse(await readFile(doctreePath, {encoding: 'utf8'})); + console.log(`🌳 Loaded doc tree from ${doctreePath}`); + } catch (err) { + console.warn(`⚠️ Could not load doctree (${doctreePath}): ${err.message}`); + console.warn(' Falling back to slug-based navigation'); + } + // Clear output directory await rm(OUTPUT_DIR, {recursive: true, force: true}); await mkdir(OUTPUT_DIR, {recursive: true}); @@ -224,43 +616,17 @@ async function createWork() { .filter(p => p !== 'index.md') .sort(); - // Root index.md - only top-level pages (no slashes in path) + // Root index.md - build using doctree when available const topLevelPaths = allPaths.filter(p => !p.includes('/')); - const rootSitemapContent = `# Sentry Documentation - -Sentry is a developer-first application monitoring platform that helps you identify and fix issues in real-time. It provides error tracking, performance monitoring, session replay, and more across all major platforms and frameworks. - -## Key Features - -- **Error Monitoring**: Capture and diagnose errors with full stack traces, breadcrumbs, and context -- **Tracing**: Track requests across services to identify performance bottlenecks -- **Session Replay**: Watch real user sessions to understand what led to errors -- **Profiling**: Identify slow functions and optimize application performance -- **Crons**: Monitor scheduled jobs and detect failures -- **Logs**: Collect and analyze application logs in context - -## Documentation Sections + let rootSitemapContent; -${topLevelPaths - .filter(p => p !== '_not-found.md') - .map(p => `- [${p.replace(/\.md$/, '')}](${DOCS_ORIGIN}/${p})`) - .join('\n')} - -${ - process.env.NEXT_PUBLIC_DEVELOPER_DOCS - ? `## Quick Links - -- [Backend Development](${DOCS_ORIGIN}/backend.md) - Backend service architecture -- [Frontend Development](${DOCS_ORIGIN}/frontend.md) - Frontend development guide -- [SDK Development](${DOCS_ORIGIN}/sdk.md) - SDK development documentation -` - : `## Quick Links - -- [Platform SDKs](${DOCS_ORIGIN}/platforms.md) - Install Sentry for your language/framework -- [API Reference](${DOCS_ORIGIN}/api.md) - Programmatic access to Sentry -- [CLI](${DOCS_ORIGIN}/cli.md) - Command-line interface for Sentry operations -` -}`; + if (docTree && !process.env.NEXT_PUBLIC_DEVELOPER_DOCS) { + rootSitemapContent = buildDocTreeRootIndex(docTree, topLevelPaths); + } else if (docTree && process.env.NEXT_PUBLIC_DEVELOPER_DOCS) { + rootSitemapContent = buildDocTreeDevRootIndex(docTree, topLevelPaths); + } else { + rootSitemapContent = buildFallbackRootIndex(topLevelPaths); + } const indexPath = path.join(OUTPUT_DIR, 'index.md'); await writeFile(indexPath, rootSitemapContent, {encoding: 'utf8'}); @@ -273,7 +639,9 @@ ${ const pathsByParent = new Map(); for (const p of allPaths) { const parts = p.replace(/\.md$/, '').split('/'); - if (parts.length <= 1) continue; + if (parts.length <= 1) { + continue; + } // Determine the parent path - always direct parent, except: // Guide index pages (platforms/X/guides/Y.md) -> parent is platform (platforms/X.md) @@ -314,35 +682,9 @@ ${ throw err; } - // Only show "## Guides" section for platform index pages (e.g., platforms/javascript.md) - // These are the only pages that have guide children (platforms/X/guides/Y.md) - const isPlatformIndex = - parentPath.startsWith('platforms/') && parentPath.split('/').length === 2; // e.g., "platforms/javascript.md" - - const guides = isPlatformIndex ? children.filter(p => p.includes('/guides/')) : []; - const otherPages = isPlatformIndex - ? children.filter(p => !p.includes('/guides/')) - : children; - - let childSection = ''; - if (guides.length > 0) { - const guideList = guides - .map(p => { - const name = p.replace(/\.md$/, '').split('/').pop(); - return `- [${name}](${DOCS_ORIGIN}/${p})`; - }) - .join('\n'); - childSection += `\n## Guides\n\n${guideList}\n`; - } - if (otherPages.length > 0) { - const pageList = otherPages - .map(p => { - const name = p.replace(/\.md$/, '').split('/').pop(); - return `- [${name}](${DOCS_ORIGIN}/${p})`; - }) - .join('\n'); - childSection += `\n## Pages in this section\n\n${pageList}\n`; - } + const childSection = docTree + ? buildDocTreeChildSection(docTree, parentPath, children) + : buildFallbackChildSection(parentPath, children); if (childSection) { const updatedContent = existingContent + childSection; From 66f6505b666d789b6ee9de4067c6a2b67c66bad8 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Tue, 17 Feb 2026 11:28:24 -0800 Subject: [PATCH 2/9] feat: add content override registry and LLM docs spec - Refactor root index.md generation to use contentOverrides registry (same pattern as sectionOverrides but for full page replacement) - Track content override results for proper R2 upload handling - Add specs/llm-friendly-docs.md documenting the content negotiation system, override architecture, and page customization patterns Co-Authored-By: Claude --- scripts/generate-md-exports.mjs | 65 ++++++++----- specs/llm-friendly-docs.md | 157 ++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+), 23 deletions(-) create mode 100644 specs/llm-friendly-docs.md diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index 0c8e09d15536c..f11ba39127bd1 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -232,9 +232,29 @@ ${ }`; } +// Registry of full-page content overrides. +// When a page matches, its entire content is replaced (not appended). +// Used for pages like root index.md that need completely custom content. +const contentOverrides = [ + { + match: relativePath => relativePath === 'index.md', + build: (docTree, _existingContent, {allPaths}) => { + const topLevelPaths = allPaths.filter(p => !p.includes('/')); + if (docTree && !process.env.NEXT_PUBLIC_DEVELOPER_DOCS) { + return buildDocTreeRootIndex(docTree, topLevelPaths); + } + if (docTree && process.env.NEXT_PUBLIC_DEVELOPER_DOCS) { + return buildDocTreeDevRootIndex(docTree, topLevelPaths); + } + return buildFallbackRootIndex(topLevelPaths); + }, + }, +]; + // Registry of custom section builders for specific page patterns. // Each entry has a `match` function and a `build` function. // First match wins; unmatched pages fall through to the generic builder. +// These are appended to the existing converted content. const sectionOverrides = [ { match: (_parts, parentPath) => parentPath === 'platforms.md', @@ -616,24 +636,21 @@ async function createWork() { .filter(p => p !== 'index.md') .sort(); - // Root index.md - build using doctree when available - const topLevelPaths = allPaths.filter(p => !p.includes('/')); - let rootSitemapContent; - - if (docTree && !process.env.NEXT_PUBLIC_DEVELOPER_DOCS) { - rootSitemapContent = buildDocTreeRootIndex(docTree, topLevelPaths); - } else if (docTree && process.env.NEXT_PUBLIC_DEVELOPER_DOCS) { - rootSitemapContent = buildDocTreeDevRootIndex(docTree, topLevelPaths); - } else { - rootSitemapContent = buildFallbackRootIndex(topLevelPaths); + // Apply full-page content overrides (e.g., root index.md) + // These replace the worker-generated content entirely. + const overrideContext = {allPaths}; + const contentOverrideResults = new Map(); + for (const override of contentOverrides) { + for (const task of workerTasks.flat()) { + if (override.match(task.relativePath)) { + const content = override.build(docTree, null, overrideContext); + await writeFile(task.targetPath, content, {encoding: 'utf8'}); + contentOverrideResults.set(task.relativePath, content); + console.log(`📑 Generated ${task.relativePath} (content override)`); + } + } } - const indexPath = path.join(OUTPUT_DIR, 'index.md'); - await writeFile(indexPath, rootSitemapContent, {encoding: 'utf8'}); - console.log( - `📑 Generated root index.md with ${topLevelPaths.length} top-level sections` - ); - // Append child page listings to section index pages // Group paths by their DIRECT parent (handling guides specially) const pathsByParent = new Map(); @@ -715,14 +732,16 @@ async function createWork() { } console.log(`📑 Added child page listings to ${updatedCount} section index files`); - // Upload index.md to R2 if configured - if (accessKeyId && secretAccessKey) { + // Upload content-overridden pages to R2 if configured + if (accessKeyId && secretAccessKey && contentOverrideResults.size > 0) { const s3Client = getS3Client(); - const indexHash = md5(rootSitemapContent); - const existingHash = existingFilesOnR2?.get('index.md'); - if (existingHash !== indexHash) { - await uploadToCFR2(s3Client, 'index.md', rootSitemapContent); - console.log(`📤 Uploaded updated index.md to R2`); + for (const [relativePath, content] of contentOverrideResults) { + const fileHash = md5(content); + const existingHash = existingFilesOnR2?.get(relativePath); + if (existingHash !== fileHash) { + await uploadToCFR2(s3Client, relativePath, content); + console.log(`📤 Uploaded updated ${relativePath} to R2`); + } } } diff --git a/specs/llm-friendly-docs.md b/specs/llm-friendly-docs.md new file mode 100644 index 0000000000000..b125b5dcccdfb --- /dev/null +++ b/specs/llm-friendly-docs.md @@ -0,0 +1,157 @@ +# LLM-Friendly Documentation Spec + +## Overview + +Sentry docs are served as both HTML (for browsers) and Markdown (for LLMs). Every page at `docs.sentry.io/` has a corresponding `docs.sentry.io/.md` URL that returns a Markdown representation optimized for LLM consumption. + +The Markdown exports are generated by `scripts/generate-md-exports.mjs` as a post-build step after both `doctree.json` and the Next.js build are complete. + +## Content Negotiation + +| Request | Response | +|---------|----------| +| `GET /platforms/javascript` | HTML page (browser) | +| `GET /platforms/javascript.md` | Markdown file (LLM) | +| `GET /index.md` | Root navigation index | + +The `.md` suffix triggers content negotiation. Markdown files are pre-generated at build time and served from `public/md-exports/` (and optionally synced to R2 CDN). + +## How Markdown Pages Differ from HTML + +Markdown exports are not just raw dumps of HTML content. They are adapted for LLM consumption: + +| Aspect | HTML Page | Markdown Export | +|--------|-----------|-----------------| +| **Navigation** | Sidebar, breadcrumbs, prev/next links | Appended navigation sections with categorized links | +| **Interactive elements** | Tabs, code switchers, copy buttons | Removed (buttons stripped during conversion) | +| **Titles** | In `` tag and breadcrumbs | H1 heading at top of document | +| **Links** | Relative HTML paths | Absolute `.md` URLs (e.g., `https://docs.sentry.io/platforms/javascript.md`) | +| **Images** | Relative paths | Absolute URLs | +| **Page structure** | Header, sidebar, main content, footer | Title + main content + navigation sections | + +## Page Customization Architecture + +The generation script has two override registries for customizing output on a per-page basis: + +### Content Overrides (full page replacement) + +For pages that need completely custom content (e.g., root `index.md`), register a `contentOverride`. The override replaces the entire worker-generated Markdown with custom content. + +```javascript +const contentOverrides = [ + { + match: (relativePath) => relativePath === 'index.md', + build: (docTree, existingContent, context) => { + // Return full page content as a string + return `# Sentry Documentation\n...`; + }, + }, +]; +``` + +**Parameters:** +- `match(relativePath)` — return `true` if this override applies to the given path +- `build(docTree, existingContent, context)` — return the full page content + - `docTree` — parsed `doctree.json` (or `null` if unavailable) + - `existingContent` — the worker-generated Markdown (can be used as base) + - `context.allPaths` — list of all generated `.md` paths + +### Section Overrides (appended navigation) + +For pages that need custom navigation sections appended to the converted content, register a `sectionOverride`. These are appended after the main content. + +```javascript +const sectionOverrides = [ + { + match: (pathParts, parentPath) => parentPath === 'platforms.md', + build: (docTree, parentParts, parentNode, children) => { + // Return markdown string to append + return '\n## Platforms\n\n- [JavaScript](...)\n'; + }, + }, +]; +``` + +**Parameters:** +- `match(pathParts, parentPath)` — path segments array and full parent path (e.g., `['platforms', 'javascript']`, `'platforms/javascript.md'`) +- `build(docTree, parentParts, parentNode, children)` — return the section to append + - `parentParts` — path segments for the parent page + - `parentNode` — doctree node for the parent (if found) + - `children` — list of child `.md` paths that belong to this parent + +### Default Behavior + +Pages without a matching override get a generic "Pages in this section" listing with: +- Proper titles from doctree (falling back to slug) +- Sorted by `sidebar_order`, then alphabetically by title +- Hidden/draft/versioned pages filtered out + +## Current Override Registry + +### Content Overrides + +| Pattern | Description | +|---------|-------------| +| `index.md` | Root documentation index with platforms, frameworks, and doc sections | + +### Section Overrides + +| Pattern | Description | +|---------|-------------| +| `platforms.md` | All platforms by title + frameworks grouped by platform | +| `platforms/<sdk>.md` | "Frameworks" (guides with titles) + "Topics" (child pages sorted) | +| `platforms/<sdk>/guides/<framework>.md` | "Other Frameworks" (sibling guides) + "Topics" | + +## Doc Tree Integration + +The script loads `public/doctree.json` (generated by `scripts/generate-doctree.ts` earlier in the build pipeline). This provides: + +- **Titles**: `frontmatter.sidebar_title || frontmatter.title` (instead of raw slugs) +- **Ordering**: `frontmatter.sidebar_order` for consistent sort order +- **Visibility**: Filters out `sidebar_hidden`, `draft`, and versioned (`__v`) pages +- **Hierarchy**: Parent/child relationships for building navigation + +If `doctree.json` is unavailable, the script falls back to slug-based navigation (raw folder names, no ordering). + +## Build Pipeline + +``` +yarn generate-doctree → public/doctree.json +yarn next build → .next/server/app/**/*.html +yarn generate-md-exports → public/md-exports/**/*.md (+ R2 sync) +``` + +## Adding a New Override + +1. Write a builder function in `scripts/generate-md-exports.mjs` +2. Add an entry to `contentOverrides` (full replacement) or `sectionOverrides` (appended section) +3. The match function determines which pages use the override +4. Build and verify: check the output in `public/md-exports/` + +## Verification + +After `yarn build`, inspect: + +```bash +# Root index +cat public/md-exports/index.md + +# Platforms index +cat public/md-exports/platforms.md + +# A platform page +cat public/md-exports/platforms/javascript.md + +# A guide page +cat public/md-exports/platforms/javascript/guides/nextjs.md + +# A generic section page +cat public/md-exports/product.md +``` + +Verify that: +- Titles are human-readable (not raw slugs) +- Links use absolute `.md` URLs +- Pages are sorted by sidebar_order +- Hidden/draft pages are excluded +- Frameworks are listed by proper name (e.g., "Next.js" not "nextjs") From e4cce2be23750a654cf624053fb79141c2a8a490 Mon Sep 17 00:00:00 2001 From: David Cramer <dcramer@gmail.com> Date: Tue, 17 Feb 2026 12:31:18 -0800 Subject: [PATCH 3/9] feat: MDX template overrides and declarative page sections for LLM exports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace imperative JS builder functions with two-layer customization: - MDX templates in md-overrides/ for full-page content (like root index.md) with custom components (PlatformList, FrameworkGroups, DocSectionList) - Declarative pageOverrides table for appended navigation sections MDX templates are rendered to HTML via React SSR, then flow through the existing HTML→MD worker pipeline automatically getting link rewriting, caching, and R2 sync. This replaces ~10 builder functions and two separate override registries with a unified architecture. Co-Authored-By: Claude <noreply@anthropic.com> --- md-overrides/dev/index.mdx | 12 + md-overrides/index.mdx | 33 ++ package.json | 1 + scripts/generate-md-exports.mjs | 628 ++++++++++++++++---------------- specs/llm-friendly-docs.md | 137 +++++-- 5 files changed, 447 insertions(+), 364 deletions(-) create mode 100644 md-overrides/dev/index.mdx create mode 100644 md-overrides/index.mdx diff --git a/md-overrides/dev/index.mdx b/md-overrides/dev/index.mdx new file mode 100644 index 0000000000000..a821cc4b84048 --- /dev/null +++ b/md-overrides/dev/index.mdx @@ -0,0 +1,12 @@ +--- +title: "Sentry Developer Documentation" +append_sections: false +--- + +<DocSectionList exclude={["_not-found"]} /> + +## Quick Links + +- [Backend Development](/backend) - Backend service architecture +- [Frontend Development](/frontend) - Frontend development guide +- [SDK Development](/sdk) - SDK development documentation diff --git a/md-overrides/index.mdx b/md-overrides/index.mdx new file mode 100644 index 0000000000000..9bfcef9cb85ac --- /dev/null +++ b/md-overrides/index.mdx @@ -0,0 +1,33 @@ +--- +title: "Sentry Documentation" +append_sections: false +--- + +Sentry is a developer-first application monitoring platform that helps you identify and fix issues in real-time. It provides error tracking, performance monitoring, session replay, and more across all major platforms and frameworks. + +## Key Features + +- **Error Monitoring**: Capture and diagnose errors with full stack traces, breadcrumbs, and context +- **Tracing**: Track requests across services to identify performance bottlenecks +- **Session Replay**: Watch real user sessions to understand what led to errors +- **Profiling**: Identify slow functions and optimize application performance +- **Crons**: Monitor scheduled jobs and detect failures +- **Logs**: Collect and analyze application logs in context + +## Platforms + +<PlatformList /> + +## Frameworks + +<FrameworkGroups /> + +## Documentation + +<DocSectionList exclude={["platforms", "_not-found"]} /> + +## Quick Links + +- [Platform SDKs](/platforms) - Install Sentry for your language/framework +- [API Reference](/api) - Programmatic access to Sentry +- [CLI](/cli) - Command-line interface for Sentry operations diff --git a/package.json b/package.json index 1e9c523afa397..8f1cba5076224 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@emotion/styled": "^11.0.0", "@google-cloud/storage": "^7.7.0", "@mdx-js/loader": "^3.0.0", + "@mdx-js/mdx": "^3.0.0", "@mdx-js/react": "^3.0.0", "@pondorasti/remark-img-links": "^1.0.8", "@popperjs/core": "^2.11.8", diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index f11ba39127bd1..49d7509440e8e 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -64,13 +64,11 @@ function isVisible(node) { } function getVisibleChildren(node) { - return (node.children || []) - .filter(isVisible) - .sort((a, b) => { - const orderDiff = - (a.frontmatter?.sidebar_order ?? 99) - (b.frontmatter?.sidebar_order ?? 99); - return orderDiff !== 0 ? orderDiff : getTitle(a).localeCompare(getTitle(b)); - }); + return (node.children || []).filter(isVisible).sort((a, b) => { + const orderDiff = + (a.frontmatter?.sidebar_order ?? 99) - (b.frontmatter?.sidebar_order ?? 99); + return orderDiff !== 0 ? orderDiff : getTitle(a).localeCompare(getTitle(b)); + }); } function getS3Client() { @@ -93,206 +91,89 @@ async function uploadToCFR2(s3Client, relativePath, data) { ContentType: 'text/markdown', }); await s3Client.send(command); - return; } -// --- Index and child section builders --- +// --- Shared link resolver utilities for page overrides --- -function titleForPath(docTree, mdPath) { - const parts = mdPath.replace(/\.md$/, '').split('/'); - const node = findNode(docTree, parts); - return node ? getTitle(node) : parts[parts.length - 1]; +function nodeLinks(node, subtreePath) { + if (!node) { + return []; + } + const subtree = subtreePath ? findNode(node, subtreePath.split('/')) : node; + if (!subtree) { + return []; + } + return getVisibleChildren(subtree).map(child => ({ + title: getTitle(child), + url: `${DOCS_ORIGIN}/${child.path}.md`, + })); } -function buildDocTreeRootIndex(docTree, topLevelPaths) { - const platformsNode = findNode(docTree, ['platforms']); - let platformsSection = ''; - let frameworksSection = ''; - - if (platformsNode) { - const platforms = getVisibleChildren(platformsNode); - platformsSection = `## Platforms\n\n${platforms - .map(p => `- [${getTitle(p)}](${DOCS_ORIGIN}/platforms/${p.slug}.md)`) - .join('\n')}\n`; - - const frameworkLines = []; - for (const platform of platforms) { - const guidesNode = findNode(platform, ['guides']); - if (!guidesNode) { - continue; - } - const guides = getVisibleChildren(guidesNode); - if (guides.length === 0) { - continue; - } - frameworkLines.push(`\n### ${getTitle(platform)}\n`); - for (const guide of guides) { - frameworkLines.push( - `- [${getTitle(guide)}](${DOCS_ORIGIN}/platforms/${platform.slug}/guides/${guide.slug}.md)` - ); - } - } - if (frameworkLines.length > 0) { - frameworksSection = `## Frameworks\n${frameworkLines.join('\n')}\n`; - } +function childLinks(ctx, filter) { + let pages = ctx.children; + if (filter) { + pages = pages.filter(filter); } - - // Build documentation sections from top-level pages (excluding platforms) - const docSections = topLevelPaths - .filter(p => p !== '_not-found.md' && p !== 'platforms.md') + return pages .map(p => { - const title = titleForPath(docTree, p); - return `- [${title}](${DOCS_ORIGIN}/${p})`; + const parts = p.replace(/\.md$/, '').split('/'); + const node = findNode(ctx.docTree, parts); + return { + title: node ? getTitle(node) : parts[parts.length - 1], + url: `${DOCS_ORIGIN}/${p}`, + order: node?.frontmatter?.sidebar_order ?? 99, + }; }) - .join('\n'); - - return `# Sentry Documentation - -Sentry is a developer-first application monitoring platform that helps you identify and fix issues in real-time. It provides error tracking, performance monitoring, session replay, and more across all major platforms and frameworks. - -## Key Features - -- **Error Monitoring**: Capture and diagnose errors with full stack traces, breadcrumbs, and context -- **Tracing**: Track requests across services to identify performance bottlenecks -- **Session Replay**: Watch real user sessions to understand what led to errors -- **Profiling**: Identify slow functions and optimize application performance -- **Crons**: Monitor scheduled jobs and detect failures -- **Logs**: Collect and analyze application logs in context - -${platformsSection} -${frameworksSection} -## Documentation - -${docSections} - -## Quick Links - -- [Platform SDKs](${DOCS_ORIGIN}/platforms.md) - Install Sentry for your language/framework -- [API Reference](${DOCS_ORIGIN}/api.md) - Programmatic access to Sentry -- [CLI](${DOCS_ORIGIN}/cli.md) - Command-line interface for Sentry operations -`; + .sort((a, b) => { + const orderDiff = a.order - b.order; + return orderDiff !== 0 ? orderDiff : a.title.localeCompare(b.title); + }); } -function buildDocTreeDevRootIndex(docTree, topLevelPaths) { - const docSections = topLevelPaths - .filter(p => p !== '_not-found.md') - .map(p => { - const title = titleForPath(docTree, p); - return `- [${title}](${DOCS_ORIGIN}/${p})`; - }) - .join('\n'); - - return `# Sentry Developer Documentation - -${docSections} - -## Quick Links - -- [Backend Development](${DOCS_ORIGIN}/backend.md) - Backend service architecture -- [Frontend Development](${DOCS_ORIGIN}/frontend.md) - Frontend development guide -- [SDK Development](${DOCS_ORIGIN}/sdk.md) - SDK development documentation -`; +function siblingGuideLinks(ctx) { + const platformSlug = ctx.pathParts[1]; + const guideSlug = ctx.pathParts[3]; + const platformNode = findNode(ctx.docTree, ['platforms', platformSlug]); + if (!platformNode) { + return []; + } + const guidesNode = findNode(platformNode, ['guides']); + if (!guidesNode) { + return []; + } + return getVisibleChildren(guidesNode) + .filter(g => g.slug !== guideSlug) + .map(g => ({ + title: getTitle(g), + url: `${DOCS_ORIGIN}/platforms/${platformSlug}/guides/${g.slug}.md`, + })); } -function buildFallbackRootIndex(topLevelPaths) { - return `# Sentry Documentation - -Sentry is a developer-first application monitoring platform that helps you identify and fix issues in real-time. It provides error tracking, performance monitoring, session replay, and more across all major platforms and frameworks. - -## Key Features - -- **Error Monitoring**: Capture and diagnose errors with full stack traces, breadcrumbs, and context -- **Tracing**: Track requests across services to identify performance bottlenecks -- **Session Replay**: Watch real user sessions to understand what led to errors -- **Profiling**: Identify slow functions and optimize application performance -- **Crons**: Monitor scheduled jobs and detect failures -- **Logs**: Collect and analyze application logs in context - -## Documentation Sections - -${topLevelPaths - .filter(p => p !== '_not-found.md') - .map(p => `- [${p.replace(/\.md$/, '')}](${DOCS_ORIGIN}/${p})`) - .join('\n')} - -${ - process.env.NEXT_PUBLIC_DEVELOPER_DOCS - ? `## Quick Links - -- [Backend Development](${DOCS_ORIGIN}/backend.md) - Backend service architecture -- [Frontend Development](${DOCS_ORIGIN}/frontend.md) - Frontend development guide -- [SDK Development](${DOCS_ORIGIN}/sdk.md) - SDK development documentation -` - : `## Quick Links - -- [Platform SDKs](${DOCS_ORIGIN}/platforms.md) - Install Sentry for your language/framework -- [API Reference](${DOCS_ORIGIN}/api.md) - Programmatic access to Sentry -- [CLI](${DOCS_ORIGIN}/cli.md) - Command-line interface for Sentry operations -` -}`; +function platformTitle(ctx) { + const platformNode = findNode(ctx.docTree, ['platforms', ctx.pathParts[1]]); + return platformNode ? getTitle(platformNode) : ctx.pathParts[1]; } -// Registry of full-page content overrides. -// When a page matches, its entire content is replaced (not appended). -// Used for pages like root index.md that need completely custom content. -const contentOverrides = [ - { - match: relativePath => relativePath === 'index.md', - build: (docTree, _existingContent, {allPaths}) => { - const topLevelPaths = allPaths.filter(p => !p.includes('/')); - if (docTree && !process.env.NEXT_PUBLIC_DEVELOPER_DOCS) { - return buildDocTreeRootIndex(docTree, topLevelPaths); - } - if (docTree && process.env.NEXT_PUBLIC_DEVELOPER_DOCS) { - return buildDocTreeDevRootIndex(docTree, topLevelPaths); - } - return buildFallbackRootIndex(topLevelPaths); - }, - }, -]; - -// Registry of custom section builders for specific page patterns. -// Each entry has a `match` function and a `build` function. -// First match wins; unmatched pages fall through to the generic builder. -// These are appended to the existing converted content. -const sectionOverrides = [ - { - match: (_parts, parentPath) => parentPath === 'platforms.md', - build: (docTree, _parentParts, _parentNode, _children) => - buildPlatformsIndexSection(docTree), - }, - { - // Platform index pages (e.g., platforms/javascript.md) - match: (parts, parentPath) => - parentPath.startsWith('platforms/') && parts.length === 2, - build: (docTree, parentParts, parentNode, children) => - buildPlatformPageSection(docTree, parentParts, parentNode, children), - }, - { - // Guide index pages (e.g., platforms/javascript/guides/nextjs.md) - match: (parts, _parentPath) => - parts.length === 4 && parts[0] === 'platforms' && parts[2] === 'guides', - build: (docTree, parentParts, _parentNode, children) => - buildGuidePageSection(docTree, parentParts, _parentNode, children), - }, -]; - -function buildDocTreeChildSection(docTree, parentPath, children) { - const parentParts = parentPath.replace(/\.md$/, '').split('/'); - const parentNode = findNode(docTree, parentParts); - - for (const override of sectionOverrides) { - if (override.match(parentParts, parentPath)) { - return override.build(docTree, parentParts, parentNode, children); +function renderSections(ctx, sections) { + let result = ''; + for (const section of sections) { + const heading = + typeof section.heading === 'function' ? section.heading(ctx) : section.heading; + const items = + typeof section.items === 'function' ? section.items(ctx) : section.items; + if (items.length === 0) { + continue; } + result += `\n## ${heading}\n\n`; + result += items.map(item => `- [${item.title}](${item.url})`).join('\n'); + result += '\n'; } - - // Default: generic section with doctree titles - return buildGenericSection(docTree, parentPath, children); + return result; } -function buildPlatformsIndexSection(docTree) { - const platformsNode = findNode(docTree, ['platforms']); +// Escape hatch for platforms.md which needs complex grouped structure +function buildPlatformsSection(ctx) { + const platformsNode = findNode(ctx.docTree, ['platforms']); if (!platformsNode) { return ''; } @@ -327,111 +208,59 @@ function buildPlatformsIndexSection(docTree) { return section + '\n'; } -function buildPlatformPageSection(docTree, parentParts, parentNode, children) { - const platformSlug = parentParts[1]; - let section = ''; - - // Frameworks (guides) - const guides = children.filter(p => p.includes('/guides/')); - if (guides.length > 0 && parentNode) { - const guidesNode = findNode(parentNode, ['guides']); - if (guidesNode) { - const sortedGuides = getVisibleChildren(guidesNode); - section += `\n## Frameworks\n\n`; - section += sortedGuides - .map( - g => - `- [${getTitle(g)}](${DOCS_ORIGIN}/platforms/${platformSlug}/guides/${g.slug}.md)` - ) - .join('\n'); - section += '\n'; - } else { - section += buildFallbackGuidesList(guides, 'Frameworks'); - } - } - - // Topics (non-guide children) - const topics = children.filter(p => !p.includes('/guides/')); - if (topics.length > 0) { - section += buildSortedTopicsSection(docTree, topics, 'Topics'); - } - - return section; -} +// Declarative page overrides for appended navigation sections. +// First match wins. Unmatched pages with children get default "Pages in this section". +const pageOverrides = [ + { + match: 'platforms.md', + build: ctx => buildPlatformsSection(ctx), + }, + { + // Platform index pages (e.g., platforms/javascript.md) + match: ctx => ctx.pathParts[0] === 'platforms' && ctx.pathParts.length === 2, + sections: [ + {heading: 'Frameworks', items: ctx => nodeLinks(ctx.node, 'guides')}, + {heading: 'Topics', items: ctx => childLinks(ctx, p => !p.includes('/guides/'))}, + ], + }, + { + // Guide index pages (e.g., platforms/javascript/guides/nextjs.md) + match: ctx => + ctx.pathParts.length === 4 && + ctx.pathParts[0] === 'platforms' && + ctx.pathParts[2] === 'guides', + sections: [ + { + heading: ctx => `Other ${platformTitle(ctx)} Frameworks`, + items: ctx => siblingGuideLinks(ctx), + }, + {heading: 'Topics', items: ctx => childLinks(ctx)}, + ], + }, +]; -function buildGuidePageSection(docTree, parentParts, _parentNode, children) { - const platformSlug = parentParts[1]; - const guideSlug = parentParts[3]; - let section = ''; - - // Other frameworks for this platform - const platformNode = findNode(docTree, ['platforms', platformSlug]); - if (platformNode) { - const guidesNode = findNode(platformNode, ['guides']); - if (guidesNode) { - const siblingGuides = getVisibleChildren(guidesNode).filter( - g => g.slug !== guideSlug - ); - if (siblingGuides.length > 0) { - const platformTitle = getTitle(platformNode); - section += `\n## Other ${platformTitle} Frameworks\n\n`; - section += siblingGuides - .map( - g => - `- [${getTitle(g)}](${DOCS_ORIGIN}/platforms/${platformSlug}/guides/${g.slug}.md)` - ) - .join('\n'); - section += '\n'; - } +function buildChildSection(ctx) { + for (const override of pageOverrides) { + const matches = + typeof override.match === 'string' + ? ctx.relativePath === override.match + : override.match(ctx); + if (!matches) { + continue; } + if (override.build) { + return override.build(ctx); + } + if (override.sections) { + return renderSections(ctx, override.sections); + } + return ''; } - // Topics (child pages of this guide) - if (children.length > 0) { - section += buildSortedTopicsSection(docTree, children, 'Topics'); - } - - return section; -} - -function buildSortedTopicsSection(docTree, pages, heading) { - // Sort using doctree order when available - const pagesWithMeta = pages.map(p => { - const parts = p.replace(/\.md$/, '').split('/'); - const node = findNode(docTree, parts); - return { - path: p, - title: node ? getTitle(node) : parts[parts.length - 1], - order: node?.frontmatter?.sidebar_order ?? 99, - }; - }); - - pagesWithMeta.sort((a, b) => { - const orderDiff = a.order - b.order; - return orderDiff !== 0 ? orderDiff : a.title.localeCompare(b.title); - }); - - let section = `\n## ${heading}\n\n`; - section += pagesWithMeta - .map(p => `- [${p.title}](${DOCS_ORIGIN}/${p.path})`) - .join('\n'); - section += '\n'; - return section; -} - -function buildGenericSection(docTree, _parentPath, children) { - // Use doctree titles for all children - return buildSortedTopicsSection(docTree, children, 'Pages in this section'); -} - -function buildFallbackGuidesList(guides, heading) { - const guideList = guides - .map(p => { - const name = p.replace(/\.md$/, '').split('/').pop(); - return `- [${name}](${DOCS_ORIGIN}/${p})`; - }) - .join('\n'); - return `\n## ${heading}\n\n${guideList}\n`; + // Default: list children sorted by sidebar_order + return renderSections(ctx, [ + {heading: 'Pages in this section', items: () => childLinks(ctx)}, + ]); } function buildFallbackChildSection(parentPath, children) { @@ -445,7 +274,13 @@ function buildFallbackChildSection(parentPath, children) { let childSection = ''; if (guides.length > 0) { - childSection += buildFallbackGuidesList(guides, 'Guides'); + const guideList = guides + .map(p => { + const name = p.replace(/\.md$/, '').split('/').pop(); + return `- [${name}](${DOCS_ORIGIN}/${p})`; + }) + .join('\n'); + childSection += `\n## Guides\n\n${guideList}\n`; } if (otherPages.length > 0) { const pageList = otherPages @@ -459,6 +294,159 @@ function buildFallbackChildSection(parentPath, children) { return childSection; } +// --- MDX template rendering for full-page overrides --- + +function buildMdxComponents(docTree, createElement) { + function PlatformList() { + if (!docTree) { + return null; + } + const platformsNode = findNode(docTree, ['platforms']); + if (!platformsNode) { + return null; + } + const platforms = getVisibleChildren(platformsNode); + return createElement( + 'ul', + null, + platforms.map(p => + createElement( + 'li', + {key: p.slug}, + createElement('a', {href: `/platforms/${p.slug}`}, getTitle(p)) + ) + ) + ); + } + + function FrameworkGroups() { + if (!docTree) { + return null; + } + const platformsNode = findNode(docTree, ['platforms']); + if (!platformsNode) { + return null; + } + const platforms = getVisibleChildren(platformsNode); + const groups = []; + for (const platform of platforms) { + const guidesNode = findNode(platform, ['guides']); + if (!guidesNode) { + continue; + } + const guides = getVisibleChildren(guidesNode); + if (guides.length === 0) { + continue; + } + groups.push( + createElement('h3', {key: `h-${platform.slug}`}, getTitle(platform)), + createElement( + 'ul', + {key: `ul-${platform.slug}`}, + guides.map(g => + createElement( + 'li', + {key: g.slug}, + createElement( + 'a', + {href: `/platforms/${platform.slug}/guides/${g.slug}`}, + getTitle(g) + ) + ) + ) + ) + ); + } + return createElement('div', null, ...groups); + } + + function DocSectionList({exclude = []}) { + if (!docTree) { + return null; + } + const sections = getVisibleChildren(docTree).filter( + child => !exclude.includes(child.slug) + ); + return createElement( + 'ul', + null, + sections.map(child => + createElement( + 'li', + {key: child.slug}, + createElement('a', {href: `/${child.slug}`}, getTitle(child)) + ) + ) + ); + } + + return {PlatformList, FrameworkGroups, DocSectionList}; +} + +async function renderMdxOverrides(root, docTree) { + const overrideDir = process.env.NEXT_PUBLIC_DEVELOPER_DOCS + ? path.join(root, 'md-overrides', 'dev') + : path.join(root, 'md-overrides'); + + const overrides = new Map(); + + if (!existsSync(overrideDir)) { + return overrides; + } + + const tempDir = path.join(root, '.next', 'cache', 'md-override-html'); + await rm(tempDir, {recursive: true, force: true}); + await mkdir(tempDir, {recursive: true}); + + const {evaluate} = await import('@mdx-js/mdx'); + const jsxRuntime = await import('react/jsx-runtime'); + const React = await import('react'); + const {renderToStaticMarkup} = await import('react-dom/server'); + const grayMatter = (await import('gray-matter')).default; + + const components = buildMdxComponents(docTree, React.createElement); + const files = await readdir(overrideDir, {recursive: true}); + + for (const file of files) { + if (!file.endsWith('.mdx')) { + continue; + } + + const mdxSource = await readFile(path.join(overrideDir, file), {encoding: 'utf8'}); + const {data: frontmatter, content} = grayMatter(mdxSource); + + const {default: MDXContent} = await evaluate(content, { + jsx: jsxRuntime.jsx, + jsxs: jsxRuntime.jsxs, + Fragment: jsxRuntime.Fragment, + }); + + const bodyHtml = renderToStaticMarkup(React.createElement(MDXContent, {components})); + + const relativePath = file.replace(/\.mdx$/, '.md'); + const urlPath = file.replace(/\.mdx$/, '').replace(/^index$/, ''); + const canonicalUrl = `${DOCS_ORIGIN}/${urlPath}`; + + const html = [ + '<!DOCTYPE html><html><head>', + `<title>${frontmatter.title || ''}`, + ``, + '', + `
${bodyHtml}
`, + '', + ].join('\n'); + + const htmlPath = path.join(tempDir, file.replace(/\.mdx$/, '.html')); + await mkdir(path.dirname(htmlPath), {recursive: true}); + await writeFile(htmlPath, html, {encoding: 'utf8'}); + + overrides.set(relativePath, {htmlPath, frontmatter}); + console.log(`📝 Rendered MDX override: ${file} → ${relativePath}`); + } + + return overrides; +} + // Global set to track which cache files are used across all workers let globalUsedCacheFiles = null; @@ -511,6 +499,9 @@ async function createWork() { console.warn(' Falling back to slug-based navigation'); } + // Render MDX template overrides (full-page content replacements) + const mdxOverrides = await renderMdxOverrides(root, docTree); + // Clear output directory await rm(OUTPUT_DIR, {recursive: true, force: true}); await mkdir(OUTPUT_DIR, {recursive: true}); @@ -575,8 +566,10 @@ async function createWork() { await mkdir(targetDir, {recursive: true}); const targetPath = path.join(targetDir, dirent.name.slice(0, -5) + '.md'); const relativePath = path.relative(OUTPUT_DIR, targetPath); + // Use MDX override HTML if available, otherwise use Next.js build HTML + const mdxOverride = mdxOverrides.get(relativePath); workerTasks[workerIdx].push({ - sourcePath, + sourcePath: mdxOverride ? mdxOverride.htmlPath : sourcePath, targetPath, relativePath, r2Hash: existingFilesOnR2 ? existingFilesOnR2.get(relativePath) : null, @@ -629,28 +622,12 @@ async function createWork() { await Promise.all(workerPromises); - // Generate hierarchical sitemaps + // Collect all generated paths const allPaths = workerTasks .flat() .map(task => task.relativePath) - .filter(p => p !== 'index.md') .sort(); - // Apply full-page content overrides (e.g., root index.md) - // These replace the worker-generated content entirely. - const overrideContext = {allPaths}; - const contentOverrideResults = new Map(); - for (const override of contentOverrides) { - for (const task of workerTasks.flat()) { - if (override.match(task.relativePath)) { - const content = override.build(docTree, null, overrideContext); - await writeFile(task.targetPath, content, {encoding: 'utf8'}); - contentOverrideResults.set(task.relativePath, content); - console.log(`📑 Generated ${task.relativePath} (content override)`); - } - } - } - // Append child page listings to section index pages // Group paths by their DIRECT parent (handling guides specially) const pathsByParent = new Map(); @@ -684,10 +661,16 @@ async function createWork() { pathsByParent.get(parentPath).push(p); } - // Append child listings to parent index files + // Append child listings to parent index files. + // Skip pages whose MDX override sets append_sections: false. let updatedCount = 0; const r2Uploads = []; for (const [parentPath, children] of pathsByParent) { + const overrideFm = mdxOverrides.get(parentPath)?.frontmatter; + if (overrideFm?.append_sections === false) { + continue; + } + const parentFile = path.join(OUTPUT_DIR, parentPath); let existingContent; try { @@ -699,8 +682,18 @@ async function createWork() { throw err; } + const pathParts = parentPath.replace(/\.md$/, '').split('/'); + const node = docTree ? findNode(docTree, pathParts) : null; + const ctx = { + docTree, + relativePath: parentPath, + pathParts, + node, + children, + }; + const childSection = docTree - ? buildDocTreeChildSection(docTree, parentPath, children) + ? buildChildSection(ctx) : buildFallbackChildSection(parentPath, children); if (childSection) { @@ -732,19 +725,6 @@ async function createWork() { } console.log(`📑 Added child page listings to ${updatedCount} section index files`); - // Upload content-overridden pages to R2 if configured - if (accessKeyId && secretAccessKey && contentOverrideResults.size > 0) { - const s3Client = getS3Client(); - for (const [relativePath, content] of contentOverrideResults) { - const fileHash = md5(content); - const existingHash = existingFilesOnR2?.get(relativePath); - if (existingHash !== fileHash) { - await uploadToCFR2(s3Client, relativePath, content); - console.log(`📤 Uploaded updated ${relativePath} to R2`); - } - } - } - // Clean up unused cache files to prevent unbounded growth if (!noCache) { try { diff --git a/specs/llm-friendly-docs.md b/specs/llm-friendly-docs.md index b125b5dcccdfb..edaddfcf4b1a9 100644 --- a/specs/llm-friendly-docs.md +++ b/specs/llm-friendly-docs.md @@ -31,53 +31,93 @@ Markdown exports are not just raw dumps of HTML content. They are adapted for LL ## Page Customization Architecture -The generation script has two override registries for customizing output on a per-page basis: +The generation script has two layers of page customization, each suited to its use case: -### Content Overrides (full page replacement) +### Layer 1: MDX Template Overrides (full page replacement) -For pages that need completely custom content (e.g., root `index.md`), register a `contentOverride`. The override replaces the entire worker-generated Markdown with custom content. +For pages that need completely custom **content** (like root `index.md`), put an `.mdx` file in `md-overrides/`. These are authored like normal docs content but with custom components for dynamic data. -```javascript -const contentOverrides = [ - { - match: (relativePath) => relativePath === 'index.md', - build: (docTree, existingContent, context) => { - // Return full page content as a string - return `# Sentry Documentation\n...`; - }, - }, -]; +**Pipeline**: `MDX → React SSR → HTML → wrap in shell → existing HTML→MD worker pipeline` + +The rendered HTML gets wrapped in a minimal `` + `<link rel="canonical">` + `<div id="main">` structure that the existing pipeline expects. Workers process it through the same unified/rehype conversion as any other page — link rewriting, caching, R2 sync all work automatically. + +**Directory structure:** +- `md-overrides/*.mdx` — overrides for `docs.sentry.io` pages +- `md-overrides/dev/*.mdx` — overrides for `develop.sentry.dev` pages + +**Example** (`md-overrides/index.mdx`): +```mdx +--- +title: "Sentry Documentation" +append_sections: false +--- + +Sentry is a developer-first application monitoring platform... + +## Platforms + +<PlatformList /> + +## Frameworks + +<FrameworkGroups /> + +## Documentation + +<DocSectionList exclude={["platforms", "_not-found"]} /> + +## Quick Links + +- [Platform SDKs](/platforms) - Install Sentry for your language/framework ``` -**Parameters:** -- `match(relativePath)` — return `true` if this override applies to the given path -- `build(docTree, existingContent, context)` — return the full page content - - `docTree` — parsed `doctree.json` (or `null` if unavailable) - - `existingContent` — the worker-generated Markdown (can be used as base) - - `context.allPaths` — list of all generated `.md` paths +**Frontmatter fields:** +- `title` — becomes `<title>` in the HTML shell → H1 in the converted markdown +- `append_sections` — `false` to skip auto-appended navigation sections (default: `true`). Set to `false` when the template already includes all navigation via components. -### Section Overrides (appended navigation) +The canonical URL is derived from the file path (`md-overrides/platforms.mdx` → `https://docs.sentry.io/platforms`). -For pages that need custom navigation sections appended to the converted content, register a `sectionOverride`. These are appended after the main content. +**Available components** (close over `docTree`): +- `PlatformList` — renders `<ul>` of all visible platforms +- `FrameworkGroups` — renders platforms with nested guide lists +- `DocSectionList` — renders top-level doc sections (with `exclude` prop) + +### Layer 2: Declarative Section Overrides (appended navigation) + +For pages that need custom **navigation sections** appended after their converted content, use the `pageOverrides` table with declarative `sections` arrays. ```javascript -const sectionOverrides = [ +const pageOverrides = [ + { + match: 'platforms.md', + build: ctx => buildPlatformsSection(ctx), // complex grouping needs custom build + }, { - match: (pathParts, parentPath) => parentPath === 'platforms.md', - build: (docTree, parentParts, parentNode, children) => { - // Return markdown string to append - return '\n## Platforms\n\n- [JavaScript](...)\n'; - }, + match: ctx => ctx.pathParts[0] === 'platforms' && ctx.pathParts.length === 2, + sections: [ + { heading: 'Frameworks', items: ctx => nodeLinks(ctx.node, 'guides') }, + { heading: 'Topics', items: ctx => childLinks(ctx, p => !p.includes('/guides/')) }, + ], }, ]; ``` -**Parameters:** -- `match(pathParts, parentPath)` — path segments array and full parent path (e.g., `['platforms', 'javascript']`, `'platforms/javascript.md'`) -- `build(docTree, parentParts, parentNode, children)` — return the section to append - - `parentParts` — path segments for the parent page - - `parentNode` — doctree node for the parent (if found) - - `children` — list of child `.md` paths that belong to this parent +**Override fields:** +- `match` — string for exact path match, or function receiving the context +- `sections` — array of `{heading, items}` for declarative sections (preferred) +- `build` — function returning markdown string (escape hatch for complex cases) + +**Shared utilities:** +- `nodeLinks(node, subtreePath)` — links to visible children of a doctree subtree +- `childLinks(ctx, filter?)` — links from `ctx.children` with optional filter, sorted by `sidebar_order` +- `siblingGuideLinks(ctx)` — links to sibling guides for the same platform +- `renderSections(ctx, sections)` — renders section arrays to markdown +- `platformTitle(ctx)` — human-readable platform title from doctree + +**Unified context object** passed to all match/build/section functions: +```javascript +{ docTree, relativePath, pathParts, node, children } +``` ### Default Behavior @@ -88,11 +128,12 @@ Pages without a matching override get a generic "Pages in this section" listing ## Current Override Registry -### Content Overrides +### MDX Template Overrides -| Pattern | Description | -|---------|-------------| -| `index.md` | Root documentation index with platforms, frameworks, and doc sections | +| File | Output | Description | +|------|--------|-------------| +| `md-overrides/index.mdx` | `index.md` | Root docs index with platforms, frameworks, and doc sections | +| `md-overrides/dev/index.mdx` | `index.md` (dev) | Root developer docs index | ### Section Overrides @@ -121,11 +162,27 @@ yarn next build → .next/server/app/**/*.html yarn generate-md-exports → public/md-exports/**/*.md (+ R2 sync) ``` +During `generate-md-exports`: +1. Load doctree +2. Render MDX templates from `md-overrides/` → HTML in `.next/cache/md-override-html/` +3. Discover HTML files, swapping source path for MDX override pages +4. Workers convert HTML → Markdown (parallel, cached, with R2 sync) +5. Append navigation sections to parent pages using `pageOverrides` + ## Adding a New Override -1. Write a builder function in `scripts/generate-md-exports.mjs` -2. Add an entry to `contentOverrides` (full replacement) or `sectionOverrides` (appended section) -3. The match function determines which pages use the override +### MDX Template Override (full page replacement) + +1. Create an `.mdx` file in `md-overrides/` (or `md-overrides/dev/` for developer docs) +2. Add frontmatter with `title` and `append_sections: false` if the template handles its own navigation +3. Use available components (`PlatformList`, `FrameworkGroups`, `DocSectionList`) or add new ones in `buildMdxComponents()` +4. Build and verify: check the output in `public/md-exports/` + +### Section Override (appended navigation) + +1. Add an entry to the `pageOverrides` array in `scripts/generate-md-exports.mjs` +2. Use `sections` array for declarative overrides, or `build` function for complex cases +3. The `match` field determines which pages use the override (string or function) 4. Build and verify: check the output in `public/md-exports/` ## Verification From c2b2eb36ed71295a57bcc76ef7c33abaedd6a7cb Mon Sep 17 00:00:00 2001 From: David Cramer <dcramer@gmail.com> Date: Tue, 17 Feb 2026 13:04:25 -0800 Subject: [PATCH 4/9] fix: prevent dev MDX overrides from leaking into production builds Use non-recursive readdir for md-overrides/ so that dev/ subdirectory files are not discovered during production builds. Co-Authored-By: Claude <noreply@anthropic.com> --- scripts/generate-md-exports.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index 49d7509440e8e..db24e7b817268 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -405,7 +405,9 @@ async function renderMdxOverrides(root, docTree) { const grayMatter = (await import('gray-matter')).default; const components = buildMdxComponents(docTree, React.createElement); - const files = await readdir(overrideDir, {recursive: true}); + // Non-recursive: only read .mdx files directly in overrideDir. + // This prevents dev/ overrides from leaking into production builds. + const files = await readdir(overrideDir); for (const file of files) { if (!file.endsWith('.mdx')) { From 081e2a90a05d7adb8b9286dd294c22ac4d59ed16 Mon Sep 17 00:00:00 2001 From: David Cramer <dcramer@gmail.com> Date: Tue, 17 Feb 2026 13:21:21 -0800 Subject: [PATCH 5/9] fix: restructure index.md as sitemap, move curated content to platforms.md - index.md now shows full sitemap: platforms, frameworks, and every top-level section with its children expanded (via SectionTree component) - platforms.md gets MDX override with curated platforms/frameworks listing, doc sections, and quick links - Add SectionTree component that renders doctree sections with children Co-Authored-By: Claude <noreply@anthropic.com> --- md-overrides/index.mdx | 4 +--- md-overrides/platforms.mdx | 24 +++++++++++++++++++++++ scripts/generate-md-exports.mjs | 34 ++++++++++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 md-overrides/platforms.mdx diff --git a/md-overrides/index.mdx b/md-overrides/index.mdx index 9bfcef9cb85ac..0062471e2dd45 100644 --- a/md-overrides/index.mdx +++ b/md-overrides/index.mdx @@ -22,9 +22,7 @@ Sentry is a developer-first application monitoring platform that helps you ident <FrameworkGroups /> -## Documentation - -<DocSectionList exclude={["platforms", "_not-found"]} /> +<SectionTree exclude={["platforms", "_not-found"]} /> ## Quick Links diff --git a/md-overrides/platforms.mdx b/md-overrides/platforms.mdx new file mode 100644 index 0000000000000..97e3dd154197c --- /dev/null +++ b/md-overrides/platforms.mdx @@ -0,0 +1,24 @@ +--- +title: "Platforms" +append_sections: false +--- + +Sentry provides official SDKs for all major platforms and frameworks. Choose your platform to get started with error monitoring, performance tracking, and more. + +## Platforms + +<PlatformList /> + +## Frameworks + +<FrameworkGroups /> + +## Documentation + +<DocSectionList exclude={["platforms", "_not-found"]} /> + +## Quick Links + +- [Platform SDKs](/platforms) - Install Sentry for your language/framework +- [API Reference](/api) - Programmatic access to Sentry +- [CLI](/cli) - Command-line interface for Sentry operations diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index db24e7b817268..3516e1ff456c5 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -380,7 +380,39 @@ function buildMdxComponents(docTree, createElement) { ); } - return {PlatformList, FrameworkGroups, DocSectionList}; + // Renders every top-level section with its visible children as a nested list. + // Used for the root index.md sitemap. + function SectionTree({exclude = []}) { + if (!docTree) { + return null; + } + const sections = getVisibleChildren(docTree).filter( + child => !exclude.includes(child.slug) + ); + const elements = []; + for (const section of sections) { + elements.push(createElement('h2', {key: `h-${section.slug}`}, getTitle(section))); + const children = getVisibleChildren(section); + if (children.length > 0) { + elements.push( + createElement( + 'ul', + {key: `ul-${section.slug}`}, + children.map(child => + createElement( + 'li', + {key: child.slug}, + createElement('a', {href: `/${child.path}`}, getTitle(child)) + ) + ) + ) + ); + } + } + return createElement('div', null, ...elements); + } + + return {PlatformList, FrameworkGroups, DocSectionList, SectionTree}; } async function renderMdxOverrides(root, docTree) { From 61fa729610387649603490bcd6fd38c790e027c0 Mon Sep 17 00:00:00 2001 From: David Cramer <dcramer@gmail.com> Date: Tue, 17 Feb 2026 13:41:38 -0800 Subject: [PATCH 6/9] fix: remove duplicate Platforms heading in platforms.md The H1 already comes from the frontmatter title, so the H2 was redundant. Co-Authored-By: Claude <noreply@anthropic.com> --- md-overrides/platforms.mdx | 2 -- 1 file changed, 2 deletions(-) diff --git a/md-overrides/platforms.mdx b/md-overrides/platforms.mdx index 97e3dd154197c..92e9a2224acf3 100644 --- a/md-overrides/platforms.mdx +++ b/md-overrides/platforms.mdx @@ -5,8 +5,6 @@ append_sections: false Sentry provides official SDKs for all major platforms and frameworks. Choose your platform to get started with error monitoring, performance tracking, and more. -## Platforms - <PlatformList /> ## Frameworks From dbd34dea3877bcb72e2cb7e01793d40e11840c14 Mon Sep 17 00:00:00 2001 From: David Cramer <dcramer@gmail.com> Date: Tue, 17 Feb 2026 14:00:42 -0800 Subject: [PATCH 7/9] fix: remove self-referential link in platforms.md quick links Replace "/platforms" link (which points to itself) with a link back to the root documentation index. Co-Authored-By: Claude <noreply@anthropic.com> --- md-overrides/platforms.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/md-overrides/platforms.mdx b/md-overrides/platforms.mdx index 92e9a2224acf3..130f26d6357e9 100644 --- a/md-overrides/platforms.mdx +++ b/md-overrides/platforms.mdx @@ -17,6 +17,6 @@ Sentry provides official SDKs for all major platforms and frameworks. Choose you ## Quick Links -- [Platform SDKs](/platforms) - Install Sentry for your language/framework +- [Sentry Documentation](/) - Main documentation index - [API Reference](/api) - Programmatic access to Sentry - [CLI](/cli) - Command-line interface for Sentry operations From 17bbbcfc57d5ff7dbdfef5da8f6fd4638b23ae1d Mon Sep 17 00:00:00 2001 From: David Cramer <dcramer@gmail.com> Date: Tue, 17 Feb 2026 14:24:16 -0800 Subject: [PATCH 8/9] fix: warn when MDX overrides don't match any HTML file Prevents silent failures when an override file has a typo or its corresponding page doesn't exist in the build output. Co-Authored-By: Claude <noreply@anthropic.com> --- scripts/generate-md-exports.mjs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index 3516e1ff456c5..cd4eeecbb034b 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -613,6 +613,16 @@ async function createWork() { } } + // Warn about MDX overrides that didn't match any HTML file + const usedOverrides = new Set( + workerTasks.flat().filter(t => mdxOverrides.has(t.relativePath)).map(t => t.relativePath) + ); + for (const [key] of mdxOverrides) { + if (!usedOverrides.has(key)) { + console.warn(`⚠️ MDX override "${key}" did not match any HTML file and will be ignored`); + } + } + console.log(`📄 Converting ${numFiles} files with ${numWorkers} workers...`); const selfPath = fileURLToPath(import.meta.url); From f1830d2ed3aa3d33fbf7c6c05fd64743c9468c16 Mon Sep 17 00:00:00 2001 From: "getsantry[bot]" <66042841+getsantry[bot]@users.noreply.github.com> Date: Tue, 17 Feb 2026 22:25:09 +0000 Subject: [PATCH 9/9] [getsentry/action-github-commit] Auto commit --- scripts/generate-md-exports.mjs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/generate-md-exports.mjs b/scripts/generate-md-exports.mjs index cd4eeecbb034b..42400b9cd00be 100644 --- a/scripts/generate-md-exports.mjs +++ b/scripts/generate-md-exports.mjs @@ -615,11 +615,16 @@ async function createWork() { // Warn about MDX overrides that didn't match any HTML file const usedOverrides = new Set( - workerTasks.flat().filter(t => mdxOverrides.has(t.relativePath)).map(t => t.relativePath) + workerTasks + .flat() + .filter(t => mdxOverrides.has(t.relativePath)) + .map(t => t.relativePath) ); for (const [key] of mdxOverrides) { if (!usedOverrides.has(key)) { - console.warn(`⚠️ MDX override "${key}" did not match any HTML file and will be ignored`); + console.warn( + `⚠️ MDX override "${key}" did not match any HTML file and will be ignored` + ); } }