Skip to content

🤖 fix: disable incomplete markdown parsing for completed content #909

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ammario merged 1 commit into from
Dec 4, 2025
Merged

🤖 fix: disable incomplete markdown parsing for completed content #909

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions bun.lock
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
"rehype-harden": "^1.1.5",
"shescape": "^2.1.6",
"source-map-support": "^0.5.21",
"streamdown": "^1.4.0",
"streamdown": "1.6.10",
"trpc-cli": "^0.12.1",
"turndown": "^7.2.2",
"undici": "^7.16.0",
Expand Down Expand Up @@ -3143,6 +3143,8 @@

"remark-stringify": ["remark-stringify@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-to-markdown": "^2.0.0", "unified": "^11.0.0" } }, "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw=="],

"remend": ["remend@1.0.1", "", {}, "sha512-152puVH0qMoRJQFnaMG+rVDdf01Jq/CaED+MBuXExurJgdbkLp0c3TIe4R12o28Klx8uyGsjvFNG05aFG69G9w=="],

"require-directory": ["require-directory@2.1.1", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="],

"require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="],
Expand Down Expand Up @@ -3295,7 +3297,7 @@

"storybook": ["storybook@10.1.4", "", { "dependencies": { "@storybook/global": "^5.0.0", "@storybook/icons": "^2.0.0", "@testing-library/jest-dom": "^6.6.3", "@testing-library/user-event": "^14.6.1", "@vitest/expect": "3.2.4", "@vitest/spy": "3.2.4", "esbuild": "^0.18.0 || ^0.19.0 || ^0.20.0 || ^0.21.0 || ^0.22.0 || ^0.23.0 || ^0.24.0 || ^0.25.0 || ^0.26.0 || ^0.27.0", "recast": "^0.23.5", "semver": "^7.6.2", "use-sync-external-store": "^1.5.0", "ws": "^8.18.0" }, "peerDependencies": { "prettier": "^2 || ^3" }, "optionalPeers": ["prettier"], "bin": "./dist/bin/dispatcher.js" }, "sha512-FrBjm8I8O+pYEOPHcdW9xWwgXSZxte7lza9q2lN3jFN4vuW79m5j0OnTQeR8z9MmIbBTvkIpp3yMBebl53Yt5Q=="],

"streamdown": ["streamdown@1.6.9", "", { "dependencies": { "clsx": "^2.1.1", "hast": "^1.0.0", "hast-util-to-jsx-runtime": "^2.3.6", "html-url-attributes": "^3.0.1", "katex": "^0.16.22", "lucide-react": "^0.542.0", "marked": "^16.2.1", "mermaid": "^11.11.0", "rehype-harden": "^1.1.6", "rehype-katex": "^7.0.1", "rehype-raw": "^7.0.0", "remark-cjk-friendly": "^1.2.3", "remark-cjk-friendly-gfm-strikethrough": "^1.2.3", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.1.2", "shiki": "^3.12.2", "tailwind-merge": "^3.3.1", "unified": "^11.0.5", "unist-util-visit": "^5.0.0" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0" } }, "sha512-rtUZcRvDYNEgduq1OxNJzuYYmchZVXq+1Pw3T445RrYwrT+SGNK1drtt1eaqC4HaD8YYIscdtMSlZFaNM+yYGA=="],
"streamdown": ["streamdown@1.6.10", "", { "dependencies": { "clsx": "^2.1.1", "hast": "^1.0.0", "hast-util-to-jsx-runtime": "^2.3.6", "html-url-attributes": "^3.0.1", "katex": "^0.16.22", "lucide-react": "^0.542.0", "marked": "^16.2.1", "mermaid": "^11.11.0", "rehype-harden": "^1.1.6", "rehype-katex": "^7.0.1", "rehype-raw": "^7.0.0", "remark-cjk-friendly": "^1.2.3", "remark-cjk-friendly-gfm-strikethrough": "^1.2.3", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.1.2", "remend": "1.0.1", "shiki": "^3.12.2", "tailwind-merge": "^3.3.1", "unified": "^11.0.5", "unist-util-visit": "^5.0.0" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0" } }, "sha512-B4Y3Z/qiXl1Dc+LzAB5c52Cd1QGRiFjaDwP+ERoj1JtCykdRDM8X6HwQnn3YkpkSk0x3R7S/6LrGe1nQiElHQQ=="],

"string-length": ["string-length@6.0.0", "", { "dependencies": { "strip-ansi": "^7.1.0" } }, "sha512-1U361pxZHEQ+FeSjzqRpV+cu2vTzYeWeafXFLykiFlv4Vc0n3njgU8HrMbyik5uwm77naWMuVG8fhEF+Ovb1Kg=="],

Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@
"rehype-harden": "^1.1.5",
"shescape": "^2.1.6",
"source-map-support": "^0.5.21",
"streamdown": "^1.4.0",
"streamdown": "1.6.10",
"trpc-cli": "^0.12.1",
"turndown": "^7.2.2",
"undici": "^7.16.0",
Expand Down
48 changes: 29 additions & 19 deletions src/browser/components/Messages/MarkdownCore.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,14 @@ import { markdownComponents } from "./MarkdownComponents";
interface MarkdownCoreProps {
content: string;
children?: React.ReactNode; // For cursor or other additions
/**
* Enable incomplete markdown parsing for streaming content.
* When true, the remend library will attempt to "repair" unclosed markdown
* syntax (e.g., adding closing ** for bold). This is useful during streaming
* but can cause bugs with content like $__variable (adds trailing __).
* Default: false for completed content, true during streaming.
*/
parseIncompleteMarkdown?: boolean;
}

// Plugin arrays are defined at module scope to maintain stable references.
Expand Down Expand Up @@ -42,25 +50,27 @@ const REHYPE_PLUGINS: Pluggable[] = [
*
* Memoized to prevent expensive re-parsing when content hasn't changed.
*/
export const MarkdownCore = React.memo<MarkdownCoreProps>(({ content, children }) => {
// Memoize the normalized content to avoid recalculating on every render
const normalizedContent = useMemo(() => normalizeMarkdown(content), [content]);
export const MarkdownCore = React.memo<MarkdownCoreProps>(
({ content, children, parseIncompleteMarkdown = false }) => {
// Memoize the normalized content to avoid recalculating on every render
const normalizedContent = useMemo(() => normalizeMarkdown(content), [content]);

return (
<>
<Streamdown
components={markdownComponents}
remarkPlugins={REMARK_PLUGINS}
rehypePlugins={REHYPE_PLUGINS}
parseIncompleteMarkdown={true}
className="space-y-2" // Reduce from default space-y-4 (16px) to space-y-2 (8px)
controls={{ table: false, code: true, mermaid: true }} // Disable table copy/download, keep code/mermaid controls
>
{normalizedContent}
</Streamdown>
{children}
</>
);
});
return (
<>
<Streamdown
components={markdownComponents}
remarkPlugins={REMARK_PLUGINS}
rehypePlugins={REHYPE_PLUGINS}
parseIncompleteMarkdown={parseIncompleteMarkdown}
className="space-y-2" // Reduce from default space-y-4 (16px) to space-y-2 (8px)
controls={{ table: false, code: true, mermaid: true }} // Disable table copy/download, keep code/mermaid controls
>
{normalizedContent}
</Streamdown>
{children}
</>
);
}
);

MarkdownCore.displayName = "MarkdownCore";
2 changes: 1 addition & 1 deletion src/browser/components/Messages/TypewriterMarkdown.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ export const TypewriterMarkdown = React.memo<TypewriterMarkdownProps>(function T
return (
<StreamingContext.Provider value={streamingContextValue}>
<div className={cn("markdown-content", className)}>
<MarkdownCore content={content} />
<MarkdownCore content={content} parseIncompleteMarkdown={isStreaming} />
</div>
</StreamingContext.Provider>
);
Expand Down
39 changes: 39 additions & 0 deletions src/browser/stories/App.markdown.stories.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,22 @@ const TABLE_CONTENT = `Here are various markdown table examples:
| \`api.timeout\` | 30000 | Timeout ms | \`API_TIMEOUT\` |
| \`cache.enabled\` | true | Enable cache | \`CACHE_ENABLED\` |`;

// Bug repro: SQL with $__timeFilter causes "__" to appear at end of code block
const SQL_WITH_DOUBLE_UNDERSCORE = `👍 Glad it's working. For reference, the final query:

\`\`\`sql
SELECT
TIMESTAMP_TRUNC(timestamp, DAY) as time,
COUNT(DISTINCT distinct_id) as dau
FROM \`mux-telemetry.posthog.events\`
WHERE
event NOT LIKE "$%"
AND $__timeFilter(timestamp)
GROUP BY time
ORDER BY time
\`\`\`
`;

const CODE_CONTENT = `Here's the implementation:

\`\`\`typescript
Expand Down Expand Up @@ -100,6 +116,29 @@ export const Tables: AppStory = {
),
};

/** SQL with double underscores in code block - tests for bug where __ leaks to end */
export const SqlWithDoubleUnderscore: AppStory = {
render: () => (
<AppWithMocks
setup={() =>
setupSimpleChatStory({
workspaceId: "ws-sql-underscore",
messages: [
createUserMessage("msg-1", "Show me the SQL query", {
historySequence: 1,
timestamp: STABLE_TIMESTAMP - 100000,
}),
createAssistantMessage("msg-2", SQL_WITH_DOUBLE_UNDERSCORE, {
historySequence: 2,
timestamp: STABLE_TIMESTAMP - 90000,
}),
],
})
}
/>
),
};

/** Code blocks with syntax highlighting */
export const CodeBlocks: AppStory = {
render: () => (
Expand Down