From 3d28867dc50cb3516fe29b2e7370d7fbe53dc234 Mon Sep 17 00:00:00 2001 From: Ammar Date: Thu, 4 Dec 2025 12:37:21 -0600 Subject: [PATCH] =?UTF-8?q?=F0=9F=A4=96=20fix:=20disable=20incomplete=20ma?= =?UTF-8?q?rkdown=20parsing=20for=20completed=20content?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The remend library (used by streamdown for 'parseIncompleteMarkdown') has a bug where content like $__timeFilter causes trailing '__' to be added to the output. This happens because remend counts all '__' in the entire string without respecting code block boundaries. Fix: Only enable parseIncompleteMarkdown during actual streaming, not for completed/historical content. This prevents the bug while still providing the streaming UX benefit. Root cause: remend@1.0.1 counts markdown emphasis markers (__) globally rather than excluding fenced code blocks, causing false 'repairs'. _Generated with `mux`_ --- bun.lock | 6 ++- package.json | 2 +- .../components/Messages/MarkdownCore.tsx | 48 +++++++++++-------- .../Messages/TypewriterMarkdown.tsx | 2 +- src/browser/stories/App.markdown.stories.tsx | 39 +++++++++++++++ 5 files changed, 74 insertions(+), 23 deletions(-) diff --git a/bun.lock b/bun.lock index 7e1786ecaf..6b1a19798f 100644 --- a/bun.lock +++ b/bun.lock @@ -53,7 +53,7 @@ "rehype-harden": "^1.1.5", "shescape": "^2.1.6", "source-map-support": "^0.5.21", - "streamdown": "^1.4.0", + "streamdown": "1.6.10", "trpc-cli": "^0.12.1", "turndown": "^7.2.2", "undici": "^7.16.0", @@ -3143,6 +3143,8 @@ "remark-stringify": ["remark-stringify@11.0.0", "", { "dependencies": { "@types/mdast": "^4.0.0", "mdast-util-to-markdown": "^2.0.0", "unified": "^11.0.0" } }, "sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw=="], + "remend": ["remend@1.0.1", "", {}, "sha512-152puVH0qMoRJQFnaMG+rVDdf01Jq/CaED+MBuXExurJgdbkLp0c3TIe4R12o28Klx8uyGsjvFNG05aFG69G9w=="], + "require-directory": ["require-directory@2.1.1", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="], "require-from-string": ["require-from-string@2.0.2", "", {}, "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw=="], @@ -3295,7 +3297,7 @@ "storybook": ["storybook@10.1.4", "", { "dependencies": { "@storybook/global": "^5.0.0", "@storybook/icons": "^2.0.0", "@testing-library/jest-dom": "^6.6.3", "@testing-library/user-event": "^14.6.1", "@vitest/expect": "3.2.4", "@vitest/spy": "3.2.4", "esbuild": "^0.18.0 || ^0.19.0 || ^0.20.0 || ^0.21.0 || ^0.22.0 || ^0.23.0 || ^0.24.0 || ^0.25.0 || ^0.26.0 || ^0.27.0", "recast": "^0.23.5", "semver": "^7.6.2", "use-sync-external-store": "^1.5.0", "ws": "^8.18.0" }, "peerDependencies": { "prettier": "^2 || ^3" }, "optionalPeers": ["prettier"], "bin": "./dist/bin/dispatcher.js" }, "sha512-FrBjm8I8O+pYEOPHcdW9xWwgXSZxte7lza9q2lN3jFN4vuW79m5j0OnTQeR8z9MmIbBTvkIpp3yMBebl53Yt5Q=="], - "streamdown": ["streamdown@1.6.9", "", { "dependencies": { "clsx": "^2.1.1", "hast": "^1.0.0", "hast-util-to-jsx-runtime": "^2.3.6", "html-url-attributes": "^3.0.1", "katex": "^0.16.22", "lucide-react": "^0.542.0", "marked": "^16.2.1", "mermaid": "^11.11.0", "rehype-harden": "^1.1.6", "rehype-katex": "^7.0.1", "rehype-raw": "^7.0.0", "remark-cjk-friendly": "^1.2.3", "remark-cjk-friendly-gfm-strikethrough": "^1.2.3", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.1.2", "shiki": "^3.12.2", "tailwind-merge": "^3.3.1", "unified": "^11.0.5", "unist-util-visit": "^5.0.0" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0" } }, "sha512-rtUZcRvDYNEgduq1OxNJzuYYmchZVXq+1Pw3T445RrYwrT+SGNK1drtt1eaqC4HaD8YYIscdtMSlZFaNM+yYGA=="], + "streamdown": ["streamdown@1.6.10", "", { "dependencies": { "clsx": "^2.1.1", "hast": "^1.0.0", "hast-util-to-jsx-runtime": "^2.3.6", "html-url-attributes": "^3.0.1", "katex": "^0.16.22", "lucide-react": "^0.542.0", "marked": "^16.2.1", "mermaid": "^11.11.0", "rehype-harden": "^1.1.6", "rehype-katex": "^7.0.1", "rehype-raw": "^7.0.0", "remark-cjk-friendly": "^1.2.3", "remark-cjk-friendly-gfm-strikethrough": "^1.2.3", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", "remark-parse": "^11.0.0", "remark-rehype": "^11.1.2", "remend": "1.0.1", "shiki": "^3.12.2", "tailwind-merge": "^3.3.1", "unified": "^11.0.5", "unist-util-visit": "^5.0.0" }, "peerDependencies": { "react": "^18.0.0 || ^19.0.0" } }, "sha512-B4Y3Z/qiXl1Dc+LzAB5c52Cd1QGRiFjaDwP+ERoj1JtCykdRDM8X6HwQnn3YkpkSk0x3R7S/6LrGe1nQiElHQQ=="], "string-length": ["string-length@6.0.0", "", { "dependencies": { "strip-ansi": "^7.1.0" } }, "sha512-1U361pxZHEQ+FeSjzqRpV+cu2vTzYeWeafXFLykiFlv4Vc0n3njgU8HrMbyik5uwm77naWMuVG8fhEF+Ovb1Kg=="], diff --git a/package.json b/package.json index 84886417dd..33dc41d6ba 100644 --- a/package.json +++ b/package.json @@ -94,7 +94,7 @@ "rehype-harden": "^1.1.5", "shescape": "^2.1.6", "source-map-support": "^0.5.21", - "streamdown": "^1.4.0", + "streamdown": "1.6.10", "trpc-cli": "^0.12.1", "turndown": "^7.2.2", "undici": "^7.16.0", diff --git a/src/browser/components/Messages/MarkdownCore.tsx b/src/browser/components/Messages/MarkdownCore.tsx index fbe25b3af5..8144ae4154 100644 --- a/src/browser/components/Messages/MarkdownCore.tsx +++ b/src/browser/components/Messages/MarkdownCore.tsx @@ -13,6 +13,14 @@ import { markdownComponents } from "./MarkdownComponents"; interface MarkdownCoreProps { content: string; children?: React.ReactNode; // For cursor or other additions + /** + * Enable incomplete markdown parsing for streaming content. + * When true, the remend library will attempt to "repair" unclosed markdown + * syntax (e.g., adding closing ** for bold). This is useful during streaming + * but can cause bugs with content like $__variable (adds trailing __). + * Default: false for completed content, true during streaming. + */ + parseIncompleteMarkdown?: boolean; } // Plugin arrays are defined at module scope to maintain stable references. @@ -42,25 +50,27 @@ const REHYPE_PLUGINS: Pluggable[] = [ * * Memoized to prevent expensive re-parsing when content hasn't changed. */ -export const MarkdownCore = React.memo(({ content, children }) => { - // Memoize the normalized content to avoid recalculating on every render - const normalizedContent = useMemo(() => normalizeMarkdown(content), [content]); +export const MarkdownCore = React.memo( + ({ content, children, parseIncompleteMarkdown = false }) => { + // Memoize the normalized content to avoid recalculating on every render + const normalizedContent = useMemo(() => normalizeMarkdown(content), [content]); - return ( - <> - - {normalizedContent} - - {children} - - ); -}); + return ( + <> + + {normalizedContent} + + {children} + + ); + } +); MarkdownCore.displayName = "MarkdownCore"; diff --git a/src/browser/components/Messages/TypewriterMarkdown.tsx b/src/browser/components/Messages/TypewriterMarkdown.tsx index 3ac52f2fb3..3555495fed 100644 --- a/src/browser/components/Messages/TypewriterMarkdown.tsx +++ b/src/browser/components/Messages/TypewriterMarkdown.tsx @@ -26,7 +26,7 @@ export const TypewriterMarkdown = React.memo(function T return (
- +
); diff --git a/src/browser/stories/App.markdown.stories.tsx b/src/browser/stories/App.markdown.stories.tsx index 1e38307024..96605cef21 100644 --- a/src/browser/stories/App.markdown.stories.tsx +++ b/src/browser/stories/App.markdown.stories.tsx @@ -45,6 +45,22 @@ const TABLE_CONTENT = `Here are various markdown table examples: | \`api.timeout\` | 30000 | Timeout ms | \`API_TIMEOUT\` | | \`cache.enabled\` | true | Enable cache | \`CACHE_ENABLED\` |`; +// Bug repro: SQL with $__timeFilter causes "__" to appear at end of code block +const SQL_WITH_DOUBLE_UNDERSCORE = `👍 Glad it's working. For reference, the final query: + +\`\`\`sql +SELECT + TIMESTAMP_TRUNC(timestamp, DAY) as time, + COUNT(DISTINCT distinct_id) as dau +FROM \`mux-telemetry.posthog.events\` +WHERE + event NOT LIKE "$%" + AND $__timeFilter(timestamp) +GROUP BY time +ORDER BY time +\`\`\` +`; + const CODE_CONTENT = `Here's the implementation: \`\`\`typescript @@ -100,6 +116,29 @@ export const Tables: AppStory = { ), }; +/** SQL with double underscores in code block - tests for bug where __ leaks to end */ +export const SqlWithDoubleUnderscore: AppStory = { + render: () => ( + + setupSimpleChatStory({ + workspaceId: "ws-sql-underscore", + messages: [ + createUserMessage("msg-1", "Show me the SQL query", { + historySequence: 1, + timestamp: STABLE_TIMESTAMP - 100000, + }), + createAssistantMessage("msg-2", SQL_WITH_DOUBLE_UNDERSCORE, { + historySequence: 2, + timestamp: STABLE_TIMESTAMP - 90000, + }), + ], + }) + } + /> + ), +}; + /** Code blocks with syntax highlighting */ export const CodeBlocks: AppStory = { render: () => (