Skip to content

Commit 601b74e

Browse files
avivkellerCopilotovflowd
authored
chore(shiki): only send required langs to client, and support more on server (#7787)
* chore(shiki): add languages used in core * chore(shiki): only send required langs to client * fixup! chore(shiki): only send required langs to client * fix unit tests * fixup! fix unit tests * Update packages/rehype-shiki/src/highlighter.mjs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Signed-off-by: Aviv Keller <me@aviv.sh> * Update packages/rehype-shiki/src/highlighter.mjs Co-authored-by: Claudio W. <cwunder@gnome.org> Signed-off-by: Aviv Keller <me@aviv.sh> --------- Signed-off-by: Aviv Keller <me@aviv.sh> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Claudio W. <cwunder@gnome.org>
1 parent 6cb8b0a commit 601b74e

File tree

12 files changed

+265
-138
lines changed

12 files changed

+265
-138
lines changed

‎apps/site/components/Downloads/Release/ReleaseCodeBox.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
'use client';
22

3-
import { highlightToHtml } from '@node-core/rehype-shiki';
3+
import { highlightToHtml } from '@node-core/rehype-shiki/minimal';
44
import AlertBox from '@node-core/ui-components/Common/AlertBox';
55
import Skeleton from '@node-core/ui-components/Common/Skeleton';
66
import { useTranslations } from 'next-intl';

‎apps/site/next.mdx.plugins.mjs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
'use strict';
22

3-
import rehypeShikiji from '@node-core/rehype-shiki';
3+
import rehypeShikiji from '@node-core/rehype-shiki/plugin';
44
import remarkHeadings from '@vcarl/remark-headings';
55
import rehypeAutolinkHeadings from 'rehype-autolink-headings';
66
import rehypeSlug from 'rehype-slug';

‎packages/rehype-shiki/package.json‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
{
22
"name": "@node-core/rehype-shiki",
33
"type": "module",
4-
"main": "./src/index.mjs",
5-
"module": "./src/index.mjs",
4+
"exports": {
5+
".": "./src/index.mjs",
6+
"./*": "./src/*.mjs"
7+
},
68
"scripts": {
79
"lint:js": "eslint \"**/*.mjs\"",
8-
"test": "node --test"
10+
"test": "turbo test:unit",
11+
"test:unit": "node --experimental-test-coverage --experimental-test-module-mocks --test \"**/*.test.mjs\""
912
},
1013
"dependencies": {
1114
"@shikijs/core": "^3.3.0",

‎packages/rehype-shiki/src/__tests__/highlighter.test.mjs‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
import assert from 'node:assert/strict';
2+
import { describe, it, mock } from 'node:test';
3+
4+
// Mock dependencies
5+
const mockShiki = {
6+
codeToHtml: mock.fn(() => '<pre><code>highlighted code</code></pre>'),
7+
codeToHast: mock.fn(() => ({ type: 'element', tagName: 'pre' })),
8+
};
9+
10+
mock.module('@shikijs/core', {
11+
namedExports: { createHighlighterCoreSync: () => mockShiki },
12+
});
13+
14+
mock.module('@shikijs/engine-javascript', {
15+
namedExports: { createJavaScriptRegexEngine: () => ({}) },
16+
});
17+
18+
mock.module('shiki/themes/nord.mjs', {
19+
defaultExport: { name: 'nord', colors: { 'editor.background': '#2e3440' } },
20+
});
21+
22+
describe('createHighlighter', async () => {
23+
const { createHighlighter } = await import('../highlighter.mjs');
24+
25+
describe('getLanguageDisplayName', () => {
26+
it('returns display name for known languages', () => {
27+
const langs = [
28+
{ name: 'javascript', displayName: 'JavaScript', aliases: ['js'] },
29+
];
30+
const highlighter = createHighlighter({ langs });
31+
32+
assert.strictEqual(
33+
highlighter.getLanguageDisplayName('javascript'),
34+
'JavaScript'
35+
);
36+
assert.strictEqual(
37+
highlighter.getLanguageDisplayName('js'),
38+
'JavaScript'
39+
);
40+
});
41+
42+
it('returns original name for unknown languages', () => {
43+
const highlighter = createHighlighter({ langs: [] });
44+
assert.strictEqual(
45+
highlighter.getLanguageDisplayName('unknown'),
46+
'unknown'
47+
);
48+
});
49+
});
50+
51+
describe('highlightToHtml', () => {
52+
it('extracts inner HTML from code tag', () => {
53+
mockShiki.codeToHtml.mock.mockImplementationOnce(
54+
() => '<pre><code>const x = 1;</code></pre>'
55+
);
56+
57+
const highlighter = createHighlighter({});
58+
const result = highlighter.highlightToHtml('const x = 1;', 'javascript');
59+
60+
assert.strictEqual(result, 'const x = 1;');
61+
});
62+
});
63+
64+
describe('highlightToHast', () => {
65+
it('returns HAST tree from shiki', () => {
66+
const expectedHast = { type: 'element', tagName: 'pre' };
67+
mockShiki.codeToHast.mock.mockImplementationOnce(() => expectedHast);
68+
69+
const highlighter = createHighlighter({});
70+
const result = highlighter.highlightToHast('code', 'javascript');
71+
72+
assert.deepStrictEqual(result, expectedHast);
73+
});
74+
});
75+
});

‎packages/rehype-shiki/src/__tests__/languages.test.mjs‎

Lines changed: 0 additions & 27 deletions
This file was deleted.

‎packages/rehype-shiki/src/__tests__/plugin.test.mjs‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import assert from 'node:assert/strict';
2+
import { describe, it, mock } from 'node:test';
3+
4+
// Simplified mocks - only mock what's actually needed
5+
mock.module('../index.mjs', {
6+
namedExports: { highlightToHast: mock.fn(() => ({ children: [] })) },
7+
});
8+
9+
mock.module('classnames', {
10+
defaultExport: (...args) => args.filter(Boolean).join(' '),
11+
});
12+
13+
mock.module('hast-util-to-string', {
14+
namedExports: { toString: () => 'code' },
15+
});
16+
17+
const mockVisit = mock.fn();
18+
mock.module('unist-util-visit', {
19+
namedExports: { visit: mockVisit, SKIP: Symbol() },
20+
});
21+
22+
describe('rehypeShikiji', async () => {
23+
const { default: rehypeShikiji } = await import('../plugin.mjs');
24+
const mockTree = { type: 'root', children: [] };
25+
26+
it('calls visit twice', () => {
27+
mockVisit.mock.resetCalls();
28+
rehypeShikiji()(mockTree);
29+
assert.strictEqual(mockVisit.mock.calls.length, 2);
30+
});
31+
32+
it('creates CodeTabs for multiple code blocks', () => {
33+
const parent = {
34+
children: [
35+
{
36+
tagName: 'pre',
37+
children: [
38+
{
39+
tagName: 'code',
40+
data: { meta: 'displayName="JS"' },
41+
properties: { className: ['language-js'] },
42+
},
43+
],
44+
},
45+
{
46+
tagName: 'pre',
47+
children: [
48+
{
49+
tagName: 'code',
50+
data: { meta: 'displayName="TS"' },
51+
properties: { className: ['language-ts'] },
52+
},
53+
],
54+
},
55+
],
56+
};
57+
58+
mockVisit.mock.mockImplementation((tree, selector, visitor) => {
59+
if (selector === 'element') {
60+
visitor(parent.children[0], 0, parent);
61+
}
62+
});
63+
64+
rehypeShikiji()(mockTree);
65+
assert.ok(parent.children.some(child => child.tagName === 'CodeTabs'));
66+
});
67+
});

‎packages/rehype-shiki/src/highlighter.mjs‎

Lines changed: 59 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,67 @@
11
import { createHighlighterCoreSync } from '@shikijs/core';
22
import { createJavaScriptRegexEngine } from '@shikijs/engine-javascript';
3+
import shikiNordTheme from 'shiki/themes/nord.mjs';
34

4-
import { LANGUAGES, DEFAULT_THEME } from './languages.mjs';
5-
6-
let _shiki;
7-
8-
/**
9-
* Lazy-load and memoize the minimal Shikiji Syntax Highlighter
10-
* @returns {import('@shikijs/core').HighlighterCore}
11-
*/
12-
export const getShiki = () => {
13-
if (!_shiki) {
14-
_shiki = createHighlighterCoreSync({
15-
themes: [DEFAULT_THEME],
16-
langs: LANGUAGES,
17-
// Let's use Shiki's new Experimental JavaScript-based regex engine!
18-
engine: createJavaScriptRegexEngine(),
19-
});
20-
}
21-
return _shiki;
5+
const DEFAULT_THEME = {
6+
// We are updating this color because the background color and comment text color
7+
// in the Codebox component do not comply with accessibility standards
8+
// @see https://www.w3.org/WAI/WCAG21/Understanding/contrast-minimum.html
9+
colorReplacements: { '#616e88': '#707e99' },
10+
...shikiNordTheme,
2211
};
2312

2413
/**
25-
* Highlights code and returns the inner HTML inside the <code> tag
26-
*
27-
* @param {string} code - The code to highlight
28-
* @param {string} language - The programming language to use for highlighting
29-
* @returns {string} The inner HTML of the highlighted code
14+
* Creates a syntax highlighter with utility functions
15+
* @param {import('@shikijs/core').HighlighterCoreOptions} options - Configuration options for the highlighter
3016
*/
31-
export const highlightToHtml = (code, language) =>
32-
getShiki()
33-
.codeToHtml(code, { lang: language, theme: DEFAULT_THEME })
34-
// Shiki will always return the Highlighted code encapsulated in a <pre> and <code> tag
35-
// since our own CodeBox component handles the <code> tag, we just want to extract
36-
// the inner highlighted code to the CodeBox
37-
.match(/<code>(.+?)<\/code>/s)[1];
17+
export const createHighlighter = options => {
18+
const shiki = createHighlighterCoreSync({
19+
themes: [DEFAULT_THEME],
20+
engine: createJavaScriptRegexEngine(),
21+
...options,
22+
});
3823

39-
/**
40-
* Highlights code and returns a HAST tree
41-
*
42-
* @param {string} code - The code to highlight
43-
* @param {string} language - The programming language to use for highlighting
44-
* @returns {import('hast').Element} The HAST representation of the highlighted code
45-
*/
46-
export const highlightToHast = (code, language) =>
47-
getShiki().codeToHast(code, { lang: language, theme: DEFAULT_THEME });
24+
const theme = options.themes?.[0] ?? DEFAULT_THEME;
25+
const langs = options.langs ?? [];
26+
27+
const getLanguageDisplayName = language => {
28+
const languageByIdOrAlias = langs.find(
29+
({ name, aliases }) =>
30+
name.toLowerCase() === language.toLowerCase() ||
31+
(aliases !== undefined && aliases.includes(language.toLowerCase()))
32+
);
33+
34+
return languageByIdOrAlias?.displayName ?? language;
35+
};
36+
37+
/**
38+
* Highlights code and returns the inner HTML inside the <code> tag
39+
*
40+
* @param {string} code - The code to highlight
41+
* @param {string} language - The programming language to use for highlighting
42+
* @returns {string} The inner HTML of the highlighted code
43+
*/
44+
const highlightToHtml = (code, language) =>
45+
shiki
46+
.codeToHtml(code, { lang: language, theme })
47+
// Shiki will always return the Highlighted code encapsulated in a <pre> and <code> tag
48+
// since our own CodeBox component handles the <code> tag, we just want to extract
49+
// the inner highlighted code to the CodeBox
50+
.match(/<code>(.+?)<\/code>/s)[1];
51+
52+
/**
53+
* Highlights code and returns a HAST tree
54+
*
55+
* @param {string} code - The code to highlight
56+
* @param {string} language - The programming language to use for highlighting
57+
*/
58+
const highlightToHast = (code, language) =>
59+
shiki.codeToHast(code, { lang: language, theme });
60+
61+
return {
62+
shiki,
63+
getLanguageDisplayName,
64+
highlightToHtml,
65+
highlightToHast,
66+
};
67+
};

‎packages/rehype-shiki/src/index.mjs‎

Lines changed: 42 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,43 @@
1-
import { rehypeShikiji } from './plugin.mjs';
2-
export * from './highlighter.mjs';
3-
export * from './languages.mjs';
1+
import cLanguage from 'shiki/langs/c.mjs';
2+
import coffeeScriptLanguage from 'shiki/langs/coffeescript.mjs';
3+
import cPlusPlusLanguage from 'shiki/langs/cpp.mjs';
4+
import diffLanguage from 'shiki/langs/diff.mjs';
5+
import dockerLanguage from 'shiki/langs/docker.mjs';
6+
import httpLanguage from 'shiki/langs/http.mjs';
7+
import iniLanguage from 'shiki/langs/ini.mjs';
8+
import javaScriptLanguage from 'shiki/langs/javascript.mjs';
9+
import jsonLanguage from 'shiki/langs/json.mjs';
10+
import powershellLanguage from 'shiki/langs/powershell.mjs';
11+
import shellScriptLanguage from 'shiki/langs/shellscript.mjs';
12+
import shellSessionLanguage from 'shiki/langs/shellsession.mjs';
13+
import typeScriptLanguage from 'shiki/langs/typescript.mjs';
14+
import yamlLanguage from 'shiki/langs/yaml.mjs';
415

5-
export default rehypeShikiji;
16+
import { createHighlighter } from './highlighter.mjs';
17+
18+
const { shiki, getLanguageDisplayName, highlightToHast, highlightToHtml } =
19+
createHighlighter({
20+
langs: [
21+
...cLanguage,
22+
...coffeeScriptLanguage,
23+
...cPlusPlusLanguage,
24+
...diffLanguage,
25+
...dockerLanguage,
26+
...httpLanguage,
27+
...iniLanguage,
28+
{
29+
...javaScriptLanguage[0],
30+
// We patch the JavaScript language to include the CommonJS and ES Module aliases
31+
// that are commonly used (non-standard aliases) within our API docs and Blog posts
32+
aliases: javaScriptLanguage[0].aliases.concat('cjs', 'mjs'),
33+
},
34+
...jsonLanguage,
35+
...powershellLanguage,
36+
...shellScriptLanguage,
37+
...shellSessionLanguage,
38+
...typeScriptLanguage,
39+
...yamlLanguage,
40+
],
41+
});
42+
43+
export { shiki, getLanguageDisplayName, highlightToHast, highlightToHtml };

0 commit comments

Comments
 (0)