first draft

Author: AVGVSTVS96

package/src/core.ts

@@ -18,6 +18,7 @@ export type {
   Themes,
   Element,
   HighlighterOptions,
+  ThemedToken,
 } from './lib/types';
 
 export { createHighlighterCore } from 'shiki/core';

package/src/index.ts

@@ -18,6 +18,7 @@ export type {
   Themes,
   Element,
   HighlighterOptions,
+  ThemedToken,
 } from './lib/types';
 
 export {

package/src/lib/component.tsx

@@ -136,6 +136,12 @@ export const createShikiHighlighterComponent = (
         ...shikiOptions,
       };
 
+      if (options.outputFormat === 'tokens') {
+        throw new Error(
+          'ShikiHighlighter component does not support outputFormat="tokens". Use the useShikiHighlighter hook to access raw tokens.'
+        );
+      }
+
       // Use resolveLanguage to get displayLanguageId directly
       const { displayLanguageId } = resolveLanguage(
         language,

package/src/lib/hook.ts

@@ -18,6 +18,10 @@ import type {
   Awaitable,
   RegexEngine,
   BundledTheme,
+  CodeToTokensBaseOptions,
+  CodeToTokensOptions,
+  TokensResult,
+  ThemedToken,
 } from 'shiki';
 
 import type { ShikiLanguageRegistration } from './extended-types';
@@ -61,7 +65,7 @@ export const useShikiHighlighter = (
   ) => Promise<Highlighter | HighlighterCore>
 ) => {
   const [highlightedCode, setHighlightedCode] = useState<
-    ReactNode | string | null
+    ReactNode | string | ThemedToken[][] | null
   >(null);
 
   // Stabilize options, language and theme inputs to prevent unnecessary
@@ -108,8 +112,10 @@ export const useShikiHighlighter = (
           theme: singleTheme || DEFAULT_THEMES.dark,
         } as CodeOptionsSingleTheme<BundledTheme>);
 
-    const transformers = restOptions.transformers || [];
-    if (showLineNumbers) {
+    const isTokensOutput = stableOpts.outputFormat === 'tokens';
+    const transformers = [...(restOptions.transformers || [])];
+
+    if (showLineNumbers && !isTokensOutput) {
       transformers.push(lineNumbersTransformer(startingLineNumber));
     }
 
@@ -142,15 +148,32 @@ export const useShikiHighlighter = (
       const finalOptions = { ...shikiOptions, lang: langToUse };
 
       if (isMounted) {
-        const output =
-          stableOpts.outputFormat === 'html'
-            ? highlighter.codeToHtml(code, finalOptions)
-            : toJsxRuntime(highlighter.codeToHast(code, finalOptions), {
-                jsx,
-                jsxs,
-                Fragment,
-              });
-        setHighlightedCode(output);
+        if (stableOpts.outputFormat === 'tokens') {
+          const tokensOutput = isMultiTheme
+            ? highlighter.codeToTokens(
+                code,
+                finalOptions as CodeToTokensOptions
+              )
+            : highlighter.codeToTokensBase(
+                code,
+                finalOptions as CodeToTokensBaseOptions
+              );
+
+          const tokens = ((tokensOutput as TokensResult).tokens ??
+            tokensOutput) as ThemedToken[][];
+
+          setHighlightedCode(tokens);
+        } else {
+          const output =
+            stableOpts.outputFormat === 'html'
+              ? highlighter.codeToHtml(code, finalOptions)
+              : toJsxRuntime(highlighter.codeToHast(code, finalOptions), {
+                  jsx,
+                  jsxs,
+                  Fragment,
+                });
+          setHighlightedCode(output);
+        }
       }
     };
 

package/src/lib/types.ts

@@ -11,6 +11,7 @@ import type {
   BundledHighlighterOptions,
   Awaitable,
   RegexEngine,
+  ThemedToken,
 } from 'shiki';
 
 import type { ReactNode } from 'react';
@@ -78,9 +79,10 @@ interface ReactShikiOptions {
    * Output format for the highlighted code.
    * - 'react': Returns React nodes (default, safer)
    * - 'html': Returns HTML string (~15-45% faster, requires dangerouslySetInnerHTML)
+   * - 'tokens': Returns raw Shiki tokens (array of themed tokens per line)
    * @default 'react'
    */
-  outputFormat?: 'react' | 'html';
+  outputFormat?: 'react' | 'html' | 'tokens';
 
   /**
    * Custom Shiki highlighter instance to use instead of the default one.
@@ -162,7 +164,7 @@ export type UseShikiHighlighter = (
   lang: Language,
   themeInput: Theme | Themes,
   options?: HighlighterOptions
-) => ReactNode | string | null;
+) => ReactNode | string | ThemedToken[][] | null;
 
 export type {
   Language,
@@ -171,4 +173,5 @@ export type {
   Element,
   TimeoutState,
   HighlighterOptions,
+  ThemedToken,
 };

package/src/web.ts

@@ -18,6 +18,7 @@ export type {
   Themes,
   Element,
   HighlighterOptions,
+  ThemedToken,
 } from './lib/types';
 
 export {

package/tests/component.test.tsx

@@ -1,5 +1,6 @@
 import React, { useRef } from 'react';
 import { render, waitFor } from '@testing-library/react';
+import { vi } from 'vitest';
 import {
   ShikiHighlighter,
   createJavaScriptRegexEngine,
@@ -194,6 +195,26 @@ describe('ShikiHighlighter Component', () => {
         expect(container.querySelector('code')).toBeInTheDocument();
       });
     });
+
+    test('throws an error when outputFormat is tokens', () => {
+      const spy = vi.spyOn(console, 'error').mockImplementation(() => {});
+
+      expect(() =>
+        render(
+          <ShikiHighlighter
+            language="javascript"
+            theme="github-dark"
+            outputFormat="tokens"
+          >
+            {codeSample}
+          </ShikiHighlighter>
+        )
+      ).toThrowError(
+        'ShikiHighlighter component does not support outputFormat="tokens". Use the useShikiHighlighter hook to access raw tokens.'
+      );
+
+      spy.mockRestore();
+    });
   });
 
   describe('Ref Forwarding', () => {

package/tests/hook.test.tsx

@@ -1,10 +1,15 @@
-import { render, waitFor } from '@testing-library/react';
+import { render, renderHook, waitFor } from '@testing-library/react';
 import { vi } from 'vitest';
 import {
   useShikiHighlighter,
   createJavaScriptRegexEngine,
 } from '../src/index';
-import type { Language, Theme, Themes } from '../src/lib/types';
+import type {
+  Language,
+  Theme,
+  Themes,
+  ThemedToken,
+} from '../src/lib/types';
 import type { ShikiTransformer } from 'shiki';
 import { throttleHighlighting } from '../src/lib/utils';
 
@@ -16,7 +21,7 @@ interface TestComponentProps {
   langAlias?: Record<string, string>;
   showLineNumbers?: boolean;
   startingLineNumber?: number;
-  outputFormat?: 'react' | 'html';
+  outputFormat?: 'react' | 'html' | 'tokens';
   defaultColor?: string;
   cssVariablePrefix?: string;
   mergeWhitespaces?: boolean;
@@ -460,6 +465,29 @@ describe('useShikiHighlighter Hook', () => {
     });
   });
 
+  describe('Token Output Format', () => {
+    test('returns themed tokens when outputFormat is tokens', async () => {
+      const code = 'console.log("token test");';
+
+      const { result } = renderHook(() =>
+        useShikiHighlighter(code, 'javascript', 'github-dark', {
+          outputFormat: 'tokens',
+        })
+      );
+
+      await waitFor(() => {
+        expect(result.current).not.toBeNull();
+      });
+
+      const tokens = result.current as ThemedToken[][];
+
+      expect(Array.isArray(tokens)).toBe(true);
+      expect(tokens.length).toBeGreaterThan(0);
+      expect(Array.isArray(tokens[0])).toBe(true);
+      expect(tokens[0]?.[0]?.content).toBeDefined();
+    });
+  });
+
   describe('Output Format', () => {
     const sampleCode = 'const x = 1;\nconst y = 2;';