|
|
@@ -5,12 +5,17 @@
|
|
|
* theme package's token sheets as `--shiki-*` custom properties (light and
|
|
|
* dark blocks), never here — the repo's tokens-only styling rule.
|
|
|
*
|
|
|
- * Grammars are the set the harness actually renders: the markdown-fence and
|
|
|
- * `run_code` languages (TypeScript, shell, JSON) plus the file-extension
|
|
|
- * language hints the read tool's `langFromPath` emits (`packages/fs/tool-fs`),
|
|
|
- * so a read card highlights the same source, config, and markup extensions the
|
|
|
- * backend recognizes. An unknown or absent language falls back to plain text
|
|
|
- * (no highlighting, still monospace) — never an error.
|
|
|
+ * Only the three markdown-fence and `run_code` grammars (TypeScript, shell,
|
|
|
+ * JSON) load into the singleton at boot — the set every session renders. The
|
|
|
+ * read card's wider extension set (the file-extension language hints the read
|
|
|
+ * tool's `langFromPath` emits — `packages/fs/tool-fs`: python, rust, yaml,
|
|
|
+ * markup, …) is imported lazily and registered the first time such a language
|
|
|
+ * is requested, so a session that never opens a read card in one of those
|
|
|
+ * languages pays neither the ~1.6 MB of grammar modules nor their synchronous
|
|
|
+ * init. The first render of a lazy language falls back to plain text while its
|
|
|
+ * grammar loads, then {@link onGrammarLoaded} notifies subscribers to re-render
|
|
|
+ * with highlighting. An unknown or absent language falls back to plain text (no
|
|
|
+ * highlighting, still monospace) — never an error.
|
|
|
*/
|
|
|
|
|
|
import { createHighlighterCoreSync, createCssVariablesTheme } from 'shiki/core'
|
|
|
@@ -18,55 +23,66 @@ import { createJavaScriptRegexEngine } from 'shiki/engine/javascript'
|
|
|
import langTs from '@shikijs/langs/typescript'
|
|
|
import langBash from '@shikijs/langs/shellscript'
|
|
|
import langJson from '@shikijs/langs/json'
|
|
|
-import langPython from '@shikijs/langs/python'
|
|
|
-import langRuby from '@shikijs/langs/ruby'
|
|
|
-import langGo from '@shikijs/langs/go'
|
|
|
-import langRust from '@shikijs/langs/rust'
|
|
|
-import langJava from '@shikijs/langs/java'
|
|
|
-import langC from '@shikijs/langs/c'
|
|
|
-import langCpp from '@shikijs/langs/cpp'
|
|
|
-import langCsharp from '@shikijs/langs/csharp'
|
|
|
-import langKotlin from '@shikijs/langs/kotlin'
|
|
|
-import langSwift from '@shikijs/langs/swift'
|
|
|
-import langPhp from '@shikijs/langs/php'
|
|
|
-import langYaml from '@shikijs/langs/yaml'
|
|
|
-import langToml from '@shikijs/langs/toml'
|
|
|
-import langIni from '@shikijs/langs/ini'
|
|
|
-import langMarkdown from '@shikijs/langs/markdown'
|
|
|
-import langMdx from '@shikijs/langs/mdx'
|
|
|
-import langHtml from '@shikijs/langs/html'
|
|
|
-import langCss from '@shikijs/langs/css'
|
|
|
-import langScss from '@shikijs/langs/scss'
|
|
|
-import langLess from '@shikijs/langs/less'
|
|
|
-import langSql from '@shikijs/langs/sql'
|
|
|
-import langXml from '@shikijs/langs/xml'
|
|
|
-import langLua from '@shikijs/langs/lua'
|
|
|
import type { HighlighterCore } from 'shiki/core'
|
|
|
import type { CSSProperties } from 'react'
|
|
|
|
|
|
+/** A shiki grammar module's default export (a `LanguageRegistration[]`), taken
|
|
|
+ * from a boot grammar so no direct `@shikijs/types` dependency is needed. */
|
|
|
+type LangModule = { default: typeof langTs }
|
|
|
+
|
|
|
/**
|
|
|
- * Grammars the singleton registers; each entry's own `name` is the id
|
|
|
+ * Grammars the singleton loads at boot; each entry's own `name` is the id
|
|
|
* `codeToTokens`/`codeToHtml` resolve. The TypeScript grammar embeds JS/JSX/TSX,
|
|
|
* so the JS-family fence aliases resolve to it rather than a separate grammar.
|
|
|
+ * The read card's wider set loads lazily through {@link LAZY_GRAMMARS}.
|
|
|
*/
|
|
|
-const LANGS = [
|
|
|
- langTs, langBash, langJson,
|
|
|
- langPython, langRuby, langGo, langRust, langJava,
|
|
|
- langC, langCpp, langCsharp, langKotlin, langSwift, langPhp,
|
|
|
- langYaml, langToml, langIni,
|
|
|
- langMarkdown, langMdx, langHtml, langCss, langScss, langLess,
|
|
|
- langSql, langXml, langLua,
|
|
|
-]
|
|
|
+const LANGS = [langTs, langBash, langJson]
|
|
|
+
|
|
|
+/**
|
|
|
+ * The read card's extension grammars, each behind a dynamic import so its
|
|
|
+ * module stays out of the boot chunk until a read of that language renders.
|
|
|
+ * Keyed by the grammar id (`LanguageRegistration.name`) the aliases resolve to.
|
|
|
+ * `@shikijs/langs`' default export is a `LanguageRegistration[]`; the loader
|
|
|
+ * hands the whole array to `loadLanguageSync`, which registers each entry
|
|
|
+ * (including embedded sub-grammars). The three boot grammars are absent —
|
|
|
+ * already loaded, so no alias value ever points at a missing entry here.
|
|
|
+ */
|
|
|
+const LAZY_GRAMMARS = new Map<string, () => Promise<LangModule>>([
|
|
|
+ ['python', () => import('@shikijs/langs/python')],
|
|
|
+ ['ruby', () => import('@shikijs/langs/ruby')],
|
|
|
+ ['go', () => import('@shikijs/langs/go')],
|
|
|
+ ['rust', () => import('@shikijs/langs/rust')],
|
|
|
+ ['java', () => import('@shikijs/langs/java')],
|
|
|
+ ['c', () => import('@shikijs/langs/c')],
|
|
|
+ ['cpp', () => import('@shikijs/langs/cpp')],
|
|
|
+ ['csharp', () => import('@shikijs/langs/csharp')],
|
|
|
+ ['kotlin', () => import('@shikijs/langs/kotlin')],
|
|
|
+ ['swift', () => import('@shikijs/langs/swift')],
|
|
|
+ ['php', () => import('@shikijs/langs/php')],
|
|
|
+ ['yaml', () => import('@shikijs/langs/yaml')],
|
|
|
+ ['toml', () => import('@shikijs/langs/toml')],
|
|
|
+ ['ini', () => import('@shikijs/langs/ini')],
|
|
|
+ ['markdown', () => import('@shikijs/langs/markdown')],
|
|
|
+ ['mdx', () => import('@shikijs/langs/mdx')],
|
|
|
+ ['html', () => import('@shikijs/langs/html')],
|
|
|
+ ['css', () => import('@shikijs/langs/css')],
|
|
|
+ ['scss', () => import('@shikijs/langs/scss')],
|
|
|
+ ['less', () => import('@shikijs/langs/less')],
|
|
|
+ ['sql', () => import('@shikijs/langs/sql')],
|
|
|
+ ['xml', () => import('@shikijs/langs/xml')],
|
|
|
+ ['lua', () => import('@shikijs/langs/lua')],
|
|
|
+])
|
|
|
|
|
|
/**
|
|
|
- * Language ids (and aliases) the singleton registers; everything else renders
|
|
|
+ * Language ids (and aliases) the highlighter accepts; everything else renders
|
|
|
* plain. A Map, not an object: fence info strings are assistant-authored, so
|
|
|
* a label like `constructor` or `__proto__` must miss instead of resolving an
|
|
|
* inherited property and crashing the renderer inside shiki. Keys cover both
|
|
|
* the markdown-fence aliases `CodeBlock` uses and the file-extension hint ids
|
|
|
* the read tool's `langFromPath` emits, so both callers resolve the same
|
|
|
* grammars. The JS family maps to the TypeScript grammar (which embeds it),
|
|
|
- * unchanged from when this was the only non-shell/JSON grammar.
|
|
|
+ * unchanged from when this was the only non-shell/JSON grammar. A value not in
|
|
|
+ * {@link LANGS} names a {@link LAZY_GRAMMARS} entry loaded on first use.
|
|
|
*/
|
|
|
const LANG_ALIASES = new Map<string, string>([
|
|
|
['typescript', 'typescript'],
|
|
|
@@ -132,6 +148,62 @@ function highlighter(): HighlighterCore {
|
|
|
return singleton
|
|
|
}
|
|
|
|
|
|
+/** Grammar ids whose lazy import is in flight or done, so it is requested once. */
|
|
|
+const requested = new Set<string>()
|
|
|
+/** Subscribers re-rendered after a lazy grammar registers (React callers). */
|
|
|
+const listeners = new Set<() => void>()
|
|
|
+/** Bumped on each lazy-grammar load; the `useSyncExternalStore` snapshot. */
|
|
|
+let loadCount = 0
|
|
|
+
|
|
|
+/**
|
|
|
+ * Subscribe to lazy-grammar load completions; `listener` fires after a
|
|
|
+ * {@link LAZY_GRAMMARS} grammar finishes registering on the singleton, so a
|
|
|
+ * caller that rendered its plain fallback while the grammar loaded can
|
|
|
+ * re-highlight. Shaped as a `useSyncExternalStore` subscribe: pair it with
|
|
|
+ * {@link grammarLoadCount} as the snapshot. Returns an unsubscribe function.
|
|
|
+ * @param listener - invoked (no args) on each grammar-load completion.
|
|
|
+ * @returns a disposer that removes the listener.
|
|
|
+ */
|
|
|
+export function subscribeGrammarLoaded(listener: () => void): () => void {
|
|
|
+ listeners.add(listener)
|
|
|
+ return () => { listeners.delete(listener) }
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * The lazy-grammar load counter — a value that changes on every load, so a
|
|
|
+ * `useSyncExternalStore` snapshot re-renders the subscriber when a grammar
|
|
|
+ * registers. Opaque: only its identity across renders matters.
|
|
|
+ * @returns the current load count.
|
|
|
+ */
|
|
|
+export function grammarLoadCount(): number {
|
|
|
+ return loadCount
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Ensure the grammar `resolved` names is registered. A boot grammar (not in
|
|
|
+ * {@link LAZY_GRAMMARS}) and an already-loaded lazy grammar report ready
|
|
|
+ * synchronously; a lazy grammar not yet loaded starts its import (once) and
|
|
|
+ * reports not-ready, so the caller renders plain until a
|
|
|
+ * {@link subscribeGrammarLoaded} listener fires.
|
|
|
+ * @param resolved - the grammar id an alias resolved to.
|
|
|
+ * @returns whether the grammar is registered and ready to tokenize now.
|
|
|
+ */
|
|
|
+function ensureGrammar(resolved: string): boolean {
|
|
|
+ const load = LAZY_GRAMMARS.get(resolved)
|
|
|
+ // A boot grammar (already registered) has no lazy loader; it is always ready.
|
|
|
+ if (load === undefined) return true
|
|
|
+ if (highlighter().getLoadedLanguages().includes(resolved)) return true
|
|
|
+ if (!requested.has(resolved)) {
|
|
|
+ requested.add(resolved)
|
|
|
+ void load().then((mod) => {
|
|
|
+ highlighter().loadLanguageSync(mod.default)
|
|
|
+ loadCount += 1
|
|
|
+ for (const listener of listeners) listener()
|
|
|
+ })
|
|
|
+ }
|
|
|
+ return false
|
|
|
+}
|
|
|
+
|
|
|
// Engine + grammar construction costs a long task (~120-175ms); building it
|
|
|
// during the first finalized fence's render would jank exactly when a stream
|
|
|
// completes. Warm the singleton in a deferred task at module load (= plugin
|
|
|
@@ -144,14 +216,17 @@ const warmupTimer = setTimeout(() => { highlighter() }, 0)
|
|
|
/**
|
|
|
* Highlight `code` into shiki's HTML (a single `<pre class="shiki">` tree)
|
|
|
* when `lang` maps to a registered grammar; `undefined` means the caller
|
|
|
- * renders its plain fallback.
|
|
|
+ * renders its plain fallback. A lazy grammar not yet loaded returns `undefined`
|
|
|
+ * for this call and loads in the background; subscribe with
|
|
|
+ * {@link onGrammarLoaded} to re-highlight once it registers.
|
|
|
* @param code - the source text.
|
|
|
* @param lang - the language hint (a markdown fence info string or a fixed caller id).
|
|
|
- * @returns the highlighted HTML, or `undefined` for unknown languages.
|
|
|
+ * @returns the highlighted HTML, or `undefined` for unknown or not-yet-loaded languages.
|
|
|
*/
|
|
|
export function highlightToHtml(code: string, lang: string | undefined): string | undefined {
|
|
|
const resolved = lang === undefined ? undefined : LANG_ALIASES.get(lang.toLowerCase())
|
|
|
if (resolved === undefined) return undefined
|
|
|
+ if (!ensureGrammar(resolved)) return undefined
|
|
|
return highlighter().codeToHtml(code, { lang: resolved, theme: 'css-variables' })
|
|
|
}
|
|
|
|
|
|
@@ -179,11 +254,12 @@ export interface HighlightSpan {
|
|
|
* is dropped so the run count matches the caller's own line array.
|
|
|
* @param code - the source text.
|
|
|
* @param lang - the language hint (a file-extension-derived language id).
|
|
|
- * @returns one entry per source line (each an array of runs), or `undefined` for unknown languages.
|
|
|
+ * @returns one entry per source line (each an array of runs), or `undefined` for unknown or not-yet-loaded languages.
|
|
|
*/
|
|
|
export function highlightLines(code: string, lang: string | undefined): HighlightSpan[][] | undefined {
|
|
|
const resolved = lang === undefined ? undefined : LANG_ALIASES.get(lang.toLowerCase())
|
|
|
if (resolved === undefined) return undefined
|
|
|
+ if (!ensureGrammar(resolved)) return undefined
|
|
|
const { tokens } = highlighter().codeToTokens(code, { lang: resolved, theme: 'css-variables' })
|
|
|
// shiki tokenizes `a\nb` into two lines; a trailing newline (`a\n`) adds a
|
|
|
// third, empty line the caller's own line array does not carry. Drop that
|
Chinesezjc