feat: synchronous Shiki usage (#764)

Author: antfu

bench/engines.bench.ts

@@ -1,39 +1,43 @@
+/* eslint-disable no-console */
 import fs from 'node:fs/promises'
 import { bench, describe } from 'vitest'
 import type { BundledLanguage } from 'shiki'
 import { createHighlighter, createJavaScriptRegexEngine, createWasmOnigEngine } from 'shiki'
 import type { ReportItem } from '../scripts/report-engine-js-compat'
 
-describe('engines', async () => {
-  const js = createJavaScriptRegexEngine()
-  const wasm = await createWasmOnigEngine(() => import('shiki/wasm'))
+const js = createJavaScriptRegexEngine()
+const wasm = await createWasmOnigEngine(() => import('shiki/wasm'))
 
-  // Run `npx jiti scripts/report-engine-js-compat.ts` to generate the report first
-  const report = await fs.readFile('../scripts/report-engine-js-compat.json', 'utf-8').then(JSON.parse) as ReportItem[]
-  const langs = report.filter(i => i.highlightMatch === true).map(i => i.lang) as BundledLanguage[]
-  const samples = await Promise.all(langs.map(lang => fs.readFile(`../tm-grammars-themes/samples/${lang}.sample`, 'utf-8')))
+const RANGE = [0, 20]
 
-  const shikiJs = await createHighlighter({
-    langs,
-    themes: ['vitesse-dark'],
-    engine: js,
-  })
+// Run `npx jiti scripts/report-engine-js-compat.ts` to generate the report first
+const report = await fs.readFile(new URL('../scripts/report-engine-js-compat.json', import.meta.url), 'utf-8').then(JSON.parse) as ReportItem[]
+const langs = report.filter(i => i.highlightMatch === true).map(i => i.lang).slice(...RANGE) as BundledLanguage[]
+// Clone https://github.com/shikijs/textmate-grammars-themes to `../tm-grammars-themes`
+const samples = await Promise.all(langs.map(lang => fs.readFile(`../tm-grammars-themes/samples/${lang}.sample`, 'utf-8')))
 
-  const shikiWasm = await createHighlighter({
-    langs,
-    themes: ['vitesse-dark'],
-    engine: wasm,
-  })
+console.log('Benchmarking engines with', langs.length, 'languages')
+
+const shikiJs = await createHighlighter({
+  langs,
+  themes: ['vitesse-dark'],
+  engine: js,
+})
 
-  bench('js', () => {
-    for (const lang of langs) {
+const shikiWasm = await createHighlighter({
+  langs,
+  themes: ['vitesse-dark'],
+  engine: wasm,
+})
+
+for (const lang of langs) {
+  describe(lang, () => {
+    bench('js', () => {
       shikiJs.codeToTokensBase(samples[langs.indexOf(lang)], { lang, theme: 'vitesse-dark' })
-    }
-  }, { warmupIterations: 10, iterations: 30 })
+    })
 
-  bench('wasm', () => {
-    for (const lang of langs) {
+    bench('wasm', () => {
       shikiWasm.codeToTokensBase(samples[langs.indexOf(lang)], { lang, theme: 'vitesse-dark' })
-    }
-  }, { warmupIterations: 10, iterations: 30 })
-})
+    })
+  })
+}

packages/core/src/constructors/highlighter.ts

@@ -5,6 +5,7 @@ import { codeToTokensBase, getLastGrammarState } from '../highlight/code-to-toke
 import { codeToTokensWithThemes } from '../highlight/code-to-tokens-themes'
 import type { HighlighterCore, HighlighterCoreOptions } from '../types'
 import { createShikiInternal } from './internal'
+import { createShikiInternalSync } from './internal-sync'
 
 /**
  * Create a Shiki core highlighter instance, with no languages or themes bundled.
@@ -27,6 +28,29 @@ export async function createHighlighterCore(options: HighlighterCoreOptions = {}
   }
 }
 
+/**
+ * Create a Shiki core highlighter instance, with no languages or themes bundled.
+ * Wasm and each language and theme must be loaded manually.
+ *
+ * Synchronous version of `createHighlighterCore`, which requires to provide the engine and all themes and languages upfront.
+ *
+ * @see http://shiki.style/guide/install#fine-grained-bundle
+ */
+export function createHighlighterCoreSync(options: HighlighterCoreOptions<true> = {}): HighlighterCore {
+  const internal = createShikiInternalSync(options)
+
+  return {
+    getLastGrammarState: (code, options) => getLastGrammarState(internal, code, options),
+    codeToTokensBase: (code, options) => codeToTokensBase(internal, code, options),
+    codeToTokensWithThemes: (code, options) => codeToTokensWithThemes(internal, code, options),
+    codeToTokens: (code, options) => codeToTokens(internal, code, options),
+    codeToHast: (code, options) => codeToHast(internal, code, options),
+    codeToHtml: (code, options) => codeToHtml(internal, code, options),
+    ...internal,
+    getInternalContext: () => internal,
+  }
+}
+
 export function makeSingletonHighlighterCore(createHighlighter: typeof createHighlighterCore) {
   let _shiki: ReturnType<typeof createHighlighterCore>
 

packages/core/src/constructors/internal-sync.ts

@@ -0,0 +1,133 @@
+import type {
+  HighlighterCoreOptions,
+  LanguageInput,
+  LanguageRegistration,
+  MaybeArray,
+  ShikiInternal,
+  SpecialLanguage,
+  SpecialTheme,
+  ThemeInput,
+  ThemeRegistrationAny,
+  ThemeRegistrationResolved,
+} from '../types'
+import { Registry } from '../textmate/registry'
+import { Resolver } from '../textmate/resolver'
+import { normalizeTheme } from '../textmate/normalize-theme'
+import { ShikiError } from '../error'
+import { resolveLangs, resolveThemes } from '../textmate/getters-resolve'
+
+let instancesCount = 0
+
+/**
+ * Get the minimal shiki context for rendering.
+ *
+ * Synchronous version of `createShikiInternal`, which requires to provide the engine and all themes and languages upfront.
+ */
+export function createShikiInternalSync(options: HighlighterCoreOptions<true>): ShikiInternal {
+  instancesCount += 1
+  if (options.warnings !== false && instancesCount >= 10 && instancesCount % 10 === 0)
+    console.warn(`[Shiki] ${instancesCount} instances have been created. Shiki is supposed to be used as a singleton, consider refactoring your code to cache your highlighter instance; Or call \`highlighter.dispose()\` to release unused instances.`)
+
+  let isDisposed = false
+
+  if (!options.engine)
+    throw new ShikiError('`engine` option is required for synchronous mode')
+
+  const langs = (options.langs || []).flat(1)
+  const themes = (options.themes || []).flat(1).map(normalizeTheme)
+
+  const resolver = new Resolver(options.engine, langs)
+  const _registry = new Registry(resolver, themes, langs, options.langAlias)
+
+  let _lastTheme: string | ThemeRegistrationAny
+
+  function getLanguage(name: string | LanguageRegistration) {
+    ensureNotDisposed()
+    const _lang = _registry.getGrammar(typeof name === 'string' ? name : name.name)
+    if (!_lang)
+      throw new ShikiError(`Language \`${name}\` not found, you may need to load it first`)
+    return _lang
+  }
+
+  function getTheme(name: string | ThemeRegistrationAny): ThemeRegistrationResolved {
+    if (name === 'none')
+      return { bg: '', fg: '', name: 'none', settings: [], type: 'dark' }
+    ensureNotDisposed()
+    const _theme = _registry.getTheme(name)
+    if (!_theme)
+      throw new ShikiError(`Theme \`${name}\` not found, you may need to load it first`)
+    return _theme
+  }
+
+  function setTheme(name: string | ThemeRegistrationAny) {
+    ensureNotDisposed()
+    const theme = getTheme(name)
+    if (_lastTheme !== name) {
+      _registry.setTheme(theme)
+      _lastTheme = name
+    }
+    const colorMap = _registry.getColorMap()
+    return {
+      theme,
+      colorMap,
+    }
+  }
+
+  function getLoadedThemes() {
+    ensureNotDisposed()
+    return _registry.getLoadedThemes()
+  }
+
+  function getLoadedLanguages() {
+    ensureNotDisposed()
+    return _registry.getLoadedLanguages()
+  }
+
+  function loadLanguageSync(...langs: MaybeArray<LanguageRegistration>[]) {
+    ensureNotDisposed()
+    _registry.loadLanguages(langs.flat(1))
+  }
+
+  async function loadLanguage(...langs: (LanguageInput | SpecialLanguage)[]) {
+    return loadLanguageSync(await resolveLangs(langs))
+  }
+
+  async function loadThemeSync(...themes: MaybeArray<ThemeRegistrationAny>[]) {
+    ensureNotDisposed()
+    for (const theme of themes.flat(1)) {
+      _registry.loadTheme(theme)
+    }
+  }
+
+  async function loadTheme(...themes: (ThemeInput | SpecialTheme)[]) {
+    ensureNotDisposed()
+    return loadThemeSync(await resolveThemes(themes))
+  }
+
+  function ensureNotDisposed() {
+    if (isDisposed)
+      throw new ShikiError('Shiki instance has been disposed')
+  }
+
+  function dispose() {
+    if (isDisposed)
+      return
+    isDisposed = true
+    _registry.dispose()
+    instancesCount -= 1
+  }
+
+  return {
+    setTheme,
+    getTheme,
+    getLanguage,
+    getLoadedThemes,
+    getLoadedLanguages,
+    loadLanguage,
+    loadLanguageSync,
+    loadTheme,
+    loadThemeSync,
+    dispose,
+    [Symbol.dispose]: dispose,
+  }
+}