Skip to content

Commit eb842a3

Browse files
antfuantfu
committed
refactor: reorganize code
1 parent 54d1963 commit eb842a3

34 files changed

+135
-124
lines changed

‎packages/core/rollup.config.mjs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import ts from 'rollup-plugin-typescript2'
1111
const entries = [
1212
'src/index.ts',
1313
'src/types.ts',
14-
'src/textmate.ts',
1514
'src/wasm-inlined.ts',
1615
]
1716

‎packages/core/src/bundle-factory.ts renamed to ‎packages/core/src/constructors/bundle-factory.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import type { Root } from 'hast'
2-
import type { BundledHighlighterOptions, CodeToHastOptions, CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, GrammarState, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, RequireKeys, SpecialLanguage, SpecialTheme, ThemeInput, ThemedToken, ThemedTokenWithVariants, TokensResult } from './types'
3-
import { isSpecialLang, isSpecialTheme } from './utils'
2+
import type { BundledHighlighterOptions, CodeToHastOptions, CodeToTokensBaseOptions, CodeToTokensOptions, CodeToTokensWithThemesOptions, GrammarState, HighlighterCoreOptions, HighlighterGeneric, LanguageInput, RequireKeys, SpecialLanguage, SpecialTheme, ThemeInput, ThemedToken, ThemedTokenWithVariants, TokensResult } from '../types'
3+
import { isSpecialLang, isSpecialTheme } from '../utils'
4+
import { ShikiError } from '../error'
45
import { createHighlighterCore } from './highlighter'
5-
import { ShikiError } from './error'
66

77
export type CreateHighlighterFactory<L extends string, T extends string> = (
88
options: BundledHighlighterOptions<L, T>

‎packages/core/src/highlighter.ts renamed to ‎packages/core/src/constructors/highlighter.ts

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
import { codeToHast } from './code-to-hast'
2-
import { codeToHtml } from './code-to-html'
3-
import { codeToTokens } from './code-to-tokens'
4-
import { codeToTokensBase, getLastGrammarState } from './code-to-tokens-base'
5-
import { codeToTokensWithThemes } from './code-to-tokens-themes'
1+
import { codeToHast } from '../highlight/code-to-hast'
2+
import { codeToHtml } from '../highlight/code-to-html'
3+
import { codeToTokens } from '../highlight/code-to-tokens'
4+
import { codeToTokensBase, getLastGrammarState } from '../highlight/code-to-tokens-base'
5+
import { codeToTokensWithThemes } from '../highlight/code-to-tokens-themes'
6+
import type { HighlighterCore, HighlighterCoreOptions } from '../types'
67
import { createShikiInternal } from './internal'
7-
import type { HighlighterCore, HighlighterCoreOptions } from './types'
88

99
/**
1010
* Create a Shiki core highlighter instance, with no languages or themes bundled.

‎packages/core/src/internal.ts renamed to ‎packages/core/src/constructors/internal.ts

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,22 @@
1-
import type { HighlighterCoreOptions, LanguageInput, LanguageRegistration, LoadWasmOptions, MaybeGetter, ShikiInternal, SpecialLanguage, SpecialTheme, ThemeInput, ThemeRegistrationAny, ThemeRegistrationResolved } from './types'
2-
import { Registry } from './registry'
3-
import { Resolver } from './resolver'
4-
import { normalizeTheme } from './normalize'
5-
import { isSpecialLang, isSpecialTheme } from './utils'
6-
import { ShikiError } from './error'
7-
import { createWasmOnigEngine } from './engines/wasm'
1+
import type {
2+
HighlighterCoreOptions,
3+
LanguageInput,
4+
LanguageRegistration,
5+
LoadWasmOptions,
6+
MaybeGetter,
7+
ShikiInternal,
8+
SpecialLanguage,
9+
SpecialTheme,
10+
ThemeInput,
11+
ThemeRegistrationAny,
12+
ThemeRegistrationResolved,
13+
} from '../types'
14+
import { Registry } from '../textmate/registry'
15+
import { Resolver } from '../textmate/resolver'
16+
import { normalizeTheme } from '../textmate/normalize-theme'
17+
import { isSpecialLang, isSpecialTheme } from '../utils'
18+
import { ShikiError } from '../error'
19+
import { createWasmOnigEngine } from '../engines/wasm'
820

921
let _defaultWasmLoader: LoadWasmOptions | undefined
1022
/**

‎packages/core/src/engines/javascript.ts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import { onigurumaToRegexp } from 'oniguruma-to-js'
2-
import type { PatternScanner, RegexEngine, RegexEngineString } from '../textmate'
3-
import type { JavaScriptRegexEngineOptions } from '../types/engines'
2+
import type { JavaScriptRegexEngineOptions, PatternScanner, RegexEngine, RegexEngineString } from '../types'
43

54
const MAX = 4294967295
65

‎packages/core/src/engines/wasm.ts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
import type { RegexEngine } from '../textmate'
2-
import type { LoadWasmOptions } from '../types'
1+
import type { LoadWasmOptions, RegexEngine } from '../types'
32
import { OnigScanner, OnigString, loadWasm } from './oniguruma'
43

54
export { loadWasm }

‎packages/core/src/transformers.ts renamed to ‎packages/core/src/highlight/_get-transformers.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import type { ShikiTransformer, TransformerOptions } from './types'
2-
import { transformerDecorations } from './transformer-decorations'
1+
import type { ShikiTransformer, TransformerOptions } from '../types'
2+
import { transformerDecorations } from '../transformer-decorations'
33

44
const builtInTransformers: ShikiTransformer[] = [
55
/* @__PURE__ */ transformerDecorations(),

‎packages/core/src/code-to-hast.ts renamed to ‎packages/core/src/highlight/code-to-hast.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,11 @@ import type {
77
ShikiTransformerContextCommon,
88
ShikiTransformerContextSource,
99
ThemedToken,
10-
} from './types'
11-
import { FontStyle } from './types'
10+
} from '../types'
11+
import { FontStyle } from '../types'
12+
import { addClassToHast, getTokenStyleObject, stringifyTokenStyle } from '../utils'
13+
import { getTransformers } from './_get-transformers'
1214
import { codeToTokens } from './code-to-tokens'
13-
import { addClassToHast, getTokenStyleObject, stringifyTokenStyle } from './utils'
14-
import { getTransformers } from './transformers'
1515

1616
export function codeToHast(
1717
internal: ShikiInternal,

‎packages/core/src/code-to-html.ts renamed to ‎packages/core/src/highlight/code-to-html.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import { toHtml as hastToHtml } from 'hast-util-to-html'
2-
import type { CodeToHastOptions, ShikiInternal, ShikiTransformerContextCommon } from './types'
2+
import type { CodeToHastOptions, ShikiInternal, ShikiTransformerContextCommon } from '../types'
3+
import { getTransformers } from './_get-transformers'
34
import { codeToHast } from './code-to-hast'
45
import { codeToTokens } from './code-to-tokens'
5-
import { getTransformers } from './transformers'
66

77
export { hastToHtml }
88

‎packages/core/src/code-to-tokens-ansi.ts renamed to ‎packages/core/src/highlight/code-to-tokens-ansi.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { createAnsiSequenceParser, createColorPalette, namedColors } from 'ansi-sequence-parser'
2-
import type { ThemeRegistrationResolved, ThemedToken, TokenizeWithThemeOptions } from './types'
3-
import { FontStyle } from './types'
4-
import { applyColorReplacements, resolveColorReplacements, splitLines } from './utils'
2+
import type { ThemeRegistrationResolved, ThemedToken, TokenizeWithThemeOptions } from '../types'
3+
import { FontStyle } from '../types'
4+
import { applyColorReplacements, resolveColorReplacements, splitLines } from '../utils'
55

66
export function tokenizeAnsiWithTheme(
77
theme: ThemeRegistrationResolved,

‎packages/core/src/code-to-tokens-base.ts renamed to ‎packages/core/src/highlight/code-to-tokens-base.ts

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
/* ---------------------------------------------------------
22
* Copyright (C) Microsoft Corporation. All rights reserved.
33
*-------------------------------------------------------- */
4-
import type { IGrammar, IRawThemeSetting, StateStack } from './textmate'
5-
import { INITIAL } from './textmate'
6-
import type { CodeToTokensBaseOptions, FontStyle, ShikiInternal, ThemeRegistrationResolved, ThemedToken, ThemedTokenScopeExplanation, TokenizeWithThemeOptions } from './types'
7-
import { StackElementMetadata } from './stack-element-metadata'
8-
import { applyColorReplacements, isNoneTheme, isPlainLang, resolveColorReplacements, splitLines } from './utils'
4+
import { INITIAL } from '@shikijs/vscode-textmate'
5+
import type { IGrammar, IRawThemeSetting, StateStack } from '@shikijs/vscode-textmate'
6+
import type { CodeToTokensBaseOptions, FontStyle, ShikiInternal, ThemeRegistrationResolved, ThemedToken, ThemedTokenScopeExplanation, TokenizeWithThemeOptions } from '../types'
7+
import { applyColorReplacements, isNoneTheme, isPlainLang, resolveColorReplacements, splitLines } from '../utils'
8+
import { ShikiError } from '../error'
9+
import { GrammarState, getGrammarStack } from '../textmate/grammar-state'
10+
import { StackElementMetadata } from '../textmate/stack-element-metadata'
911
import { tokenizeAnsiWithTheme } from './code-to-tokens-ansi'
10-
import { ShikiError } from './error'
11-
import { GrammarState, getGrammarStack } from './grammar-state'
1212

1313
/**
1414
* Code to tokens, with a simple theme.

‎packages/core/src/code-to-tokens-themes.ts renamed to ‎packages/core/src/highlight/code-to-tokens-themes.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import type {
33
ShikiInternal,
44
ThemedToken,
55
ThemedTokenWithVariants,
6-
} from './types'
6+
} from '../types'
77
import { codeToTokensBase } from './code-to-tokens-base'
88

99
/**

‎packages/core/src/code-to-tokens.ts renamed to ‎packages/core/src/highlight/code-to-tokens.ts

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
import { codeToTokensBase } from './code-to-tokens-base'
1+
import { ShikiError } from '../error'
2+
import type { CodeToTokensOptions, ShikiInternal, ThemedToken, ThemedTokenWithVariants, TokensResult } from '../types'
3+
import { applyColorReplacements, getTokenStyleObject, resolveColorReplacements, stringifyTokenStyle } from '../utils'
24
import { codeToTokensWithThemes } from './code-to-tokens-themes'
3-
import { ShikiError } from './error'
4-
import type { CodeToTokensOptions, ShikiInternal, ThemedToken, ThemedTokenWithVariants, TokensResult } from './types'
5-
import { applyColorReplacements, getTokenStyleObject, resolveColorReplacements, stringifyTokenStyle } from './utils'
5+
import { codeToTokensBase } from './code-to-tokens-base'
66

77
/**
88
* High-level code-to-tokens API.

‎packages/core/src/index.ts

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,28 @@
1-
export * from './highlighter'
2-
export * from './bundle-factory'
3-
export * from './utils'
1+
// Types
42
export * from './types'
53

4+
// Constructors
5+
export * from './constructors/highlighter'
6+
export * from './constructors/bundle-factory'
7+
export { createShikiInternal, getShikiInternal, setDefaultWasmLoader } from './constructors/internal'
8+
9+
// Engines
610
export { createWasmOnigEngine, loadWasm } from './engines/wasm'
711
export { createJavaScriptRegexEngine } from './engines/javascript'
812

9-
export { createShikiInternal, getShikiInternal, setDefaultWasmLoader } from './internal'
10-
export { codeToTokensBase, tokenizeWithTheme } from './code-to-tokens-base'
11-
export { codeToTokens } from './code-to-tokens'
12-
export { tokenizeAnsiWithTheme } from './code-to-tokens-ansi'
13-
export { codeToHast, tokensToHast } from './code-to-hast'
14-
export { codeToHtml, hastToHtml } from './code-to-html'
15-
export { codeToTokensWithThemes } from './code-to-tokens-themes'
16-
export { normalizeTheme } from './normalize'
13+
// TextMate Utilities
14+
export { StackElementMetadata } from './textmate/stack-element-metadata'
15+
export { normalizeTheme } from './textmate/normalize-theme'
16+
17+
// Low-level Highlighting
18+
export { codeToTokensBase, tokenizeWithTheme } from './highlight/code-to-tokens-base'
19+
export { codeToTokens } from './highlight/code-to-tokens'
20+
export { tokenizeAnsiWithTheme } from './highlight/code-to-tokens-ansi'
21+
export { codeToHast, tokensToHast } from './highlight/code-to-hast'
22+
export { codeToHtml, hastToHtml } from './highlight/code-to-html'
23+
export { codeToTokensWithThemes } from './highlight/code-to-tokens-themes'
24+
25+
// Utils and Misc
26+
export * from './utils'
1727
export { transformerDecorations } from './transformer-decorations'
1828
export { ShikiError } from './error'

‎packages/core/src/textmate.ts

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎packages/core/src/grammar-state.ts renamed to ‎packages/core/src/textmate/grammar-state.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
import type { StateStack, StateStackImpl } from './textmate'
2-
import { ShikiError } from './error'
3-
import { INITIAL } from './textmate'
1+
import { INITIAL } from '@shikijs/vscode-textmate'
2+
import type { StateStack, StateStackImpl } from '@shikijs/vscode-textmate'
3+
import { ShikiError } from '../error'
44

55
/**
66
* GrammarState is a special reference object that holds the state of a grammar.

‎packages/core/src/normalize.ts renamed to ‎packages/core/src/textmate/normalize-theme.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import type { ThemeRegistrationAny, ThemeRegistrationResolved } from './types'
1+
import type { ThemeRegistrationAny, ThemeRegistrationResolved } from '../types'
22

33
/**
44
* https://github.com/microsoft/vscode/blob/f7f05dee53fb33fe023db2e06e30a89d3094488f/src/vs/platform/theme/common/colorRegistry.ts#L258-L268

‎packages/core/src/registry.ts renamed to ‎packages/core/src/textmate/registry.ts

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
import type { IGrammarConfiguration, IRawTheme } from './textmate'
2-
import { Registry as TextMateRegistry, Theme as TextMateTheme } from './textmate'
3-
import type { Grammar, LanguageRegistration, ThemeRegistrationAny, ThemeRegistrationResolved } from './types'
1+
import { Registry as TextMateRegistry, Theme as TextMateTheme } from '@shikijs/vscode-textmate'
2+
import type { IGrammarConfiguration, IRawTheme } from '@shikijs/vscode-textmate'
3+
import type { Grammar, LanguageRegistration, ThemeRegistrationAny, ThemeRegistrationResolved } from '../types'
4+
import { ShikiError } from '../error'
5+
import { normalizeTheme } from './normalize-theme'
46
import type { Resolver } from './resolver'
5-
import { normalizeTheme } from './normalize'
6-
import { ShikiError } from './error'
77

88
export class Registry extends TextMateRegistry {
99
private _resolvedThemes: Map<string, ThemeRegistrationResolved> = new Map()

‎packages/core/src/resolver.ts renamed to ‎packages/core/src/textmate/resolver.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import type { IOnigLib, RegexEngine, RegistryOptions } from './textmate'
2-
import type { LanguageRegistration } from './types'
1+
import type { IOnigLib, RegistryOptions } from '@shikijs/vscode-textmate'
2+
import type { LanguageRegistration, RegexEngine } from '../types'
33

44
export class Resolver implements RegistryOptions {
55
private readonly _langs = new Map<string, LanguageRegistration>()

‎packages/core/src/stack-element-metadata.ts renamed to ‎packages/core/src/textmate/stack-element-metadata.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { FontStyle } from './types'
1+
import { FontStyle } from '../types'
22

33
/**
44
* Helpers to manage the "collapsed" metadata of an entire StackElement stack.

‎packages/core/src/transformer-decorations.ts

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,14 @@
11
import type { Element, ElementContent } from 'hast'
2-
import type { DecorationItem, DecorationTransformType, OffsetOrPosition, ResolvedDecorationItem, ResolvedPosition, ShikiTransformer, ShikiTransformerContextMeta, ShikiTransformerContextSource } from './types'
2+
import type {
3+
DecorationItem,
4+
DecorationTransformType,
5+
OffsetOrPosition,
6+
ResolvedDecorationItem,
7+
ResolvedPosition,
8+
ShikiTransformer,
9+
ShikiTransformerContextMeta,
10+
ShikiTransformerContextSource,
11+
} from './types'
312
import { addClassToHast, createPositionConverter, splitTokens } from './utils'
413
import { ShikiError } from './error'
514

‎packages/core/src/types/engines.ts

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,17 @@
1+
import type { OnigScanner, OnigString } from '@shikijs/vscode-textmate'
2+
3+
export interface PatternScanner extends OnigScanner {}
4+
5+
export interface RegexEngineString extends OnigString {}
6+
7+
/**
8+
* Engine for RegExp matching and scanning.
9+
*/
10+
export interface RegexEngine {
11+
createScanner: (patterns: string[]) => PatternScanner
12+
createString: (s: string) => RegexEngineString
13+
}
14+
115
type Awaitable<T> = T | Promise<T>
216

317
export interface WebAssemblyInstantiator {

‎packages/core/src/types/index.ts

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,4 @@ export * from './tokens'
77
export * from './transformers'
88
export * from './utils'
99
export * from './decorations'
10-
11-
export type {
12-
LoadWasmOptions,
13-
WebAssemblyInstantiator,
14-
WebAssemblyInstance,
15-
} from './engines'
10+
export * from './engines'

‎packages/core/src/types/options.ts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
1-
import type { LoadWasmOptions } from '../types'
2-
import type { RegexEngine } from '../textmate'
1+
import type { LoadWasmOptions, RegexEngine } from '../types'
32
import type { StringLiteralUnion } from './utils'
43
import type { LanguageInput, SpecialLanguage } from './langs'
54
import type { SpecialTheme, ThemeInput, ThemeRegistrationAny } from './themes'

‎packages/core/src/types/textmate.ts

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,7 @@ import type {
33
IRawGrammar as RawGrammar,
44
IRawTheme as RawTheme,
55
IRawThemeSetting as RawThemeSetting,
6-
} from '../textmate'
7-
8-
export type {
9-
RegexEngine,
10-
PatternScanner,
11-
RegexEngineString,
12-
} from '../textmate'
6+
} from '@shikijs/vscode-textmate'
137

148
export type {
159
RawGrammar,

‎packages/core/src/types/tokens.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
import type { GrammarState } from '../grammar-state'
2-
import type { IRawThemeSetting } from '../textmate'
1+
import type { IRawThemeSetting } from '@shikijs/vscode-textmate'
2+
import type { GrammarState } from '../textmate/grammar-state'
33
import type { SpecialLanguage } from './langs'
44
import type { SpecialTheme, ThemeRegistrationAny } from './themes'
55
import type { CodeOptionsThemes } from './options'

‎packages/monaco/package.json

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@
4646
"play": "nr -C playground play"
4747
},
4848
"dependencies": {
49-
"@shikijs/core": "workspace:*"
49+
"@shikijs/core": "workspace:*",
50+
"@shikijs/vscode-textmate": "catalog:"
5051
},
5152
"devDependencies": {
5253
"monaco-editor-core": "catalog:"

‎packages/monaco/src/index.ts

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import type monacoNs from 'monaco-editor-core'
22
import type { ShikiInternal, ThemeRegistrationResolved } from '@shikijs/core'
3-
import type { StateStack } from '@shikijs/core/textmate'
4-
import { INITIAL, StackElementMetadata } from '@shikijs/core/textmate'
3+
import type { StateStack } from '@shikijs/vscode-textmate'
4+
import { INITIAL } from '@shikijs/vscode-textmate'
5+
import { StackElementMetadata } from '@shikijs/core'
56

67
export interface MonacoTheme extends monacoNs.editor.IStandaloneThemeData {}
78

‎packages/shiki/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@
101101
},
102102
"dependencies": {
103103
"@shikijs/core": "workspace:*",
104+
"@shikijs/vscode-textmate": "catalog:",
104105
"@types/hast": "catalog:"
105106
},
106107
"devDependencies": {

‎packages/shiki/test/engine-js/compare.test.ts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import { describe, expect, it } from 'vitest'
22
import { wasmBinary } from '@shikijs/core/wasm-inlined'
3-
import type { RegexEngine } from '@shikijs/core/textmate'
4-
import type { LanguageRegistration, ThemeRegistration } from '../../src/core'
3+
import type { LanguageRegistration, RegexEngine, ThemeRegistration } from '../../src/core'
54
import { createHighlighterCore, createJavaScriptRegexEngine, loadWasm } from '../../src/core'
65

76
import { OnigScanner, OnigString } from '../../../core/src/engines/oniguruma'

0 commit comments

Comments
 (0)