Skip to content

Commit 4ddfdb1

Browse files
antfuantfu
committed
feat: upgrade oniguruma-to-js, support more languages
1 parent 930dddc commit 4ddfdb1

25 files changed

+1306
-316
lines changed

‎docs/references/engine-js-compat.md

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
# JavaScript RegExp Engine Compatibility References
22

3-
> Genreated on Sunday, September 1, 2024
3+
> Genreated on Wednesday, September 4, 2024
44
>
5-
> Version `1.15.2`
5+
> Version `1.16.1`
66
>
77
> Runtime: Node.js v20.12.2
88
@@ -11,9 +11,9 @@
1111
| | Count |
1212
| :-------------- | --------------------------------: |
1313
| Total Languages | 213 |
14-
| Fully Supported | [178](#fully-supported-languages) |
15-
| Mismatched | [30](#mismatched-languages) |
16-
| Unsupported | [5](#unsupported-languages) |
14+
| Fully Supported | [187](#fully-supported-languages) |
15+
| Mismatched | [18](#mismatched-languages) |
16+
| Unsupported | [8](#unsupported-languages) |
1717

1818
## Fully Supported Languages
1919

@@ -28,6 +28,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
2828
| apl | ✅ OK | 179 | - |
2929
| applescript | ✅ OK | 151 | - |
3030
| ara | ✅ OK | 54 | - |
31+
| asciidoc | ✅ OK | 262 | - |
3132
| asm | ✅ OK | 297 | - |
3233
| astro | ✅ OK | 59 | - |
3334
| awk | ✅ OK | 36 | - |
@@ -38,6 +39,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
3839
| bibtex | ✅ OK | 19 | - |
3940
| bicep | ✅ OK | 28 | - |
4041
| c | ✅ OK | 158 | - |
42+
| cadence | ✅ OK | 71 | - |
4143
| clarity | ✅ OK | 43 | - |
4244
| clj | ✅ OK | 38 | - |
4345
| clojure | ✅ OK | 38 | - |
@@ -47,10 +49,13 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
4749
| codeql | ✅ OK | 150 | - |
4850
| coffee | ✅ OK | 120 | - |
4951
| common-lisp | ✅ OK | 57 | - |
52+
| coq | ✅ OK | 25 | - |
5053
| cpp | ✅ OK | 220 | - |
5154
| crystal | ✅ OK | 140 | - |
5255
| css | ✅ OK | 141 | - |
5356
| csv | ✅ OK | 1 | - |
57+
| cue | ✅ OK | 85 | - |
58+
| cypher | ✅ OK | 39 | - |
5459
| d | ✅ OK | 270 | - |
5560
| dart | ✅ OK | 71 | - |
5661
| dax | ✅ OK | 23 | - |
@@ -133,6 +138,8 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
133138
| perl | ✅ OK | 156 | - |
134139
| plsql | ✅ OK | 43 | - |
135140
| postcss | ✅ OK | 47 | - |
141+
| powerquery | ✅ OK | 30 | - |
142+
| powershell | ✅ OK | 88 | - |
136143
| prisma | ✅ OK | 26 | - |
137144
| prolog | ✅ OK | 26 | - |
138145
| proto | ✅ OK | 33 | - |
@@ -171,6 +178,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
171178
| tcl | ✅ OK | 33 | - |
172179
| templ | ✅ OK | 74 | - |
173180
| terraform | ✅ OK | 62 | - |
181+
| tex | ✅ OK | 38 | - |
174182
| toml | ✅ OK | 40 | - |
175183
| ts-tags | ✅ OK | - | - |
176184
| tsv | ✅ OK | 1 | - |
@@ -196,6 +204,7 @@ Languages that works with the JavaScript RegExp engine, and will produce the sam
196204
| wolfram | ✅ OK | 501 | - |
197205
| xml | ✅ OK | 30 | - |
198206
| xsl | ✅ OK | 5 | - |
207+
| yaml | ✅ OK | 46 | - |
199208
| zenscript | ✅ OK | 21 | - |
200209
| zig | ✅ OK | 51 | - |
201210
| zsh | ✅ OK | 146 | - |
@@ -208,43 +217,34 @@ Languages that does not throw with the JavaScript RegExp engine, but will produc
208217
| ------------ | :-------------- | ----------------: | --------------: |
209218
| angular-html | ⚠️ Mismatch | 2 | - |
210219
| apex | ⚠️ Mismatch | 189 | - |
211-
| asciidoc | ⚠️ Mismatch | 262 | - |
212220
| beancount | ⚠️ Mismatch | 39 | - |
213221
| blade | ⚠️ Mismatch | 330 | - |
214-
| cadence | ⚠️ Mismatch | 71 | - |
215-
| coq | ⚠️ Mismatch | 25 | - |
216-
| cue | ⚠️ Mismatch | 85 | - |
217-
| cypher | ⚠️ Mismatch | 39 | - |
218222
| haml | ⚠️ Mismatch | 64 | - |
219-
| haskell | ⚠️ Mismatch | 157 | - |
220223
| kusto | ⚠️ Mismatch | 60 | - |
221224
| latex | ⚠️ Mismatch | 183 | - |
222225
| markdown | ⚠️ Mismatch | 103 | - |
223226
| mdc | ⚠️ Mismatch | 27 | - |
224-
| mdx | ⚠️ Mismatch | 180 | - |
225227
| mermaid | ⚠️ Mismatch | 129 | - |
226228
| nginx | ⚠️ Mismatch | 102 | - |
227229
| php | ⚠️ Mismatch | 328 | - |
228230
| po | ⚠️ Mismatch | 23 | - |
229-
| powerquery | ⚠️ Mismatch | 30 | - |
230-
| powershell | ⚠️ Mismatch | 88 | - |
231231
| pug | ⚠️ Mismatch | 92 | - |
232-
| purescript | ⚠️ Mismatch | 72 | - |
233232
| rst | ⚠️ Mismatch | 61 | - |
234233
| splunk | ⚠️ Mismatch | 17 | - |
235234
| stata | ⚠️ Mismatch | 189 | - |
236235
| systemd | ⚠️ Mismatch | 32 | - |
237-
| tex | ⚠️ Mismatch | 38 | - |
238-
| yaml | ⚠️ Mismatch | 46 | - |
239236

240237
## Unsupported Languages
241238

242239
Languages that throws with the JavaScript RegExp engine (contains syntaxes that we can't polyfill yet). If you need to use these languages, please use the Oniguruma engine.
243240

244-
| Language | Highlight Match | Patterns Parsable | Patterns Failed |
245-
| -------- | :-------------- | ----------------: | --------------: |
246-
| ada | ✅ OK | 199 | 1 |
247-
| csharp | ⚠️ Mismatch | 298 | 1 |
248-
| razor | ⚠️ Mismatch | 83 | 2 |
249-
| swift | ❌ Error | 302 | 4 |
250-
| julia | ❌ Error | 77 | 18 |
241+
| Language | Highlight Match | Patterns Parsable | Patterns Failed |
242+
| ---------- | :-------------- | ----------------: | --------------: |
243+
| ada | ✅ OK | 199 | 1 |
244+
| csharp | ⚠️ Mismatch | 298 | 1 |
245+
| razor | ⚠️ Mismatch | 83 | 2 |
246+
| mdx | ❌ Error | 177 | 4 |
247+
| julia | ❌ Error | 90 | 5 |
248+
| swift | ❌ Error | 301 | 5 |
249+
| purescript | ❌ Error | 64 | 8 |
250+
| haskell | ❌ Error | 113 | 44 |

‎packages/compat/test/fixtures.test.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ import getHighlighter from '../src'
88
describe('fixtures', () => {
99
const files = import.meta.glob('./input/*.*', { as: 'raw', eager: true })
1010
const filter = process.env.FILTER
11-
Object.entries(files)
11+
Object
12+
.entries(files)
1213
.forEach(([path, content]) => {
1314
const run = !filter || path.includes(filter)
1415
? it

‎packages/core/src/highlight/code-to-tokens-ansi.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,8 @@ function dimColor(color: string) {
7070
if (hexMatch) {
7171
if (hexMatch[3]) {
7272
// convert from #rrggbbaa to #rrggbb(aa/2)
73-
const alpha = Math.round(Number.parseInt(hexMatch[3], 16) / 2)
73+
const alpha = Math
74+
.round(Number.parseInt(hexMatch[3], 16) / 2)
7475
.toString(16)
7576
.padStart(2, '0')
7677
return `#${hexMatch[1]}${hexMatch[2]}${alpha}`
@@ -81,7 +82,8 @@ function dimColor(color: string) {
8182
}
8283
else {
8384
// convert from #rgb to #rrggbb80
84-
return `#${Array.from(hexMatch[1])
85+
return `#${Array
86+
.from(hexMatch[1])
8587
.map(x => `${x}${x}`)
8688
.join('')}80`
8789
}

‎packages/core/src/highlight/code-to-tokens-themes.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ export function codeToTokensWithThemes(
1414
code: string,
1515
options: CodeToTokensWithThemesOptions,
1616
): ThemedTokenWithVariants[][] {
17-
const themes = Object.entries(options.themes)
17+
const themes = Object
18+
.entries(options.themes)
1819
.filter(i => i[1])
1920
.map(i => ({ color: i[0], theme: i[1]! }))
2021

‎packages/core/src/highlight/code-to-tokens.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ export function codeToTokens(
2626
cssVariablePrefix = '--shiki-',
2727
} = options
2828

29-
const themes = Object.entries(options.themes)
29+
const themes = Object
30+
.entries(options.themes)
3031
.filter(i => i[1])
3132
.map(i => ({ color: i[0], theme: i[1]! }))
3233
.sort((a, b) => a.color === defaultColor ? -1 : b.color === defaultColor ? 1 : 0)

‎packages/core/src/utils.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -137,7 +137,8 @@ export function splitToken<
137137
export function splitTokens<
138138
T extends Pick<ThemedToken, 'content' | 'offset'>,
139139
>(tokens: T[][], breakpoints: number[] | Set<number>) {
140-
const sorted = Array.from(breakpoints instanceof Set ? breakpoints : new Set(breakpoints))
140+
const sorted = Array
141+
.from(breakpoints instanceof Set ? breakpoints : new Set(breakpoints))
141142
.sort((a, b) => a - b)
142143

143144
if (!sorted.length)

‎packages/rehype/test/core.test.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ it('run with rehype-raw', async () => {
4848
],
4949
})
5050

51-
// eslint-disable-next-line unicorn/consistent-function-scoping
5251
const rehypeMetaString = () => (tree: Root) => {
5352
visit(tree, 'element', (node) => {
5453
if (node.tagName === 'code' && node.data?.meta) {

‎packages/shiki/scripts/prepare/langs.ts

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -118,12 +118,13 @@ export default langs
118118

119119
const bundled = Array.from(bundledIds).map(id => grammars.find(i => i.name === id)!).filter(Boolean)
120120

121-
const info = bundled.map(i => ({
122-
id: i.name,
123-
name: i.displayName || i.name,
124-
aliases: i.aliases,
125-
import: `__(() => import('./langs/${i.name}')) as DynamicImportLanguageRegistration__`,
126-
}) as const)
121+
const info = bundled
122+
.map(i => ({
123+
id: i.name,
124+
name: i.displayName || i.name,
125+
aliases: i.aliases,
126+
import: `__(() => import('./langs/${i.name}')) as DynamicImportLanguageRegistration__`,
127+
}) as const)
127128
.sort((a, b) => a.id.localeCompare(b.id))
128129

129130
const type = info.flatMap(i => [...i.aliases || [], i.id]).sort().map(i => ` | '${i}'`).join('\n')

‎packages/shiki/test/core.test.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,9 @@ describe('errors', () => {
158158
})
159159

160160
const code = shiki.codeToHtml('console.log("Hi")', { lang: 'javascript', theme: mtp })
161-
expect.soft(code)
161+
162+
expect
163+
.soft(code)
162164
.toMatchInlineSnapshot(`"<pre class="shiki material-theme-palenight" style="background-color:#292D3E;color:#babed8" tabindex="0"><code><span class="line"><span style="color:#BABED8">console</span><span style="color:#89DDFF">.</span><span style="color:#82AAFF">log</span><span style="color:#BABED8">(</span><span style="color:#89DDFF">"</span><span style="color:#C3E88D">Hi</span><span style="color:#89DDFF">"</span><span style="color:#BABED8">)</span></span></code></pre>"`)
163165

164166
expect.soft(shiki.getLoadedThemes()).toContain('material-theme-palenight')

‎packages/shiki/test/decorations.test.ts

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,8 @@ describe('decorations errors', () => {
105105
{ start: 10, end: 0 },
106106
],
107107
})
108-
}).rejects
108+
})
109+
.rejects
109110
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration range: {"line":1,"character":6,"offset":10} - {"line":0,"character":0,"offset":0}]`)
110111
})
111112

@@ -119,8 +120,7 @@ describe('decorations errors', () => {
119120
{ start: 5, end: 15 },
120121
],
121122
})
122-
}).rejects
123-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Decorations {"line":0,"character":0,"offset":0} and {"line":1,"character":1,"offset":5} intersect.]`)
123+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Decorations {"line":0,"character":0,"offset":0} and {"line":1,"character":1,"offset":5} intersect.]`)
124124
})
125125

126126
it('throws when lines overflow', async () => {
@@ -132,8 +132,7 @@ describe('decorations errors', () => {
132132
{ start: { line: 100, character: 0 }, end: { line: 100, character: 1 } },
133133
],
134134
})
135-
}).rejects
136-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":100,"character":0}. Lines length: 12]`)
135+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":100,"character":0}. Lines length: 12]`)
137136
})
138137

139138
it('throws when chars overflow', async () => {
@@ -145,8 +144,7 @@ describe('decorations errors', () => {
145144
{ start: { line: 0, character: 0 }, end: { line: 0, character: 10 } },
146145
],
147146
})
148-
}).rejects
149-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":0,"character":10}. Line 0 length: 4]`)
147+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":0,"character":10}. Line 0 length: 4]`)
150148

151149
expect(async () => {
152150
await codeToHtml(code, {
@@ -159,8 +157,7 @@ describe('decorations errors', () => {
159157
},
160158
],
161159
})
162-
}).rejects
163-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":1,"character":36}. Line 1 length: 33]`)
160+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration position {"line":1,"character":36}. Line 1 length: 33]`)
164161
})
165162

166163
it('throws when offset underflows/overflows', async () => {
@@ -170,16 +167,14 @@ describe('decorations errors', () => {
170167
lang: 'ts',
171168
decorations: [{ start: 1, end: 1000 }],
172169
})
173-
}).rejects
174-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: 1000. Code length: 252]`)
170+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: 1000. Code length: 252]`)
175171

176172
expect(async () => {
177173
await codeToHtml(code, {
178174
theme: 'vitesse-light',
179175
lang: 'ts',
180176
decorations: [{ start: -3, end: 5 }],
181177
})
182-
}).rejects
183-
.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: -3. Code length: 252]`)
178+
}).rejects.toThrowErrorMatchingInlineSnapshot(`[ShikiError: Invalid decoration offset: -3. Code length: 252]`)
184179
})
185180
})

‎packages/shiki/test/engine-js/compare.test.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,8 @@ describe('cases', async () => {
183183
])
184184
}
185185

186-
await expect.soft(JSON.stringify(engineWasm.instances, null, 2))
186+
await expect
187+
.soft(JSON.stringify(engineWasm.instances, null, 2))
187188
.toMatchFileSnapshot(`./__records__/${c.c.name}.json`)
188189

189190
compare.forEach(([a, b]) => {

‎packages/shiki/test/grammar-state.test.ts

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,13 +41,17 @@ it('getLastGrammarState', async () => {
4141
grammarState: state,
4242
})
4343

44-
expect.soft(highlightedNatural)
45-
.not.toEqual(highlightedContext)
44+
expect
45+
.soft(highlightedNatural)
46+
.not
47+
.toEqual(highlightedContext)
4648

47-
expect.soft(highlightedContext)
49+
expect
50+
.soft(highlightedContext)
4851
.toEqual(highlightedContext2)
4952

50-
expect.soft(highlightedNatural)
53+
expect
54+
.soft(highlightedNatural)
5155
.toMatchInlineSnapshot(`
5256
{
5357
"bg": "#ffffff",
@@ -115,8 +119,7 @@ it('getLastGrammarState', async () => {
115119
}
116120
`)
117121

118-
expect.soft(highlightedContext)
119-
.toMatchInlineSnapshot(`
122+
expect.soft(highlightedContext).toMatchInlineSnapshot(`
120123
{
121124
"bg": "#ffffff",
122125
"fg": "#393a34",
@@ -203,7 +206,8 @@ it('grammarContextCode', async () => {
203206
.toMatchInlineSnapshot(`"<span style="color:#999999">&#x3C;</span><span style="color:#1E754F">div</span><span style="color:#999999"> :</span><span style="color:#59873A">value</span><span style="color:#999999">=</span><span style="color:#999999">"</span><span style="color:#2F798A">1</span><span style="color:#AB5959"> +</span><span style="color:#2F798A"> 2</span><span style="color:#999999">"</span><span style="color:#999999">></span><span style="color:#393A34">&#x3C;button /></span><span style="color:#999999">&#x3C;/</span><span style="color:#1E754F">div</span><span style="color:#999999">></span>"`)
204207

205208
expect(highlightedVueTemplate)
206-
.not.toEqual(highlightedVueBare)
209+
.not
210+
.toEqual(highlightedVueBare)
207211
})
208212

209213
describe('errors', () => {

‎packages/shiki/test/wasm4.test.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ it('wasm', async () => {
1111
const shiki = await createHighlighterCore({
1212
themes: [nord],
1313
langs: [js as any],
14-
// eslint-disable-next-line unicorn/consistent-function-scoping
1514
loadWasm: Promise.resolve().then(() => obj => WebAssembly.instantiate(wasmBinary, obj).then(r => r.instance)),
1615
})
1716

‎packages/transformers/src/transformers/transformer-meta-highlight.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ export function parseMetaHighlightString(meta: string) {
66
const match = meta.match(/\{([\d,-]+)\}/)
77
if (!match)
88
return null
9-
const lines = match[1].split(',')
9+
const lines = match[1]
10+
.split(',')
1011
.flatMap((v) => {
1112
const num = v.split('-').map(v => Number.parseInt(v, 10))
1213
if (num.length === 1)

‎packages/twoslash/src/renderer-rich.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -419,7 +419,8 @@ export function rendererRich(options: RendererRichOptions = {}): TwoslashRendere
419419
if (node.type !== 'text')
420420
throw new ShikiTwoslashError(`Renderer hook nodeCompletion only works on text nodes, got ${node.type}`)
421421

422-
const items: Element[] = query.completions
422+
const items: Element[] = query
423+
.completions
423424
.map((i) => {
424425
const kind = i.kind || 'default'
425426
const isDeprecated = 'kindModifiers' in i && typeof i.kindModifiers === 'string' && i.kindModifiers?.split(',').includes('deprecated')

‎packages/twoslash/test/fixtures.test.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,13 @@ describe('fixtures', () => {
3030

3131
const html = hastToHtml(hast)
3232

33-
expect.soft(JSON.stringify(hast, null, 2))
33+
expect
34+
.soft(JSON.stringify(hast, null, 2))
3435
.toMatchFileSnapshot(`./out/${name}.json`)
3536

3637
const style = '<link rel="stylesheet" href="../../style-rich.css" />'
37-
expect.soft(style + html)
38+
expect
39+
.soft(style + html)
3840
.toMatchFileSnapshot(`./out/${name}.html`)
3941
})
4042
}

0 commit comments

Comments
 (0)