Skip to content

Commit 54330fe

Browse files
fgnassfgnass
committed
fix: adhere to remark's tokenizer rules decaporg#7315 
1 parent 86d41d7 commit 54330fe

File tree

2 files changed

+26
-109
lines changed

2 files changed

+26
-109
lines changed

‎packages/decap-cms-widget-markdown/src/serializers/__tests__/remarkShortcodes.spec.js

Lines changed: 5 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
import { Map, OrderedMap } from 'immutable';
1+
import { Map } from 'immutable';
22

3-
import { remarkParseShortcodes, getLinesWithOffsets } from '../remarkShortcodes';
3+
import { remarkParseShortcodes } from '../remarkShortcodes';
44

55
// Stub of Remark Parser
66
function process(value, plugins, processEat = () => {}) {
@@ -26,14 +26,9 @@ function EditorComponent({ id = 'foo', fromBlock = jest.fn(), pattern }) {
2626
describe('remarkParseShortcodes', () => {
2727
describe('pattern matching', () => {
2828
it('should work', () => {
29-
const editorComponent = EditorComponent({ pattern: /bar/ });
29+
const editorComponent = EditorComponent({ pattern: /^foo/ });
3030
process('foo bar', Map({ [editorComponent.id]: editorComponent }));
31-
expect(editorComponent.fromBlock).toHaveBeenCalledWith(expect.arrayContaining(['bar']));
32-
});
33-
it('should match value surrounded in newlines', () => {
34-
const editorComponent = EditorComponent({ pattern: /^bar$/ });
35-
process('foo\n\nbar\n', Map({ [editorComponent.id]: editorComponent }));
36-
expect(editorComponent.fromBlock).toHaveBeenCalledWith(expect.arrayContaining(['bar']));
31+
expect(editorComponent.fromBlock).toHaveBeenCalledWith(expect.arrayContaining(['foo']));
3732
});
3833
it('should match multiline shortcodes', () => {
3934
const editorComponent = EditorComponent({ pattern: /^foo\nbar$/ });
@@ -47,60 +42,15 @@ describe('remarkParseShortcodes', () => {
4742
expect.arrayContaining(['foo\n\nbar']),
4843
);
4944
});
50-
it('should match shortcodes based on order of occurrence in value', () => {
51-
const fooEditorComponent = EditorComponent({ id: 'foo', pattern: /foo/ });
52-
const barEditorComponent = EditorComponent({ id: 'bar', pattern: /bar/ });
53-
process(
54-
'foo\n\nbar',
55-
OrderedMap([
56-
[barEditorComponent.id, barEditorComponent],
57-
[fooEditorComponent.id, fooEditorComponent],
58-
]),
59-
);
60-
expect(fooEditorComponent.fromBlock).toHaveBeenCalledWith(expect.arrayContaining(['foo']));
61-
});
62-
it('should match shortcodes based on order of occurrence in value even when some use line anchors', () => {
63-
const barEditorComponent = EditorComponent({ id: 'bar', pattern: /bar/ });
64-
const bazEditorComponent = EditorComponent({ id: 'baz', pattern: /^baz$/ });
65-
process(
66-
'foo\n\nbar\n\nbaz',
67-
OrderedMap([
68-
[bazEditorComponent.id, bazEditorComponent],
69-
[barEditorComponent.id, barEditorComponent],
70-
]),
71-
);
72-
expect(barEditorComponent.fromBlock).toHaveBeenCalledWith(expect.arrayContaining(['bar']));
73-
});
7445
});
7546
describe('output', () => {
7647
it('should be a remark shortcode node', () => {
7748
const processEat = jest.fn();
7849
const shortcodeData = { bar: 'baz' };
7950
const expectedNode = { type: 'shortcode', data: { shortcode: 'foo', shortcodeData } };
80-
const editorComponent = EditorComponent({ pattern: /bar/, fromBlock: () => shortcodeData });
51+
const editorComponent = EditorComponent({ pattern: /^foo/, fromBlock: () => shortcodeData });
8152
process('foo bar', Map({ [editorComponent.id]: editorComponent }), processEat);
8253
expect(processEat).toHaveBeenCalledWith(expectedNode);
8354
});
8455
});
8556
});
86-
87-
describe('getLinesWithOffsets', () => {
88-
test('should split into lines', () => {
89-
const value = ' line1\n\nline2 \n\n line3 \n\n';
90-
91-
const lines = getLinesWithOffsets(value);
92-
expect(lines).toEqual([
93-
{ line: ' line1', start: 0 },
94-
{ line: 'line2', start: 8 },
95-
{ line: ' line3', start: 16 },
96-
{ line: '', start: 30 },
97-
]);
98-
});
99-
100-
test('should return single item on no match', () => {
101-
const value = ' line1 ';
102-
103-
const lines = getLinesWithOffsets(value);
104-
expect(lines).toEqual([{ line: ' line1', start: 0 }]);
105-
});
106-
});

‎packages/decap-cms-widget-markdown/src/serializers/remarkShortcodes.js

Lines changed: 21 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -8,65 +8,32 @@ export function remarkParseShortcodes({ plugins }) {
88
methods.unshift('shortcode');
99
}
1010

11-
export function getLinesWithOffsets(value) {
12-
const SEPARATOR = '\n\n';
13-
const splitted = value.split(SEPARATOR);
14-
const trimmedLines = splitted
15-
.reduce(
16-
(acc, line) => {
17-
const { start: previousLineStart, originalLength: previousLineOriginalLength } =
18-
acc[acc.length - 1];
19-
20-
return [
21-
...acc,
22-
{
23-
line: line.trimEnd(),
24-
start: previousLineStart + previousLineOriginalLength + SEPARATOR.length,
25-
originalLength: line.length,
26-
},
27-
];
28-
},
29-
[{ start: -SEPARATOR.length, originalLength: 0 }],
30-
)
31-
.slice(1)
32-
.map(({ line, start }) => ({ line, start }));
33-
return trimmedLines;
34-
}
35-
36-
function matchFromLines({ trimmedLines, plugin }) {
37-
for (const { line, start } of trimmedLines) {
38-
const match = line.match(plugin.pattern);
39-
if (match) {
40-
match.index += start;
41-
return match;
42-
}
43-
}
44-
}
45-
4611
function createShortcodeTokenizer({ plugins }) {
12+
plugins.forEach(plugin => {
13+
if (plugin.pattern.flags.includes('m')) {
14+
console.warn(
15+
`Invalid RegExp: editor component '${plugin.id}' must not use the multiline flag in its pattern.`,
16+
);
17+
}
18+
});
4719
return function tokenizeShortcode(eat, value, silent) {
48-
// Plugin patterns may rely on `^` and `$` tokens, even if they don't
49-
// use the multiline flag. To support this, we fall back to searching
50-
// through each line individually, trimming trailing whitespace and
51-
// newlines, if we don't initially match on a pattern. We keep track of
52-
// the starting position of each line so that we can sort correctly
53-
// across the full multiline matches.
54-
const trimmedLines = getLinesWithOffsets(value);
20+
let match;
21+
const potentialMatchValue = value.split('\n\n')[0].trimEnd();
22+
const plugin = plugins.find(plugin => {
23+
match = value.match(plugin.pattern);
24+
if (!match) {
25+
match = potentialMatchValue.match(plugin.pattern);
26+
}
5527

56-
// Attempt to find a regex match for each plugin's pattern, and then
57-
// select the first by its occurrence in `value`. This ensures we won't
58-
// skip a plugin that occurs later in the plugin registry, but earlier
59-
// in the `value`.
60-
const [{ plugin, match } = {}] = plugins
61-
.toArray()
62-
.map(plugin => ({
63-
match: value.match(plugin.pattern) || matchFromLines({ trimmedLines, plugin }),
64-
plugin,
65-
}))
66-
.filter(({ match }) => !!match)
67-
.sort((a, b) => a.match.index - b.match.index);
28+
return !!match;
29+
});
6830

6931
if (match) {
32+
if (match.index > 0) {
33+
console.warn(
34+
`Invalid RegExp: editor component '${plugin.id}' must match from the beginning of the block.`,
35+
);
36+
}
7037
if (silent) {
7138
return true;
7239
}

0 commit comments

Comments
 (0)