Skip to content

Commit b0ae8d7

Browse files
lionel-rowelionel-rowe
committed
fix(text-to-unicode): handle non-BMP + more conversion options
1 parent e876d03 commit b0ae8d7

File tree

3 files changed

+257
-28
lines changed

3 files changed

+257
-28
lines changed

‎src/tools/text-to-unicode/text-to-unicode.service.test.ts

Lines changed: 73 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
import { describe, expect, it } from 'vitest';
2-
import { convertTextToUnicode, convertUnicodeToText } from './text-to-unicode.service';
2+
import { type ConverterId, SKIP_PRINTABLE_ASCII_RE, converters } from './text-to-unicode.service';
3+
4+
describe('text-to-unicode (legacy tests)', () => {
5+
const convertTextToUnicode = converters.decimalEntities.escape;
6+
const convertUnicodeToText = converters.decimalEntities.unescape;
37

4-
describe('text-to-unicode', () => {
58
describe('convertTextToUnicode', () => {
69
it('a text string is converted to unicode representation', () => {
710
expect(convertTextToUnicode('A')).toBe('A');
@@ -18,3 +21,71 @@ describe('text-to-unicode', () => {
1821
});
1922
});
2023
});
24+
25+
describe('text-to-unicode', () => {
26+
interface TestConfig {
27+
text: string
28+
results: Record<ConverterId, string>
29+
skipPrintableAscii?: boolean
30+
};
31+
const tests: TestConfig[] = [
32+
{
33+
text: 'ABC',
34+
results: {
35+
fullUnicode: String.raw`\u0041\u0042\u0043`,
36+
utf16: String.raw`\u0041\u0042\u0043`,
37+
hexEntities: String.raw`&#x41;&#x42;&#x43;`,
38+
decimalEntities: String.raw`&#65;&#66;&#67;`,
39+
},
40+
},
41+
{
42+
text: 'ABC',
43+
skipPrintableAscii: true,
44+
results: {
45+
fullUnicode: 'ABC',
46+
utf16: 'ABC',
47+
hexEntities: 'ABC',
48+
decimalEntities: 'ABC',
49+
},
50+
},
51+
{
52+
text: '文字',
53+
results: {
54+
// eslint-disable-next-line unicorn/escape-case
55+
fullUnicode: String.raw`\u6587\u5b57`,
56+
// eslint-disable-next-line unicorn/escape-case
57+
utf16: String.raw`\u6587\u5b57`,
58+
hexEntities: String.raw`&#x6587;&#x5b57;`,
59+
decimalEntities: String.raw`&#25991;&#23383;`,
60+
},
61+
},
62+
{
63+
text: 'a 💩 b',
64+
skipPrintableAscii: true,
65+
results: {
66+
// eslint-disable-next-line unicorn/escape-case
67+
fullUnicode: String.raw`a \u{1f4a9} b`,
68+
// eslint-disable-next-line unicorn/escape-case
69+
utf16: String.raw`a \ud83d\udca9 b`,
70+
hexEntities: String.raw`a &#x1f4a9; b`,
71+
decimalEntities: String.raw`a &#128169; b`,
72+
},
73+
},
74+
];
75+
76+
for (const { text, skipPrintableAscii: skipAscii, results } of tests) {
77+
describe(`${text} (skipAscii=${skipAscii})`, () => {
78+
for (const [key, result] of Object.entries(results)) {
79+
describe(key, () => {
80+
const converter = converters[key as ConverterId];
81+
it('Escaping', () => {
82+
expect(converter.escape(text, skipAscii ? SKIP_PRINTABLE_ASCII_RE : undefined)).toBe(result);
83+
});
84+
it('Unescaping', () => {
85+
expect(converter.unescape(result)).toBe(text);
86+
});
87+
});
88+
}
89+
});
90+
}
91+
});

‎src/tools/text-to-unicode/text-to-unicode.service.ts

Lines changed: 91 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,95 @@
1-
function convertTextToUnicode(text: string): string {
2-
return text.split('').map(value => `&#${value.charCodeAt(0)};`).join('');
1+
// regex that never matches
2+
const SKIP_NOTHING_RE = /(\b\B)/;
3+
export const SKIP_PRINTABLE_ASCII_RE = /([ -~]+)/g;
4+
5+
function _codeUnits(text: string): number[] {
6+
return text.split('').map(char => char.codePointAt(0));
7+
}
8+
9+
function _codePoints(text: string): number[] {
10+
return [...text].map(char => char.codePointAt(0));
11+
}
12+
13+
export interface Converter {
14+
name: string
15+
escape(text: string, skip: RegExp): string
16+
unescape(text: string): string
17+
};
18+
19+
interface EscapeConfig {
20+
getCharValues?(text: string): number[]
21+
mapper(charValue: number): string
22+
};
23+
24+
function escaper({ getCharValues, mapper }: EscapeConfig) {
25+
/**
26+
* @param text text input to escape
27+
* @param skipper regular expression for content _not_ to escape. Must have exactly 1 capture group.
28+
*/
29+
return (text: string, skipper?: RegExp): string => {
30+
skipper ??= SKIP_NOTHING_RE;
31+
getCharValues ??= _codePoints;
32+
33+
return text
34+
.split(skipper)
35+
.flatMap((x, i) => {
36+
if (i % 2) {
37+
return x;
38+
}
39+
return getCharValues(x).map(mapper);
40+
})
41+
.join('');
42+
};
43+
}
44+
45+
interface UnescapeConfig {
46+
regex: RegExp
47+
radix: number
48+
};
49+
50+
function unescaper({ regex, radix }: UnescapeConfig) {
51+
return (escaped: string): string => {
52+
return escaped.replace(regex, (match) => {
53+
return String.fromCodePoint(Number.parseInt(match.replace(/\P{AHex}/gu, ''), radix));
54+
});
55+
};
56+
}
57+
58+
export type ConverterId = keyof typeof converters;
59+
const converters = {
60+
fullUnicode: {
61+
name: 'Full Unicode',
62+
escape: escaper({ mapper: convertCodePointToUnicode }),
63+
unescape: unescaper({ regex: /\\u\p{AHex}{4}|\\u\{\p{AHex}{1,6}\}/gu, radix: 16 }),
64+
},
65+
utf16: {
66+
name: 'UTF-16 Code Units',
67+
escape: escaper({ getCharValues: _codeUnits, mapper: convertCodePointToUnicode }),
68+
unescape: unescaper({ regex: /\\u\p{AHex}{4}/gu, radix: 16 }),
69+
},
70+
hexEntities: {
71+
name: 'HTML Entities (Hex)',
72+
escape: escaper({ mapper: toHexEntities }),
73+
unescape: unescaper({ regex: /&#x\p{AHex}{1,6};/gu, radix: 16 }),
74+
},
75+
decimalEntities: {
76+
name: 'HTML Entities (Decimal)',
77+
escape: escaper({ mapper: toDecimalEntities }),
78+
unescape: unescaper({ regex: /&#\d+;/gu, radix: 10 }),
79+
},
80+
} satisfies Record<string, Converter>;
81+
82+
function convertCodePointToUnicode(codePoint: number): string {
83+
const hex = codePoint.toString(16);
84+
return hex.length > 4 ? String.raw`\u{${hex}}` : String.raw`\u${hex.padStart(4, '0')}`;
85+
}
86+
87+
function toHexEntities(codePoint: number): string {
88+
return `&#x${codePoint.toString(16)};`;
389
}
490

5-
function convertUnicodeToText(unicodeStr: string): string {
6-
return unicodeStr.replace(/&#(\d+);/g, (match, dec) => String.fromCharCode(dec));
91+
function toDecimalEntities(codePoint: number): string {
92+
return `&#${codePoint};`;
793
}
894

9-
export { convertTextToUnicode, convertUnicodeToText };
95+
export { converters };

‎src/tools/text-to-unicode/text-to-unicode.vue

Lines changed: 93 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,106 @@
11
<script setup lang="ts">
2-
import { convertTextToUnicode, convertUnicodeToText } from './text-to-unicode.service';
2+
import { type ConverterId, SKIP_PRINTABLE_ASCII_RE, converters } from './text-to-unicode.service';
33
import { useCopy } from '@/composable/copy';
44
5+
const converterId = ref<ConverterId>('fullUnicode');
6+
const skipAscii = ref(true);
7+
58
const inputText = ref('');
6-
const unicodeFromText = computed(() => inputText.value.trim() === '' ? '' : convertTextToUnicode(inputText.value));
9+
const unicodeFromText = computed(() =>
10+
inputText.value.trim() === ''
11+
? ''
12+
: converters[converterId.value].escape(inputText.value, skipAscii.value ? SKIP_PRINTABLE_ASCII_RE : undefined),
13+
);
714
const { copy: copyUnicode } = useCopy({ source: unicodeFromText });
815
916
const inputUnicode = ref('');
10-
const textFromUnicode = computed(() => inputUnicode.value.trim() === '' ? '' : convertUnicodeToText(inputUnicode.value));
17+
const textFromUnicode = computed(() =>
18+
inputUnicode.value.trim() === '' ? '' : converters[converterId.value].unescape(inputUnicode.value),
19+
);
1120
const { copy: copyText } = useCopy({ source: textFromUnicode });
1221
</script>
1322

1423
<template>
15-
<c-card title="Text to Unicode">
16-
<c-input-text v-model:value="inputText" multiline placeholder="e.g. 'Hello Avengers'" label="Enter text to convert to unicode" autosize autofocus raw-text test-id="text-to-unicode-input" />
17-
<c-input-text v-model:value="unicodeFromText" label="Unicode from your text" multiline raw-text readonly mt-2 placeholder="The unicode representation of your text will be here" test-id="text-to-unicode-output" />
18-
<div mt-2 flex justify-center>
19-
<c-button :disabled="!unicodeFromText" @click="copyUnicode()">
20-
Copy unicode to clipboard
21-
</c-button>
22-
</div>
23-
</c-card>
24-
25-
<c-card title="Unicode to Text">
26-
<c-input-text v-model:value="inputUnicode" multiline placeholder="Input Unicode" label="Enter unicode to convert to text" autosize raw-text test-id="unicode-to-text-input" />
27-
<c-input-text v-model:value="textFromUnicode" label="Text from your Unicode" multiline raw-text readonly mt-2 placeholder="The text representation of your unicode will be here" test-id="unicode-to-text-output" />
28-
<div mt-2 flex justify-center>
29-
<c-button :disabled="!textFromUnicode" @click="copyText()">
30-
Copy text to clipboard
31-
</c-button>
24+
<div class="outer" flex flex-col gap-6>
25+
<div class="controls">
26+
<c-select
27+
v-model:value="converterId"
28+
searchable
29+
label="Conversion type:"
30+
:options="Object.entries(converters).map(([key, val]) => ({ label: val.name, value: key }))"
31+
/>
3232
</div>
33-
</c-card>
33+
<c-card class="card" title="Text to Unicode">
34+
<c-input-text
35+
v-model:value="inputText"
36+
multiline
37+
placeholder="e.g. 'Hello Avengers'"
38+
label="Enter text to convert to Unicode"
39+
autosize
40+
autofocus
41+
raw-text
42+
test-id="text-to-unicode-input"
43+
/>
44+
<c-input-text
45+
v-model:value="unicodeFromText"
46+
label="Unicode from your text"
47+
multiline
48+
raw-text
49+
readonly
50+
mt-2
51+
placeholder="The unicode representation of your text will be here"
52+
test-id="text-to-unicode-output"
53+
/>
54+
<div mt-2 flex justify-start>
55+
<n-form-item label="Skip ASCII?" :show-feedback="false" label-placement="left">
56+
<n-switch v-model:value="skipAscii" />
57+
</n-form-item>
58+
</div>
59+
<div mt-2 flex justify-center>
60+
<c-button :disabled="!unicodeFromText" @click="copyUnicode()"> Copy unicode to clipboard </c-button>
61+
</div>
62+
</c-card>
63+
<c-card class="card" title="Unicode to Text">
64+
<c-input-text
65+
v-model:value="inputUnicode"
66+
multiline
67+
placeholder="Input Unicode"
68+
label="Enter unicode to convert to text"
69+
autosize
70+
raw-text
71+
test-id="unicode-to-text-input"
72+
/>
73+
<c-input-text
74+
v-model:value="textFromUnicode"
75+
label="Text from your Unicode"
76+
multiline
77+
raw-text
78+
readonly
79+
mt-2
80+
placeholder="The text representation of your unicode will be here"
81+
test-id="unicode-to-text-output"
82+
/>
83+
<div mt-2 flex justify-center>
84+
<c-button :disabled="!textFromUnicode" @click="copyText()"> Copy text to clipboard </c-button>
85+
</div>
86+
</c-card>
87+
</div>
3488
</template>
89+
90+
<style lang="less" scoped>
91+
.outer {
92+
flex: 0 1 1200px;
93+
margin-inline: 50px;
94+
display: flex;
95+
flex-direction: row;
96+
flex-wrap: wrap;
97+
}
98+
99+
.controls {
100+
flex: 0 1 100%;
101+
}
102+
103+
.card {
104+
flex: 1 0 max(40%, 500px);
105+
}
106+
</style>

0 commit comments

Comments
 (0)