Skip to content

Commit a423fe6

Browse files
authored
Merge pull request #2802 from jasonlyu123/semantic-tokens
Semantic tokens for typescript
2 parents c1f0536 + 4e54f39 commit a423fe6

File tree

14 files changed

+535
-11
lines changed

14 files changed

+535
-11
lines changed

package.json

+13-1
Original file line numberDiff line numberDiff line change
@@ -516,6 +516,11 @@
516516
"default": true,
517517
"description": "Whether to automatic updating import path when rename or move a file"
518518
},
519+
"vetur.languageFeatures.semanticTokens": {
520+
"type": "boolean",
521+
"default": true,
522+
"description": "Whether to enable semantic highlighting. Currently only works for typescript"
523+
},
519524
"vetur.trace.server": {
520525
"type": "string",
521526
"enum": [
@@ -555,7 +560,14 @@
555560
"description": "Enable template interpolation service that offers hover / definition / references in Vue interpolations."
556561
}
557562
}
558-
}
563+
},
564+
"semanticTokenScopes": [
565+
{
566+
"scopes": {
567+
"property.refValue": ["entity.name.function"]
568+
}
569+
}
570+
]
559571
},
560572
"devDependencies": {
561573
"@rollup/plugin-commonjs": "^17.1.0",

server/src/config.ts

+3-1
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ export interface VLSConfig {
6565
languageFeatures: {
6666
codeActions: boolean;
6767
updateImportOnFileMove: boolean;
68+
semanticTokens: boolean;
6869
};
6970
trace: {
7071
server: 'off' | 'messages' | 'verbose';
@@ -128,7 +129,8 @@ export function getDefaultVLSConfig(): VLSFullConfig {
128129
},
129130
languageFeatures: {
130131
codeActions: true,
131-
updateImportOnFileMove: true
132+
updateImportOnFileMove: true,
133+
semanticTokens: true
132134
},
133135
trace: {
134136
server: 'off'

server/src/embeddedSupport/languageModes.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ import { getCSSMode, getSCSSMode, getLESSMode, getPostCSSMode } from '../modes/s
3131
import { getJavascriptMode } from '../modes/script/javascript';
3232
import { VueHTMLMode } from '../modes/template';
3333
import { getStylusMode } from '../modes/style/stylus';
34-
import { DocumentContext } from '../types';
34+
import { DocumentContext, SemanticTokenData } from '../types';
3535
import { VueInfoService } from '../services/vueInfoService';
3636
import { DependencyService } from '../services/dependencyService';
3737
import { nullMode } from '../modes/nullMode';
@@ -74,6 +74,7 @@ export interface LanguageMode {
7474
getColorPresentations?(document: TextDocument, color: Color, range: Range): ColorPresentation[];
7575
getFoldingRanges?(document: TextDocument): FoldingRange[];
7676
getRenameFileEdit?(renames: FileRename): TextDocumentEdit[];
77+
getSemanticTokens?(document: TextDocument, range?: Range): SemanticTokenData[];
7778

7879
onDocumentChanged?(filePath: string): void;
7980
onDocumentRemoved(document: TextDocument): void;

server/src/modes/script/javascript.ts

+82-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,13 @@ import { BasicComponentInfo, VLSFormatConfig } from '../../config';
4444
import { VueInfoService } from '../../services/vueInfoService';
4545
import { getComponentInfo } from './componentInfo';
4646
import { DependencyService, RuntimeLibrary } from '../../services/dependencyService';
47-
import { CodeActionData, CodeActionDataKind, OrganizeImportsActionData, RefactorActionData } from '../../types';
47+
import {
48+
CodeActionData,
49+
CodeActionDataKind,
50+
OrganizeImportsActionData,
51+
RefactorActionData,
52+
SemanticTokenOffsetData
53+
} from '../../types';
4854
import { IServiceHost } from '../../services/typescriptService/serviceHost';
4955
import {
5056
isVirtualVueTemplateFile,
@@ -57,11 +63,18 @@ import { isVCancellationRequested, VCancellationToken } from '../../utils/cancel
5763
import { EnvironmentService } from '../../services/EnvironmentService';
5864
import { getCodeActionKind } from './CodeActionKindConverter';
5965
import { FileRename } from 'vscode-languageserver';
66+
import {
67+
addCompositionApiRefTokens,
68+
getTokenModifierFromClassification,
69+
getTokenTypeFromClassification
70+
} from './semanticToken';
6071

6172
// Todo: After upgrading to LS server 4.0, use CompletionContext for filtering trigger chars
6273
// https://microsoft.github.io/language-server-protocol/specification#completion-request-leftwards_arrow_with_hook
6374
const NON_SCRIPT_TRIGGERS = ['<', '*', ':'];
6475

76+
const SEMANTIC_TOKEN_CONTENT_LENGTH_LIMIT = 80000;
77+
6578
export async function getJavascriptMode(
6679
serviceHost: IServiceHost,
6780
env: EnvironmentService,
@@ -788,6 +801,64 @@ export async function getJavascriptMode(
788801

789802
return textDocumentEdit;
790803
},
804+
getSemanticTokens(doc: TextDocument, range?: Range) {
805+
const { scriptDoc, service } = updateCurrentVueTextDocument(doc);
806+
const scriptText = scriptDoc.getText();
807+
if (scriptText.trim().length > SEMANTIC_TOKEN_CONTENT_LENGTH_LIMIT) {
808+
return [];
809+
}
810+
811+
const fileFsPath = getFileFsPath(doc.uri);
812+
const textSpan = range
813+
? convertTextSpan(range, scriptDoc)
814+
: {
815+
start: 0,
816+
length: scriptText.length
817+
};
818+
const { spans } = service.getEncodedSemanticClassifications(
819+
fileFsPath,
820+
textSpan,
821+
tsModule.SemanticClassificationFormat.TwentyTwenty
822+
);
823+
824+
const data: SemanticTokenOffsetData[] = [];
825+
let index = 0;
826+
827+
while (index < spans.length) {
828+
// [start, length, encodedClassification, start2, length2, encodedClassification2]
829+
const start = spans[index++];
830+
const length = spans[index++];
831+
const encodedClassification = spans[index++];
832+
const classificationType = getTokenTypeFromClassification(encodedClassification);
833+
if (classificationType < 0) {
834+
continue;
835+
}
836+
837+
const modifierSet = getTokenModifierFromClassification(encodedClassification);
838+
839+
data.push({
840+
start,
841+
length,
842+
classificationType,
843+
modifierSet
844+
});
845+
}
846+
847+
const program = service.getProgram();
848+
if (program) {
849+
addCompositionApiRefTokens(tsModule, program, fileFsPath, data);
850+
}
851+
852+
return data.map(({ start, ...rest }) => {
853+
const startPosition = scriptDoc.positionAt(start);
854+
855+
return {
856+
...rest,
857+
line: startPosition.line,
858+
character: startPosition.character
859+
};
860+
});
861+
},
791862
dispose() {
792863
jsDocuments.dispose();
793864
}
@@ -1114,3 +1185,13 @@ function getFoldingRangeKind(span: ts.OutliningSpan): FoldingRangeKind | undefin
11141185
return undefined;
11151186
}
11161187
}
1188+
1189+
function convertTextSpan(range: Range, doc: TextDocument): ts.TextSpan {
1190+
const start = doc.offsetAt(range.start);
1191+
const end = doc.offsetAt(range.end);
1192+
1193+
return {
1194+
start,
1195+
length: end - start
1196+
};
1197+
}
+149
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
import ts from 'typescript';
2+
import { SemanticTokensLegend, SemanticTokenModifiers, SemanticTokenTypes } from 'vscode-languageserver';
3+
import { RuntimeLibrary } from '../../services/dependencyService';
4+
import { SemanticTokenOffsetData } from '../../types';
5+
6+
/* tslint:disable:max-line-length */
7+
/**
8+
* extended from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L9
9+
* so that we don't have to map it into our own legend
10+
*/
11+
export const enum TokenType {
12+
class,
13+
enum,
14+
interface,
15+
namespace,
16+
typeParameter,
17+
type,
18+
parameter,
19+
variable,
20+
enumMember,
21+
property,
22+
function,
23+
member
24+
}
25+
26+
/* tslint:disable:max-line-length */
27+
/**
28+
* adopted from https://github.com/microsoft/TypeScript/blob/35c8df04ad959224fad9037e340c1e50f0540a49/src/services/classifier2020.ts#L13
29+
* so that we don't have to map it into our own legend
30+
*/
31+
export const enum TokenModifier {
32+
declaration,
33+
static,
34+
async,
35+
readonly,
36+
defaultLibrary,
37+
local,
38+
39+
// vue composition api
40+
refValue
41+
}
42+
43+
export function getSemanticTokenLegends(): SemanticTokensLegend {
44+
const tokenModifiers: string[] = [];
45+
46+
([
47+
[TokenModifier.declaration, SemanticTokenModifiers.declaration],
48+
[TokenModifier.static, SemanticTokenModifiers.static],
49+
[TokenModifier.async, SemanticTokenModifiers.async],
50+
[TokenModifier.readonly, SemanticTokenModifiers.readonly],
51+
[TokenModifier.defaultLibrary, SemanticTokenModifiers.defaultLibrary],
52+
[TokenModifier.local, 'local'],
53+
54+
// vue
55+
[TokenModifier.refValue, 'refValue']
56+
] as const).forEach(([tsModifier, legend]) => (tokenModifiers[tsModifier] = legend));
57+
58+
const tokenTypes: string[] = [];
59+
60+
([
61+
[TokenType.class, SemanticTokenTypes.class],
62+
[TokenType.enum, SemanticTokenTypes.enum],
63+
[TokenType.interface, SemanticTokenTypes.interface],
64+
[TokenType.namespace, SemanticTokenTypes.namespace],
65+
[TokenType.typeParameter, SemanticTokenTypes.typeParameter],
66+
[TokenType.type, SemanticTokenTypes.type],
67+
[TokenType.parameter, SemanticTokenTypes.parameter],
68+
[TokenType.variable, SemanticTokenTypes.variable],
69+
[TokenType.enumMember, SemanticTokenTypes.enumMember],
70+
[TokenType.property, SemanticTokenTypes.property],
71+
[TokenType.function, SemanticTokenTypes.function],
72+
73+
// member is renamed to method in vscode codebase to match LSP default
74+
[TokenType.member, SemanticTokenTypes.method]
75+
] as const).forEach(([tokenType, legend]) => (tokenTypes[tokenType] = legend));
76+
77+
return {
78+
tokenModifiers,
79+
tokenTypes
80+
};
81+
}
82+
83+
export function getTokenTypeFromClassification(tsClassification: number): number {
84+
return (tsClassification >> TokenEncodingConsts.typeOffset) - 1;
85+
}
86+
87+
export function getTokenModifierFromClassification(tsClassification: number) {
88+
return tsClassification & TokenEncodingConsts.modifierMask;
89+
}
90+
91+
const enum TokenEncodingConsts {
92+
typeOffset = 8,
93+
modifierMask = (1 << typeOffset) - 1
94+
}
95+
96+
export function addCompositionApiRefTokens(
97+
tsModule: RuntimeLibrary['typescript'],
98+
program: ts.Program,
99+
fileFsPath: string,
100+
exists: SemanticTokenOffsetData[]
101+
): void {
102+
const sourceFile = program.getSourceFile(fileFsPath);
103+
104+
if (!sourceFile) {
105+
return;
106+
}
107+
108+
const typeChecker = program.getTypeChecker();
109+
110+
walk(sourceFile, node => {
111+
if (!ts.isIdentifier(node) || node.text !== 'value' || !ts.isPropertyAccessExpression(node.parent)) {
112+
return;
113+
}
114+
const propertyAccess = node.parent;
115+
116+
let parentSymbol = typeChecker.getTypeAtLocation(propertyAccess.expression).symbol;
117+
118+
if (parentSymbol.flags & tsModule.SymbolFlags.Alias) {
119+
parentSymbol = typeChecker.getAliasedSymbol(parentSymbol);
120+
}
121+
122+
if (parentSymbol.name !== 'Ref') {
123+
return;
124+
}
125+
126+
const start = node.getStart();
127+
const length = node.getWidth();
128+
const exist = exists.find(token => token.start === start && token.length === length);
129+
const encodedModifier = 1 << TokenModifier.refValue;
130+
131+
if (exist) {
132+
exist.modifierSet |= encodedModifier;
133+
} else {
134+
exists.push({
135+
classificationType: TokenType.property,
136+
length: node.getEnd() - node.getStart(),
137+
modifierSet: encodedModifier,
138+
start: node.getStart()
139+
});
140+
}
141+
});
142+
}
143+
144+
function walk(node: ts.Node, callback: (node: ts.Node) => void) {
145+
node.forEachChild(child => {
146+
callback(child);
147+
walk(child, callback);
148+
});
149+
}

server/src/services/projectService.ts

+35-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@ import {
2222
FoldingRangeParams,
2323
Hover,
2424
Location,
25+
SemanticTokens,
26+
SemanticTokensBuilder,
27+
SemanticTokensParams,
28+
SemanticTokensRangeParams,
2529
SignatureHelp,
2630
SymbolInformation,
2731
TextDocumentEdit,
@@ -33,7 +37,7 @@ import { URI } from 'vscode-uri';
3337
import { LanguageId } from '../embeddedSupport/embeddedSupport';
3438
import { LanguageMode, LanguageModes } from '../embeddedSupport/languageModes';
3539
import { NULL_COMPLETION, NULL_HOVER, NULL_SIGNATURE } from '../modes/nullMode';
36-
import { DocumentContext, CodeActionData } from '../types';
40+
import { DocumentContext, CodeActionData, SemanticTokenData } from '../types';
3741
import { VCancellationToken } from '../utils/cancellationToken';
3842
import { getFileFsPath } from '../utils/paths';
3943
import { DependencyService } from './dependencyService';
@@ -60,6 +64,7 @@ export interface ProjectService {
6064
onCodeAction(params: CodeActionParams): Promise<CodeAction[]>;
6165
onCodeActionResolve(action: CodeAction): Promise<CodeAction>;
6266
onWillRenameFile(fileRename: FileRename): Promise<TextDocumentEdit[]>;
67+
onSemanticTokens(params: SemanticTokensParams | SemanticTokensRangeParams): Promise<SemanticTokens>;
6368
doValidate(doc: TextDocument, cancellationToken?: VCancellationToken): Promise<Diagnostic[] | null>;
6469
dispose(): Promise<void>;
6570
}
@@ -336,6 +341,35 @@ export async function createProjectService(
336341

337342
return textDocumentEdit ?? [];
338343
},
344+
async onSemanticTokens(params: SemanticTokensParams | SemanticTokensRangeParams) {
345+
if (!env.getConfig().vetur.languageFeatures.semanticTokens) {
346+
return {
347+
data: []
348+
};
349+
}
350+
351+
const { textDocument } = params;
352+
const range = 'range' in params ? params.range : undefined;
353+
const doc = documentService.getDocument(textDocument.uri)!;
354+
const modes = languageModes.getAllLanguageModeRangesInDocument(doc);
355+
const data: SemanticTokenData[] = [];
356+
357+
for (const mode of modes) {
358+
const tokenData = mode.mode.getSemanticTokens?.(doc, range);
359+
360+
data.push(...(tokenData ?? []));
361+
}
362+
363+
const builder = new SemanticTokensBuilder();
364+
const sorted = data.sort((a, b) => {
365+
return a.line - b.line || a.character - b.character;
366+
});
367+
sorted.forEach(token =>
368+
builder.push(token.line, token.character, token.length, token.classificationType, token.modifierSet)
369+
);
370+
371+
return builder.build();
372+
},
339373
async doValidate(doc: TextDocument, cancellationToken?: VCancellationToken) {
340374
const diagnostics: Diagnostic[] = [];
341375
if (doc.languageId === 'vue') {

0 commit comments

Comments
 (0)