Trim trailing commas from semantic tokens output to ensure valid JSON (#1113)

* Trim trailing commas from semantic tokens output to ensure valid JSON

* Add CHANGELOG entry

Author: mediremi

CHANGELOG.md

@@ -19,6 +19,8 @@
 
 #### :bug: Bug fix
 
+- Fix: `rescript-editor-analysis.exe semanticTokens` sometimes returned invalid JSON, which affected syntax highlighting. https://github.com/rescript-lang/rescript-vscode/pull/1113
+
  - Fix: hang in `rescript-editor-analysis.exe codeAction` that sometimes prevented ReScript files from being saved in VS Code. https://github.com/rescript-lang/rescript-vscode/pull/1112
 
 - Fix: show existing compiler errors and warnings on file open. https://github.com/rescript-lang/rescript-vscode/pull/1103

analysis/src/SemanticTokens.ml

@@ -79,6 +79,11 @@ module Token = struct
        ^ string_of_int length ^ "," ^ tokenTypeToString type_ ^ ","
        ^ tokenModifiersString)
 
+  let remove_trailing_comma buffer =
+    let len = Buffer.length buffer in
+    if len > 0 && Buffer.nth buffer (len - 1) = ',' then
+      Buffer.truncate buffer (len - 1)
+
   let emit e =
     let sortedTokens =
       e.tokens
@@ -87,6 +92,10 @@ module Token = struct
     in
     let buf = Buffer.create 1 in
     sortedTokens |> List.iter (fun t -> e |> emitToken buf t);
+
+    (* Valid JSON arrays cannot have trailing commas *)
+    remove_trailing_comma buf;
+
     Buffer.contents buf
 end