fix: Rust identifiers normally recognised now; generics highlight support; doc comments support (#4868)

Author: mkslanc

demo/kitchen-sink/docs/rust.rs

@@ -18,3 +18,41 @@ fn map<T, U>(vector: &[T], function: &fn(v: &T) -> U) -> ~[U] {
     }
     return accumulator;
 }
+
+struct ConstGenericStruct<const N: usize>([(); N]);
+// T constrains by being an argument to GenericTrait.
+impl<T> GenericTrait<T> for i32 { /* ... */ }
+
+// T constrains by being an arguement to GenericStruct
+impl<T> Trait for GenericStruct<T> { /* ... */ }
+
+// Likewise, N constrains by being an argument to ConstGenericStruct
+impl<const N: usize> Trait for ConstGenericStruct<N> { /* ... */ }
+
+// T constrains by being in an associated type in a bound for type `U` which is
+// itself a generic parameter constraining the trait.
+impl<T, U> GenericTrait<U> for u32 where U: HasAssocType<Ty = T> { /* ... */ }
+
+// Like previous, except the type is `(U, isize)`. `U` appears inside the type
+// that includes `T`, and is not the type itself.
+impl<T, U> GenericStruct<U> where (U, isize): HasAssocType<Ty = T> { /* ... */ }
+
+//!  - Inner line doc
+//!! - Still an inner line doc (but with a bang at the beginning)
+
+/*!  - Inner block doc */
+/*!! - Still an inner block doc (but with a bang at the beginning) */
+
+/**  - Outer block doc (exactly) 2 asterisks */
+
+macro_rules! mac_variant {
+    ($vis:vis $name:ident) => {
+        enum $name {
+            $vis Unit,
+
+            $vis Tuple(u8, u16),
+
+            $vis Struct { f: u8 },
+        }
+    }
+}

src/mode/_test/tokens_rust.json

@@ -3,7 +3,7 @@
   ["keyword.source.rust","use"],
   ["text"," "],
   ["support.constant","core::rand::"],
-  ["text","RngUtil"],
+  ["identifier","RngUtil"],
   ["punctuation.operator",";"]
 ],[
    "start"
@@ -31,31 +31,36 @@
   ["string.quoted.double.source.rust","\"Carol\""],
   ["paren.rparen","]"],
   ["punctuation.operator","."],
-  ["text","each "],
+  ["identifier","each"],
+  ["text"," "],
   ["keyword.operator","|&"],
-  ["text","name"],
+  ["identifier","name"],
   ["keyword.operator","|"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
    "start",
   ["text","        "],
   ["keyword.source.rust","do"],
-  ["text"," spawn "],
+  ["text"," "],
+  ["identifier","spawn"],
+  ["text"," "],
   ["paren.lparen","{"]
 ],[
    "start",
   ["text","            "],
   ["keyword.source.rust","let"],
-  ["text"," v "],
+  ["text"," "],
+  ["identifier","v"],
+  ["text"," "],
   ["keyword.operator","="],
   ["text"," "],
   ["support.constant","rand::"],
-  ["text","Rng"],
+  ["identifier","Rng"],
   ["paren.lparen","("],
   ["paren.rparen",")"],
   ["punctuation.operator","."],
-  ["text","shuffle"],
+  ["identifier","shuffle"],
   ["paren.lparen","(["],
   ["constant.numeric.source.rust","1"],
   ["punctuation.operator",","],
@@ -70,28 +75,34 @@
    "start",
   ["text","            "],
   ["keyword.source.rust","for"],
-  ["text"," v"],
+  ["text"," "],
+  ["identifier","v"],
   ["punctuation.operator","."],
-  ["text","each "],
+  ["identifier","each"],
+  ["text"," "],
   ["keyword.operator","|&"],
-  ["text","num"],
+  ["identifier","num"],
   ["keyword.operator","|"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
    "start",
-  ["text","                print"],
+  ["text","                "],
+  ["identifier","print"],
   ["paren.lparen","("],
-  ["text","fmt"],
+  ["identifier","fmt"],
   ["keyword.operator","!"],
   ["paren.lparen","("],
   ["string.quoted.double.source.rust","\"%s says: '%d'"],
-  ["constant.character.escape.source.rust","\\n\\\\"],
+  ["constant.character.escape.source.rust","\\n"],
   ["string.quoted.double.source.rust","\""],
   ["punctuation.operator",","],
-  ["text"," name"],
+  ["text"," "],
+  ["identifier","name"],
   ["punctuation.operator",","],
-  ["text"," num "],
+  ["text"," "],
+  ["identifier","num"],
+  ["text"," "],
   ["keyword.operator","+"],
   ["text"," "],
   ["constant.numeric.source.rust","1"],
@@ -112,20 +123,31 @@
    "start"
 ],[
    "start",
+  ["text","    "],
   ["keyword.source.rust","let"],
-  ["text"," _ "],
+  ["text"," "],
+  ["identifier","_"],
   ["punctuation.operator",":"],
+  ["text"," "],
   ["storage.type.source.rust","i128"],
-  ["keyword.operator","=-"],
+  ["text"," "],
+  ["keyword.operator","="],
+  ["text"," "],
+  ["keyword.operator","-"],
   ["constant.numeric.source.rust","42i128"],
   ["punctuation.operator",";"]
 ],[
    "start",
+  ["text","    "],
   ["keyword.source.rust","let"],
-  ["text"," _ "],
+  ["text"," "],
+  ["identifier","_"],
   ["punctuation.operator",":"],
+  ["text"," "],
   ["storage.type.source.rust","u128"],
+  ["text"," "],
   ["keyword.operator","="],
+  ["text"," "],
   ["constant.numeric.source.rust","42u128"],
   ["punctuation.operator",";"]
 ],[
@@ -152,41 +174,44 @@
   ["keyword.source.rust","fn"],
   ["text"," "],
   ["entity.name.function.source.rust","map"],
-  ["keyword.operator","<"],
-  ["text","T"],
+  ["punctuation","<"],
+  ["identifier","T"],
   ["punctuation.operator",","],
-  ["text"," U"],
-  ["keyword.operator",">"],
+  ["text"," "],
+  ["identifier","U"],
+  ["punctuation",">"],
   ["paren.lparen","("],
-  ["text","vector"],
+  ["identifier","vector"],
   ["punctuation.operator",":"],
   ["text"," "],
   ["keyword.operator","&"],
   ["paren.lparen","["],
-  ["text","T"],
+  ["identifier","T"],
   ["paren.rparen","]"],
   ["punctuation.operator",","],
-  ["text"," function"],
+  ["text"," "],
+  ["identifier","function"],
   ["punctuation.operator",":"],
   ["text"," "],
   ["keyword.operator","&"],
-  ["text","fn"],
+  ["identifier","fn"],
   ["paren.lparen","("],
-  ["text","v"],
+  ["identifier","v"],
   ["punctuation.operator",":"],
   ["text"," "],
   ["keyword.operator","&"],
-  ["text","T"],
+  ["identifier","T"],
   ["paren.rparen",")"],
   ["text"," "],
   ["keyword.operator","->"],
-  ["text"," U"],
+  ["text"," "],
+  ["identifier","U"],
   ["paren.rparen",")"],
   ["text"," "],
   ["keyword.operator","->"],
   ["text"," ~"],
   ["paren.lparen","["],
-  ["text","U"],
+  ["identifier","U"],
   ["paren.rparen","]"],
   ["text"," "],
   ["paren.lparen","{"]
@@ -196,7 +221,9 @@
   ["keyword.source.rust","let"],
   ["text"," "],
   ["keyword.source.rust","mut"],
-  ["text"," accumulator "],
+  ["text"," "],
+  ["identifier","accumulator"],
+  ["text"," "],
   ["keyword.operator","="],
   ["text"," ~"],
   ["paren.lparen","["],
@@ -208,25 +235,26 @@
   ["keyword.source.rust","for"],
   ["text"," "],
   ["support.constant","vec::"],
-  ["text","each"],
+  ["identifier","each"],
   ["paren.lparen","("],
-  ["text","vector"],
+  ["identifier","vector"],
   ["paren.rparen",")"],
   ["text"," "],
   ["keyword.operator","|"],
-  ["text","element"],
+  ["identifier","element"],
   ["keyword.operator","|"],
   ["text"," "],
   ["paren.lparen","{"]
 ],[
    "start",
-  ["text","        accumulator"],
+  ["text","        "],
+  ["identifier","accumulator"],
   ["punctuation.operator","."],
-  ["text","push"],
+  ["identifier","push"],
   ["paren.lparen","("],
-  ["text","function"],
+  ["identifier","function"],
   ["paren.lparen","("],
-  ["text","element"],
+  ["identifier","element"],
   ["paren.rparen","))"],
   ["punctuation.operator",";"]
 ],[
@@ -237,7 +265,8 @@
    "start",
   ["text","    "],
   ["keyword.source.rust","return"],
-  ["text"," accumulator"],
+  ["text"," "],
+  ["identifier","accumulator"],
   ["punctuation.operator",";"]
 ],[
    "start",
@@ -276,24 +305,37 @@
    "start",
   ["constant.numeric.source.rust","14"],
   ["punctuation.operator","."],
-  ["text","_E"],
+  ["identifier","_E"],
   ["keyword.operator","-"],
   ["constant.numeric.source.rust","111_f64"],
   ["punctuation.operator",";"],
-  ["text","0xi32"],
+  ["text"," 0"],
+  ["identifier","xi32"],
   ["punctuation.operator",";"],
-  ["text","0b777u"]
+  ["text"," 0"],
+  ["identifier","b777u"]
+],[
+   "start"
+],[
+   "start",
+  ["comment.line.double-dash.source.rust","// identifiers ending in constant.numeric"]
 ],[
    "start",
-  ["text","foo1"],
+  ["identifier","foo1"],
   ["punctuation.operator",";"],
-  ["text","foo1u32"],
+  ["text"," "],
+  ["identifier","foo1u32"],
   ["punctuation.operator",";"],
-  ["text","foo1f32"],
+  ["text"," "],
+  ["identifier","foo1f32"],
   ["punctuation.operator",";"],
-  ["text","foo0xF"],
+  ["text"," "],
+  ["identifier","foo0xF"],
   ["punctuation.operator",";"],
-  ["text","foo1"],
+  ["text"," "],
+  ["identifier","foo1"],
   ["punctuation.operator","."],
   ["constant.numeric.source.rust","0"]
-]]
+],[
+   "start"
+]]