fix: not closed generic state in case of left-shift operator (#5572)

Author: mkslanc

src/mode/_test/text_rust.txt

@@ -34,3 +34,8 @@ fn map<T, U>(vector: &[T], function: &fn(v: &T) -> U) -> ~[U] {
 
 // identifiers ending in constant.numeric
 foo1; foo1u32; foo1f32; foo0xF; foo1.0
+
+
+pub fn g<T>() -> std::mem::MaybeUninit<[T; 1 << 2]> {
+    std::mem::MaybeUninit::uninit()
+}

src/mode/_test/tokens_rust.json

@@ -338,4 +338,49 @@
   ["constant.numeric.source.rust","0"]
 ],[
    "start"
+],[
+   "start"
+],[
+   "start",
+  ["keyword.source.rust","pub"],
+  ["text"," "],
+  ["keyword.source.rust","fn"],
+  ["text"," "],
+  ["entity.name.function.source.rust","g"],
+  ["punctuation","<"],
+  ["identifier","T"],
+  ["punctuation",">"],
+  ["paren.lparen","("],
+  ["paren.rparen",")"],
+  ["text"," "],
+  ["keyword.operator","->"],
+  ["text"," "],
+  ["support.constant","std::mem::"],
+  ["identifier","MaybeUninit"],
+  ["punctuaction","<"],
+  ["paren.lparen","["],
+  ["identifier","T"],
+  ["punctuation.operator",";"],
+  ["text"," "],
+  ["constant.numeric.source.rust","1"],
+  ["text"," "],
+  ["keyword.operator","<<"],
+  ["text"," "],
+  ["constant.numeric.source.rust","2"],
+  ["paren.rparen","]"],
+  ["punctuation",">"],
+  ["text"," "],
+  ["paren.lparen","{"]
+],[
+   "start",
+  ["text","    "],
+  ["support.constant","std::mem::MaybeUninit::"],
+  ["identifier","uninit"],
+  ["paren.lparen","("],
+  ["paren.rparen",")"]
+],[
+   "start",
+  ["paren.rparen","}"]
+],[
+   "start"
 ]]

src/mode/rust_highlight_rules.js

@@ -85,14 +85,14 @@ var RustHighlightRules = function() {
                 ]
             }, {
                 token: ['keyword.source.rust', 'text', 'entity.name.function.source.rust', 'punctuation'],
-                regex: '\\b(fn)(\\s+)((?:r#)?' + wordPattern + ')(<)',
+                regex: '\\b(fn)(\\s+)((?:r#)?' + wordPattern + ')(<)(?!<)',
                 push: "generics"
             }, {
                 token: ['keyword.source.rust', 'text', 'entity.name.function.source.rust'],
                 regex: '\\b(fn)(\\s+)((?:r#)?' + wordPattern + ')'
             }, {
                 token: ['support.constant', "punctuation"],
-                regex: "(" + wordPattern + '::)(<)',
+                regex: "(" + wordPattern + '::)(<)(?!<)',
                 push: "generics"
             }, {
                 token: 'support.constant',
@@ -126,12 +126,15 @@ var RustHighlightRules = function() {
                 ]
             }, {
                 token: ["keyword.source.rust", "identifier", "punctuaction"],
-                regex: "(?:(impl)|(" + wordPattern + "))(<)",
+                regex: "(?:(impl)|(" + wordPattern + "))(<)(?!<)",
                 stateName: 'generics',
                 push: [
                     {
+                        token: 'keyword.operator',
+                        regex: /<<|=/
+                    }, {
                         token: "punctuaction",
-                        regex: "<",
+                        regex: "<(?!<)",
                         push: "generics"
                     }, {
                         token: 'variable.other.source.rust', // `(?![\\\'])` to keep a lifetime name highlighting from continuing one character
@@ -141,51 +144,55 @@ var RustHighlightRules = function() {
                     }, {
                         token: "storage.type.source.rust",
                         regex: "\\b(u8|u16|u32|u64|u128|usize|i8|i16|i32|i64|i128|isize|char|bool)\\b"
-                    }, {
-                        token: "punctuation.operator",
-                        regex: "[,:]"
                     }, {
                         token: "keyword",
                         regex: "\\b(?:const|dyn)\\b"
                     }, {
                         token: "punctuation",
                         regex: ">",
                         next: "pop"
-                    }, {
-                        token: "paren.lparen",
-                        regex: "[(]"
-                    }, {
-                        token: "paren.rparen",
-                        regex: "[)]"
-                    }, {
+                    }, 
+                    {include: "punctuation"},
+                    {include: "operators"},
+                    {include: "constants"},
+                    {
                         token: "identifier",
                         regex: "\\b"+wordPattern+"\\b"
-                    }, {
-                        token: 'keyword.operator',
-                        regex: "="
                     }
                 ]
             }, {
                 token: keywordMapper,
                 regex: wordPattern
             }, {
-                token: 'keyword.operator', // `[*/](?![*/])=?` is separated because `//` and `/* */` become comments and must be
-                // guarded against. This states either `*` or `/` may be matched as long as the match
-                // it isn't followed by either of the two. An `=` may be on the end.
-                regex: /\$|[-=]>|[-+%^=!&|<>]=?|[*/](?![*/])=?/
-            }, {
-                token: "punctuation.operator",
-                regex: /[?:,;.]/
-            }, {
+                token: 'meta.preprocessor.source.rust',
+                regex: '\\b\\w\\(\\w\\)*!|#\\[[\\w=\\(\\)_]+\\]\\b'
+            }, 
+            {include: "punctuation"},
+            {include: "operators"},
+            {include: "constants"}
+        ],
+        punctuation: [
+            {
                 token: "paren.lparen",
                 regex: /[\[({]/
             }, {
                 token: "paren.rparen",
                 regex: /[\])}]/
             }, {
-                token: 'meta.preprocessor.source.rust',
-                regex: '\\b\\w\\(\\w\\)*!|#\\[[\\w=\\(\\)_]+\\]\\b'
-            }, {
+                token: "punctuation.operator",
+                regex: /[?:,;.]/
+            }
+        ],
+        operators: [
+            {
+                token: 'keyword.operator', // `[*/](?![*/])=?` is separated because `//` and `/* */` become comments and must be
+                // guarded against. This states either `*` or `/` may be matched as long as the match
+                // it isn't followed by either of the two. An `=` may be on the end.
+                regex: /\$|[-=]>|[-+%^=!&|<>]=?|[*/](?![*/])=?/
+            }
+        ],
+        constants: [
+            {
                 token: 'constant.numeric.source.rust',
                 regex: /\b(?:0x[a-fA-F0-9_]+|0o[0-7_]+|0b[01_]+|[0-9][0-9_]*(?!\.))(?:[iu](?:size|8|16|32|64|128))?\b/
             }, {