@@ -331,7 +331,6 @@ fn void Tokenizer.lex_internal(Tokenizer* t, Token* result) {
331
331
if (t.raw_mode) {
332
332
result.kind = Kind.Invalid;
333
333
string.memcpy(result.invalid, t.cur, len);
334
- result.invalid[len] = '\0';
335
334
t.cur += len;
336
335
return;
337
336
}
@@ -341,7 +340,6 @@ fn void Tokenizer.lex_internal(Tokenizer* t, Token* result) {
341
340
if (t.raw_mode) {
342
341
result.kind = Kind.Invalid;
343
342
result.invalid[0] = *t.cur;
344
- result.invalid[1] = '\0';
345
343
t.cur += 1;
346
344
return;
347
345
}
@@ -410,7 +408,6 @@ fn void Tokenizer.lex_internal(Tokenizer* t, Token* result) {
410
408
if (t.raw_mode) {
411
409
result.kind = Kind.Invalid;
412
410
result.invalid[0] = *t.cur;
413
- result.invalid[1] = '\0';
414
411
t.cur += 1;
415
412
return;
416
413
}
@@ -664,21 +661,6 @@ public fn Token Tokenizer.lookahead(Tokenizer* t, u32 n) {
664
661
return t.next[slot];
665
662
}
666
663
667
- /*
668
- public fn u32 Tokenizer.get_indent(const Tokenizer* t, const Token* tok) {
669
- if (tok.loc <= t.loc_start) return 0;
670
- u32 offset = tok.loc - t.loc_start;
671
- const char* cp = t.input_start + offset;
672
- u32 indent = 0;
673
- while (cp > t.input_start) {
674
- if (*cp == '\n') break;
675
- cp--;
676
- indent++;
677
- }
678
- return indent -1;
679
- }
680
- */
681
-
682
664
fn void Tokenizer.error(Tokenizer* t, Token* result, const char* format @(printf_format), ...) {
683
665
Va_list args;
684
666
va_start(args, format);
@@ -767,8 +749,6 @@ fn void Tokenizer.lex_number_error(Tokenizer* t, Token* result, const char *p, c
767
749
768
750
fn void Tokenizer.lex_number(Tokenizer* t, Token* result) {
769
751
result.kind = Kind.IntegerLiteral;
770
- result.radix = Radix.Default;
771
- result.int_value = 0;
772
752
const char* start = t.cur;
773
753
const char* p = start;
774
754
u64 value = 0;
0 commit comments