syntax: fix multiple issues with escape sequences and character constants

* accept 8-bit octal espace sequences
* reject malformed hex espace sequences
* make 8-bit character literal values consistent with type `char`
* fix output of 8-bit octal and hex literals in `CharLiteral.printLiteral`
* fix formating bug in tools/c2cat.c2
* add tests for various issues

Author: chqrlie

analyser_utils/ctv_analyser.c2

@@ -88,7 +88,7 @@ public fn Value get_value(const Expr* e) {
         break;
     case CharLiteral:
         const CharLiteral* c = cast<CharLiteral*>(e);
-        result.uvalue = c.getValue();
+        result.uvalue = cast<u64>(c.getValue());  // cast required if char is signed
         break;
     case StringLiteral:
         assert(0);

ast/char_literal.c2

@@ -41,8 +41,8 @@ public fn CharLiteral* CharLiteral.create(ast_context.Context* c, SrcLoc loc, u8
     return e;
 }
 
-public fn u8 CharLiteral.getValue(const CharLiteral* e) {
-    return cast<u8>(e.parent.parent.charLiteralBits.value);
+public fn char CharLiteral.getValue(const CharLiteral* e) {
+    return cast<char>(e.parent.parent.charLiteralBits.value);
 }
 
 fn void CharLiteral.print(const CharLiteral* e, string_buffer.Buf* out, u32 indent) {
@@ -55,14 +55,14 @@ fn void CharLiteral.print(const CharLiteral* e, string_buffer.Buf* out, u32 inde
 }
 
 public fn void CharLiteral.printLiteral(const CharLiteral* e, string_buffer.Buf* out) {
-    char c = cast<char>(e.parent.parent.charLiteralBits.value);
+    u8 c = cast<u8>(e.parent.parent.charLiteralBits.value);
 
     switch (e.parent.parent.charLiteralBits.radix) {
     case 8:
         out.print("'\\%o'", c);
         return;
     case 16:
-        out.print("'\\x%x'", c);
+        out.print("'\\x%02x'", c);
         return;
     default:
         break;

generator/c_generator_pure_call.c2

@@ -69,7 +69,7 @@ fn Value Evaluator.get_value(Evaluator* eval, const Expr* e) {
         break;
     case CharLiteral:
         const CharLiteral* c = cast<CharLiteral*>(e);
-        result.uvalue = c.getValue();
+        result.uvalue = cast<u64>(c.getValue());   // cast required if char is signed
         break;
     case StringLiteral:
         assert(0);

parser/c2_tokenizer.c2

@@ -1005,25 +1005,37 @@ fn u32 Tokenizer.lex_escaped_char(Tokenizer* t, Token* result) {
         result.char_value = '\v';
         break;
     case 'x':
-        if (!isxdigit(input[1]) || !isxdigit(input[2])) {
+        if (!isxdigit(input[1])) {
             t.cur++;
             t.error(result, "expect hexadecimal number after '\\x'");
             return 0;
         }
+        // C consumes all hex digits after \x (at least one)
+        // C2 requires 2 hex digits, but rejects extra digits to simplify C generation
+        if (!isxdigit(input[2])) {
+            t.cur += 2;
+            t.error(result, "expect 2 hexadecimal digits after '\\x'");
+            return 0;
+        }
+        if (isxdigit(input[3])) {
+            t.cur += 3;
+            t.error(result, "too many digits in hexadecimal escape sequence '\\x'");
+            return 0;
+        }
         result.char_value = hex2val(input[1]) * 16 + hex2val(input[2]);
         result.radix = 16;
         return 3;
     default:
         if (is_octal(input[0])) {
             u32 offset = 0;
             u32 value = 0;
-            while (is_octal(input[offset]) && offset <= 2) {
+            while (is_octal(input[offset]) && offset < 3) {
                 value *= 8;
                 value += cast<u32>(input[offset] - '0');
                 offset++;
             }
 
-            if (value > 127) {
+            if (value > 255) {
                 t.cur++;
                 t.error(result, "octal escape sequence out of range");
                 return 0;

test/parser/char_escape_sequence.c2

@@ -17,6 +17,13 @@ char o1 = '\0';
 char o2 = '\07';
 char o3 = '\077';
 char o4 = '\177';
+char o5 = '\00';
+char o6 = '\000';
+char o7 = '\200';
+char o8 = '\377';
 
 char h1 = '\x1e';
+char h2 = '\x00';
+char h3 = '\x80';
+char h4 = '\xFF';
 

test/parser/char_hex_error1.c2

@@ -0,0 +1,5 @@
+// @warnings{no-unused}
+module test;
+
+char c = '\x';    // @error{expect hexadecimal number after '\x'}
+

test/parser/char_hex_error2.c2

@@ -0,0 +1,5 @@
+// @warnings{no-unused}
+module test;
+
+char c = '\xx';    // @error{expect hexadecimal number after '\x'}
+

test/parser/char_hex_error3.c2

@@ -0,0 +1,5 @@
+// @warnings{no-unused}
+module test;
+
+char c = '\x1';    // @error{expect 2 hexadecimal digits after '\x'}
+

test/parser/char_hex_error4.c2

@@ -0,0 +1,5 @@
+// @warnings{no-unused}
+module test;
+
+char c = '\x1x';    // @error{expect 2 hexadecimal digits after '\x'}
+

test/parser/char_hex_error5.c2

@@ -0,0 +1,5 @@
+// @warnings{no-unused}
+module test;
+
+char c = '\x012';    // @error{too many digits in hexadecimal escape sequence '\x'}
+

test/parser/char_hex_error6.c2

@@ -0,0 +1,5 @@
+// @warnings{no-unused}
+module test;
+
+char c = '\x123';    // @error{too many digits in hexadecimal escape sequence '\x'}
+

test/parser/char_octal_out_of_range.c2

@@ -1,5 +1,5 @@
 // @warnings{no-unused}
 module test;
 
-char c = '\200';    // @error{octal escape sequence out of range}
+char c = '\400';    // @error{octal escape sequence out of range}
 

tools/c2cat.c2

@@ -175,7 +175,7 @@ fn void print_token(const Token* tok) {
             len = sprintf(tmp, "'\\%o'", tok.char_value);
             break;
         case 16:
-            len = sprintf(tmp, "'\\x%%%x'", tok.char_value);
+            len = sprintf(tmp, "'\\x%02x'", tok.char_value);
             break;
         default:
             if (ctype.isprint(tok.char_value)) {