feat(frontend): Enhance parsing error reporting by providing location information. (risingwavelabs#8646)

Author: chenzl25

src/sqlparser/examples/parse.rs

@@ -27,7 +27,7 @@ fn main() {
             continue;
         }
 
-        let tokens = Tokenizer::new(&sql).tokenize().unwrap();
+        let tokens = Tokenizer::new(&sql).tokenize_with_location().unwrap();
         println!("tokens: {:?}", tokens);
         let ast = Parser::parse_sql(&sql).unwrap();
         println!("ast: {:?}", ast);

src/sqlparser/src/ast/statement.rs

@@ -710,7 +710,7 @@ impl ParseTo for UserOptions {
                 break;
             }
 
-            if let Token::Word(ref w) = token {
+            if let Token::Word(ref w) = token.token {
                 parser.next_token();
                 let (item_mut_ref, user_option) = match w.keyword {
                     Keyword::SUPERUSER => (&mut builder.super_user, UserOption::SuperUser),

src/sqlparser/src/parser.rs

@@ -34,7 +34,7 @@ where
     F: Fn(&mut Parser) -> T,
 {
     let mut tokenizer = Tokenizer::new(sql);
-    let tokens = tokenizer.tokenize().unwrap();
+    let tokens = tokenizer.tokenize_with_location().unwrap();
     f(&mut Parser::new(tokens))
 }
 

src/sqlparser/src/test_utils.rs

@@ -25,6 +25,7 @@ use alloc::{
     vec::Vec,
 };
 use core::fmt;
+use core::fmt::Debug;
 use core::iter::Peekable;
 use core::str::Chars;
 
@@ -237,6 +238,10 @@ impl Token {
             },
         })
     }
+
+    pub fn with_location(self, location: Location) -> TokenWithLocation {
+        TokenWithLocation::new(self, location.line, location.column)
+    }
 }
 
 /// A keyword (like SELECT) or an optionally quoted SQL identifier
@@ -300,6 +305,61 @@ impl fmt::Display for Whitespace {
     }
 }
 
+/// Location in input string
+#[derive(Debug, Eq, PartialEq, Clone)]
+pub struct Location {
+    /// Line number, starting from 1
+    pub line: u64,
+    /// Line column, starting from 1
+    pub column: u64,
+}
+
+/// A [Token] with [Location] attached to it
+#[derive(Debug, Eq, PartialEq, Clone)]
+pub struct TokenWithLocation {
+    pub token: Token,
+    pub location: Location,
+}
+
+impl TokenWithLocation {
+    pub fn new(token: Token, line: u64, column: u64) -> TokenWithLocation {
+        TokenWithLocation {
+            token,
+            location: Location { line, column },
+        }
+    }
+
+    pub fn wrap(token: Token) -> TokenWithLocation {
+        TokenWithLocation::new(token, 0, 0)
+    }
+}
+
+impl PartialEq<Token> for TokenWithLocation {
+    fn eq(&self, other: &Token) -> bool {
+        &self.token == other
+    }
+}
+
+impl PartialEq<TokenWithLocation> for Token {
+    fn eq(&self, other: &TokenWithLocation) -> bool {
+        self == &other.token
+    }
+}
+
+impl fmt::Display for TokenWithLocation {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        if self.token == Token::EOF {
+            write!(f, "EOF at the end")
+        } else {
+            write!(
+                f,
+                "{} at line:{}, column:{}",
+                self.token, self.location.line, self.location.column
+            )
+        }
+    }
+}
+
 /// Tokenizer error
 #[derive(Debug, PartialEq)]
 pub struct TokenizerError {
@@ -338,11 +398,11 @@ impl<'a> Tokenizer<'a> {
         }
     }
 
-    /// Tokenize the statement and produce a vector of tokens
-    pub fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError> {
+    /// Tokenize the statement and produce a vector of tokens with locations.
+    pub fn tokenize_with_location(&mut self) -> Result<Vec<TokenWithLocation>, TokenizerError> {
         let mut peekable = self.query.chars().peekable();
 
-        let mut tokens: Vec<Token> = vec![];
+        let mut tokens: Vec<TokenWithLocation> = vec![];
 
         while let Some(token) = self.next_token(&mut peekable)? {
             match &token {
@@ -359,11 +419,20 @@ impl<'a> Tokenizer<'a> {
                 _ => self.col += 1,
             }
 
-            tokens.push(token);
+            let token_with_location = TokenWithLocation::new(token, self.line, self.col);
+
+            tokens.push(token_with_location);
         }
         Ok(tokens)
     }
 
+    /// Tokenize the statement and produce a vector of tokens without locations.
+    #[allow(dead_code)]
+    fn tokenize(&mut self) -> Result<Vec<Token>, TokenizerError> {
+        self.tokenize_with_location()
+            .map(|v| v.into_iter().map(|t| t.token).collect())
+    }
+
     /// Get the next token or return None
     fn next_token(&self, chars: &mut Peekable<Chars<'_>>) -> Result<Option<Token>, TokenizerError> {
         match chars.peek() {

src/sqlparser/src/tokenizer.rs

@@ -1021,31 +1021,31 @@ fn parse_array() {
     assert_eq!(
         parse_sql_statements(sql),
         Err(ParserError::ParserError(
-            "syntax error at or near '['".to_string()
+            "syntax error at or near '[ at line:1, column:28'".to_string()
         ))
     );
 
     let sql = "SELECT ARRAY[ARRAY[], []]";
     assert_eq!(
         parse_sql_statements(sql),
         Err(ParserError::ParserError(
-            "syntax error at or near '['".to_string()
+            "syntax error at or near '[ at line:1, column:24'".to_string()
         ))
     );
 
     let sql = "SELECT ARRAY[[1, 2], ARRAY[3, 4]]";
     assert_eq!(
         parse_sql_statements(sql),
         Err(ParserError::ParserError(
-            "syntax error at or near 'ARRAY'".to_string()
+            "syntax error at or near 'ARRAY at line:1, column:27'".to_string()
         ))
     );
 
     let sql = "SELECT ARRAY[[], ARRAY[]]";
     assert_eq!(
         parse_sql_statements(sql),
         Err(ParserError::ParserError(
-            "syntax error at or near 'ARRAY'".to_string()
+            "syntax error at or near 'ARRAY at line:1, column:23'".to_string()
         ))
     );
 

src/sqlparser/tests/sqlparser_postgres.rs

@@ -7,15 +7,15 @@
   formatted_sql: CREATE TABLE t (a INT[][][])
 - input: CREATE TABLE t(a int[);
   error_msg: |-
-    sql parser error: Expected ], found: )
+    sql parser error: Expected ], found: ) at line:1, column:23
     Near "CREATE TABLE t(a int["
 - input: CREATE TABLE t(a int[[]);
   error_msg: |-
-    sql parser error: Expected ], found: [
+    sql parser error: Expected ], found: [ at line:1, column:23
     Near "CREATE TABLE t(a int["
 - input: CREATE TABLE t(a int]);
   error_msg: |-
-    sql parser error: Expected ',' or ')' after column definition, found: ]
+    sql parser error: Expected ',' or ')' after column definition, found: ] at line:1, column:22
     Near "CREATE TABLE t(a int"
 - input: SELECT foo[0] FROM foos
   formatted_sql: SELECT foo[0] FROM foos

src/sqlparser/tests/testdata/array.yaml

@@ -17,7 +17,7 @@
   formatted_sql: CREATE TABLE t (a INT, b INT) AS SELECT 1 AS b, 2 AS a
 - input: CREATE SOURCE src
   error_msg: |-
-    sql parser error: Expected ROW, found: EOF
+    sql parser error: Expected ROW, found: EOF at the end
     Near "CREATE SOURCE src"
 - input: CREATE SOURCE src ROW FORMAT JSON
   formatted_sql: CREATE SOURCE src ROW FORMAT JSON
@@ -44,7 +44,7 @@
   formatted_sql: CREATE USER user WITH SUPERUSER CREATEDB PASSWORD 'password'
 - input: CREATE SINK snk
   error_msg: |-
-    sql parser error: Expected FROM or AS after CREATE SINK sink_name, found: EOF
+    sql parser error: Expected FROM or AS after CREATE SINK sink_name, found: EOF at the end
     Near "CREATE SINK snk"
 - input: CREATE SINK IF NOT EXISTS snk FROM mv WITH (connector = 'mysql', mysql.endpoint = '127.0.0.1:3306', mysql.table = '<table_name>', mysql.database = '<database_name>', mysql.user = '<user_name>', mysql.password = '<password>')
   formatted_sql: CREATE SINK IF NOT EXISTS snk FROM mv WITH (connector = 'mysql', mysql.endpoint = '127.0.0.1:3306', mysql.table = '<table_name>', mysql.database = '<database_name>', mysql.user = '<user_name>', mysql.password = '<password>')
@@ -60,5 +60,5 @@
   error_msg: 'sql parser error: conflicting or redundant options'
 - input: create user tmp with encrypted password null
   error_msg: |-
-    sql parser error: Expected literal string, found: null
+    sql parser error: Expected literal string, found: null at line:1, column:45
     Near " tmp with encrypted password null"

src/sqlparser/tests/testdata/create.yaml

@@ -1,7 +1,7 @@
 # This file is automatically generated. See `src/sqlparser/test_runner/src/bin/apply.rs` for more information.
 - input: INSERT public.customer (id, name, active) VALUES (1, 2, 3)
   error_msg: |-
-    sql parser error: Expected INTO, found: public
+    sql parser error: Expected INTO, found: public at line:1, column:14
     Near "INSERT"
 - input: INSERT INTO t VALUES(1,3), (2,4) RETURNING *, a, a as aaa
   formatted_sql: INSERT INTO t VALUES (1, 3), (2, 4) RETURNING (*, a, a AS aaa)

src/sqlparser/tests/testdata/insert.yaml

@@ -43,11 +43,11 @@
   error_msg: 'sql parser error: WITH TIES cannot be specified without ORDER BY clause'
 - input: select * from (select 1 from 1);
   error_msg: |-
-    sql parser error: Expected identifier, found: 1
+    sql parser error: Expected identifier, found: 1 at line:1, column:31
     Near "from (select 1 from 1"
 - input: select * from (select * from tumble(t, x, interval '10' minutes))
   error_msg: |-
-    sql parser error: Expected ), found: minutes
+    sql parser error: Expected ), found: minutes at line:1, column:62
     Near "(t, x, interval '10'"
 - input: SELECT 1, FROM t
   error_msg: 'sql parser error: syntax error at or near "FROM"'
@@ -74,7 +74,7 @@
   error_msg: 'sql parser error: precision for type float must be less than 54 bits'
 - input: SELECT 1::int(2)
   error_msg: |-
-    sql parser error: Expected end of statement, found: (
+    sql parser error: Expected end of statement, found: ( at line:1, column:14
     Near "SELECT 1::int"
 - input: select id1, a1, id2, a2 from stream as S join version FOR SYSTEM_TIME AS OF NOW() AS V on id1= id2
   formatted_sql: SELECT id1, a1, id2, a2 FROM stream AS S JOIN version FOR SYSTEM_TIME AS OF NOW() AS V ON id1 = id2