Skip to content

Fixes parser for the top level tokens #369

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Mar 8, 2021
Merged

Fixes parser for the top level tokens #369

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7d7e2e9
Fixed parser for the top level tokens
abhikuhikar Feb 25, 2021
caebc9f
Small changes
abhikuhikar Feb 26, 2021
1a8ed3d
Added more tests and more top level checks
abhikuhikar Mar 2, 2021
fec50de
Updated algorithm to include DDL parse types
abhikuhikar Mar 4, 2021
a9de025
Added logic to handle top level DML tokens
abhikuhikar Mar 5, 2021
f375d2d
Addressed comments
abhikuhikar Mar 8, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 0 additions & 5 deletions lang/src/org/partiql/lang/errors/ErrorCode.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,11 +302,6 @@ enum class ErrorCode(private val category: ErrorCategory,
LOC_TOKEN,
"No stored procedure provided"),

PARSE_EXEC_AT_UNEXPECTED_LOCATION(
ErrorCategory.PARSER,
LOC_TOKEN,
"EXEC call found at unexpected location"),

PARSE_MALFORMED_JOIN(
ErrorCategory.PARSER,
LOC_TOKEN,
Expand Down
76 changes: 51 additions & 25 deletions lang/src/org/partiql/lang/syntax/SqlParser.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,10 @@ class SqlParser(private val ion: IonSystem) : Parser {
SIMPLE_PATH
}

internal enum class ParseType(val isJoin: Boolean = false) {
/** If the property [isTopLevelType] is true, then the parse node of this ParseType will only be valid at top level in the query.
* For example, EXEC, DDL and DML keywords can only be used at the top level in the query.
*/
internal enum class ParseType(val isJoin: Boolean = false, val isTopLevelType: Boolean = false, val isDml: Boolean = false) {
Comment on lines +58 to +61
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How is the isDml parameter defined? SET is a DML op, yet ParseType.SET isDml is false.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a legit observation. However, the SET parseType is not used anywhere. We can actually get rid of it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Changed the property isDml to true for SET.

ATOM,
CASE_SENSITIVE_ATOM,
CASE_INSENSITIVE_ATOM,
Expand Down Expand Up @@ -100,22 +103,22 @@ class SqlParser(private val ion: IonSystem) : Parser {
WHEN,
ELSE,
BAG,
INSERT,
INSERT_VALUE,
REMOVE,
SET,
UPDATE,
DELETE,
INSERT(isTopLevelType = true, isDml = true),
INSERT_VALUE(isTopLevelType = true, isDml = true),
REMOVE(isTopLevelType = true, isDml = true),
SET(isTopLevelType = true, isDml = true),
UPDATE(isTopLevelType = true, isDml = true),
DELETE(isTopLevelType = true, isDml = true),
ASSIGNMENT,
FROM,
FROM_CLAUSE,
FROM_SOURCE_JOIN,
CREATE_TABLE,
DROP_TABLE,
DROP_INDEX,
CREATE_INDEX,
CREATE_TABLE(isTopLevelType = true),
DROP_TABLE(isTopLevelType = true),
DROP_INDEX(isTopLevelType = true),
CREATE_INDEX(isTopLevelType = true),
PARAMETER,
EXEC;
EXEC(isTopLevelType = true);

val identifier = name.toLowerCase()
}
Expand Down Expand Up @@ -997,7 +1000,7 @@ class SqlParser(private val ion: IonSystem) : Parser {
KEYWORD -> when (head?.keywordText) {
in BASE_DML_KEYWORDS -> parseBaseDml()
"update" -> tail.parseUpdate()
"delete" -> tail.parseDelete()
"delete" -> tail.parseDelete(head!!)
"case" -> when (tail.head?.keywordText) {
"when" -> tail.parseCase(isSimple = false)
else -> tail.parseCase(isSimple = true)
Expand Down Expand Up @@ -1220,12 +1223,12 @@ class SqlParser(private val ion: IonSystem) : Parser {
copy(type = type)
}

private fun List<Token>.parseDelete(): ParseNode {
private fun List<Token>.parseDelete(name: Token): ParseNode {
if (head?.keywordText != "from") {
err("Expected FROM after DELETE", PARSE_UNEXPECTED_TOKEN)
}

return tail.parseLegacyDml { ParseNode(DELETE, null, emptyList(), this) }
return tail.parseLegacyDml { ParseNode(DELETE, name, emptyList(), this) }
}

private fun List<Token>.parseUpdate(): ParseNode = parseLegacyDml {
Expand Down Expand Up @@ -1724,6 +1727,12 @@ class SqlParser(private val ion: IonSystem) : Parser {
rem.err("No stored procedure provided", PARSE_NO_STORED_PROCEDURE_PROVIDED)
}

rem.forEach {
if (it.keywordText?.toLowerCase() == "exec") {
it.err("EXEC call found at unexpected location", PARSE_UNEXPECTED_TERM)
}
}

val procedureName = rem.head
rem = rem.tail

Expand Down Expand Up @@ -2244,25 +2253,39 @@ class SqlParser(private val ion: IonSystem) : Parser {
return ParseNode(ARG_LIST, null, items, rem)
}

private fun ParseNode.throwTopLevelParserError(): Nothing =
token?.err("Keyword ${token.text} only expected at the top level in the query", PARSE_UNEXPECTED_TERM)
?: throw ParserException("Keyword ${token?.text} only expected at the top level in the query", PARSE_UNEXPECTED_TERM, PropertyValueMap())

/**
* Checks that the given [Token] list does not have any top-level tokens outside of the top level query. Throws
* an error if so.
*
* Currently only checks for `EXEC`. DDL and DML along with corresponding error codes to be added in the future
* (https://github.com/partiql/partiql-lang-kotlin/issues/354).
* Validates tree to make sure that the top level tokens are not found below the top level.
* Top level tokens are the tokens or keywords which are valid to be used only at the top level in the query.
* i.e. these tokens cannot be used with a mix of other commands. Hence if more than one top level tokens are found
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: "i.e. these tokens cannot be used with a mix of other commands top level tokens"? "commands" seems too generalized.

* in the query then it is invalid.
* [level] is the current traversal level in the parse tree.
* If [topLevelTokenSeen] is true, it means it has been encountered at least once before while traversing the parse tree.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: description of topLevelTokenSeen could be more more specific. Consider "...it means a top level token has been previously encountered while traversing the parse tree"

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe the variable name itself tells it all.

*/
private fun List<Token>.checkUnexpectedTopLevelToken() {
for ((index, token) in this.withIndex()) {
if (token.keywordText == "exec" && index != 0) {
token.err("EXEC call found at unexpected location", PARSE_EXEC_AT_UNEXPECTED_LOCATION)
private fun validateTopLevelNodes(node: ParseNode, level: Int, topLevelTokenSeen: Boolean) {
if (topLevelTokenSeen && node.type.isTopLevelType) {
node.throwTopLevelParserError()
}

if (node.type.isTopLevelType && level > 0) {
// Note that for DML operations, top level parse node may be of type 'FROM'. Hence the check level > 1
if (node.type.isDml) {
if (level > 1) {
node.throwTopLevelParserError()
}
} else {
node.throwTopLevelParserError()
}
}
node.children.map { validateTopLevelNodes(node = it, level = level + 1, topLevelTokenSeen = topLevelTokenSeen || node.type.isTopLevelType) }
}

/** Entry point into the parser. */
override fun parseExprNode(source: String): ExprNode {
val tokens = SqlLexer(ion).tokenize(source)
tokens.checkUnexpectedTopLevelToken()
val node = tokens.parseExpression()
val rem = node.remaining
if (!rem.onlyEndOfStatement()) {
Expand All @@ -2272,6 +2295,9 @@ class SqlParser(private val ion: IonSystem) : Parser {
else -> rem.err("Unexpected token after expression", PARSE_UNEXPECTED_TOKEN)
}
}

validateTopLevelNodes(node = node, level = 0, topLevelTokenSeen = false)

return node.toExprNode()
}

Expand Down
134 changes: 128 additions & 6 deletions lang/test/org/partiql/lang/errors/ParserErrorsTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1246,6 +1246,116 @@ class ParserErrorsTest : TestBase() {
Property.TOKEN_TYPE to TokenType.EOF,
Property.TOKEN_VALUE to ion.newSymbol("EOF")))

@Test
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could add more tests. At least one for each top level token we're checking.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks. Added tests for other top level tokens.

fun updateWithNestedSet() = checkInputThrowingParserException(
"UPDATE test SET x = SET test.y = 6",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So when I'm testing this query in the REPL, it gives the following:

PartiQL> UPDATE test SET x = SET test.y = 6;
org.partiql.lang.syntax.ParserException: Type UPDATE only expected at the top level
	Parser Error: at line 1, column 21: unexpected term found, KEYWORD : set

Shouldn't the 2nd line say SET and not UPDATE? I haven't had a chance to test w/ the other top-level tokens.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The associated parse type is UPDATE.
I will change this to - "Keyword ${node.token} only expected at the top level".
It makes more sense I guess and would be less confusing.

ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 21L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("set")))

@Test
fun updateWithRemove() = checkInputThrowingParserException(
"UPDATE test SET x = REMOVE y",
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 21L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("remove")))

@Test
fun updateWithInsert() = checkInputThrowingParserException(
"UPDATE test SET x = INSERT INTO foo VALUE 1",
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 21L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("insert_into")))

@Test
fun updateWithDelete() = checkInputThrowingParserException(
"UPDATE test SET x = DELETE FROM y",
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 21L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("delete")))

@Test
fun updateWithExec() = checkInputThrowingParserException(
"UPDATE test SET x = EXEC foo arg1, arg2",
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 26L,
Property.TOKEN_TYPE to TokenType.IDENTIFIER,
Property.TOKEN_VALUE to ion.newSymbol("foo")))

@Test
fun updateWithCreateTable() = checkInputThrowingParserException(
"UPDATE test SET x = CREATE TABLE foo",
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 21L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("create")))

@Test
fun updateWithDropTable() = checkInputThrowingParserException(
"UPDATE test SET x = DROP TABLE foo",
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 21L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("drop")))

@Test
fun updateWithCreateIndex() = checkInputThrowingParserException(
"UPDATE test SET x = CREATE INDEX ON foo (x, y.z)",
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 21L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("create")))

@Test
fun nestedRemove() = checkInputThrowingParserException(
"REMOVE REMOVE y",
ErrorCode.PARSE_INVALID_PATH_COMPONENT,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason this error is not PARSE_UNEXPECTED_TERM?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is handled actively by the parser while parsing the REMOVE keyword.

mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 8L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("remove")))

@Test
fun nestedInsertInto() = checkInputThrowingParserException(
"INSERT INTO foo VALUE INSERT INTO foo VALUE 1 AT bar",
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 23L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("insert_into")))

@Test
fun updateWithDropIndex() = checkInputThrowingParserException(
"UPDATE test SET x = DROP INDEX bar ON foo",
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 21L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("drop")))

@Test
fun updateFromList() = checkInputThrowingParserException(
"UPDATE x, y SET a = b",
Expand Down Expand Up @@ -1296,6 +1406,16 @@ class ParserErrorsTest : TestBase() {
Property.TOKEN_TYPE to TokenType.OPERATOR,
Property.TOKEN_VALUE to ion.newSymbol("-")))

@Test
fun nestedCreateTable() = checkInputThrowingParserException(
"CREATE TABLE CREATE TABLE foo",
ErrorCode.PARSE_UNEXPECTED_TOKEN,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 14L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("create")))

@Test
fun dropTableWithOperatorAfterIdentifier() = checkInputThrowingParserException(
"DROP TABLE foo+bar",
Expand Down Expand Up @@ -1603,7 +1723,7 @@ class ParserErrorsTest : TestBase() {
@Test
fun execAtUnexpectedLocation() = checkInputThrowingParserException(
"EXEC EXEC",
ErrorCode.PARSE_EXEC_AT_UNEXPECTED_LOCATION,
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 6L,
Expand All @@ -1613,20 +1733,22 @@ class ParserErrorsTest : TestBase() {
@Test
fun execAtUnexpectedLocationAfterExec() = checkInputThrowingParserException(
"EXEC foo EXEC",
ErrorCode.PARSE_EXEC_AT_UNEXPECTED_LOCATION,
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 10L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("exec")))

// TODO: The token in the error message here should be "exec" instead of "undrop".
// Check this issue for more details. https://github.com/partiql/partiql-lang-kotlin/issues/372
@Test
fun execAtUnexpectedLocationInExpression() = checkInputThrowingParserException(
"SELECT * FROM (EXEC undrop 'foo')",
ErrorCode.PARSE_EXEC_AT_UNEXPECTED_LOCATION,
ErrorCode.PARSE_UNEXPECTED_TERM,
mapOf(
Property.LINE_NUMBER to 1L,
Property.COLUMN_NUMBER to 16L,
Property.TOKEN_TYPE to TokenType.KEYWORD,
Property.TOKEN_VALUE to ion.newSymbol("exec")))
Property.COLUMN_NUMBER to 21L,
Property.TOKEN_TYPE to TokenType.IDENTIFIER,
Property.TOKEN_VALUE to ion.newSymbol("undrop")))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't the unexpected term be EXEC here?

Copy link
Contributor Author

@abhikuhikar abhikuhikar Mar 4, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should be. But the associated token with the EXEC ParseNode is the name of the stored procedure. Hence, the issue. To get around with this, I think we can pass the name of the stored procedure in the arguments along with the other arglist or some such. But that's worthy of a separate issue.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the EXEC parsing should be changed (but won't be changed in this PR), can we add a comment linking to an issue to track this?

}
1 change: 0 additions & 1 deletion lang/test/org/partiql/lang/syntax/SqlParserTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2636,7 +2636,6 @@ class SqlParserTest : SqlParserTestBase() {
"""
)


@Test
fun updateWhereDml() = assertExpression(
"UPDATE x SET k = 5, m = 6 WHERE a = b",
Expand Down