Skip to content

Commit 2879f3a

Browse files
yliuuuuam357alancai98
authored
V1 ddl extended keyword (#1455)
Co-authored-by: Arash Maymandi <[email protected]> Co-authored-by: Alan Cai <[email protected]>
1 parent 28edbb7 commit 2879f3a

File tree

11 files changed

+763
-43
lines changed

11 files changed

+763
-43
lines changed

‎partiql-ast/api/partiql-ast.api

Lines changed: 137 additions & 24 deletions
Large diffs are not rendered by default.

‎partiql-ast/src/main/kotlin/org/partiql/ast/helpers/ToLegacyAst.kt

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,12 @@ private class AstTranslator(val metas: Map<String, MetaContainer>) : AstBaseVisi
136136
}
137137
val tableName = node.name.symbol
138138
val def = node.definition?.let { visitTableDefinition(it, ctx) }
139+
if (node.partitionBy != null) {
140+
error("The legacy AST does not support Partition BY in create Table")
141+
}
142+
if (node.tableProperties.isNotEmpty()) {
143+
error("The legacy AST does not support TBLProperties in create Table")
144+
}
139145
ddl(createTable(tableName, def), metas)
140146
}
141147

@@ -187,6 +193,12 @@ private class AstTranslator(val metas: Map<String, MetaContainer>) : AstBaseVisi
187193
val name = node.name.symbol
188194
val type = visitType(node.type, ctx)
189195
val constraints = node.constraints.translate<PartiqlAst.ColumnConstraint>(ctx)
196+
if (node.isOptional) {
197+
error("The legacy AST does not support optional field declaration")
198+
}
199+
if (node.comment != null) {
200+
error("The legacy AST does not support comment clause")
201+
}
190202
columnDeclaration(name, type, constraints, metas)
191203
}
192204

‎partiql-ast/src/main/resources/partiql_ast.ion

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,12 @@ ddl_op::[
133133
create_table::{
134134
name: identifier,
135135
definition: optional::table_definition,
136+
// This is an expression for backward compatibility
137+
// For now, we support `PARTITION BY (<id> (, <id>)* ))`
138+
// But in the future, we might support additional partition expressions:
139+
// such as `PARTITION BY RANGE(..)`
140+
partition_by: optional::partition_by,
141+
table_properties: list::[table_property],
136142
},
137143

138144
// CREATE INDEX [<identifier>] ON <identifier> (<path> [, <path>]...)
@@ -233,6 +239,8 @@ type::[
233239
// for struct subfield. But modeling this to be a list of constraints
234240
// to prevent future breaking changes.
235241
constraints: list::[constraint],
242+
is_optional: bool,
243+
comment: optional::string,
236244
}
237245
],
238246
}, // STRUCT <fields>
@@ -829,6 +837,9 @@ table_definition::{
829837
name: '.identifier.symbol',
830838
type: '.type',
831839
constraints: list::[constraint],
840+
// TODO: Consider to model this as a constraint?
841+
is_optional: bool,
842+
comment: optional::string,
832843
}
833844
],
834845
}
@@ -847,6 +858,18 @@ constraint::{
847858
],
848859
}
849860

861+
partition_by::[
862+
attr_list :: {
863+
list: list::['.identifier.symbol']
864+
},
865+
// We can add other commonly support Partition Expression like `range` later
866+
]
867+
868+
table_property::{
869+
name: string,
870+
value: '.value',
871+
}
872+
850873
// SQL-99 Table 11
851874
datetime_field::[
852875
YEAR, // 0001-9999

‎partiql-ast/src/test/kotlin/org/partiql/ast/helpers/ToLegacyAstTest.kt

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,9 @@ class ToLegacyAstTest {
423423
fields += org.partiql.ast.typeStructField(
424424
org.partiql.ast.identifierSymbol("a", Identifier.CaseSensitivity.INSENSITIVE),
425425
typeInt2(),
426-
emptyList()
426+
emptyList(),
427+
false,
428+
null
427429
)
428430
}
429431
},

‎partiql-lang/src/main/kotlin/org/partiql/lang/syntax/impl/PartiQLPigVisitor.kt

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,7 @@ internal class PartiQLPigVisitor(
246246
throw ParserException("PIG Parser does not support qualified name as table name", ErrorCode.PARSE_UNEXPECTED_TOKEN)
247247
}
248248
val def = ctx.tableDef()?.let { visitTableDef(it) }
249+
ctx.tableExtension().map { visit(it) }
249250
createTable_(name, def, ctx.CREATE().getSourceMetaContainer())
250251
}
251252

@@ -264,6 +265,12 @@ internal class PartiQLPigVisitor(
264265
val name = visitSymbolPrimitive(ctx.columnName().symbolPrimitive()).name.text
265266
val type = visit(ctx.type()) as PartiqlAst.Type
266267
val constrs = ctx.columnConstraint().map { visitColumnConstraint(it) }
268+
if (ctx.OPTIONAL() != null) {
269+
throw ParserException("PIG Parser does not support OPTIONAL Field", ErrorCode.PARSE_UNEXPECTED_TOKEN)
270+
}
271+
if (ctx.comment() != null) {
272+
throw ParserException("PIG Parser does not support COMMENT Clause", ErrorCode.PARSE_UNEXPECTED_TOKEN)
273+
}
267274
columnDeclaration(name, type, constrs)
268275
}
269276

@@ -281,6 +288,12 @@ internal class PartiQLPigVisitor(
281288
columnNull()
282289
}
283290

291+
override fun visitTableConstrDeclaration(ctx: PartiQLParser.TableConstrDeclarationContext) = throw ParserException("PIG Parser does not support tuple level constraint", ErrorCode.PARSE_UNEXPECTED_TOKEN)
292+
293+
override fun visitTblExtensionPartition(ctx: PartiQLParser.TblExtensionPartitionContext) = throw ParserException("PIG Parser does not support PARTITION BY Clause", ErrorCode.PARSE_UNEXPECTED_TOKEN)
294+
295+
override fun visitTblExtensionTblProperties(ctx: PartiQLParser.TblExtensionTblPropertiesContext) = throw ParserException("PIG Parser does not support TBLPROPERTIES Clause", ErrorCode.PARSE_UNEXPECTED_TOKEN)
296+
284297
/**
285298
*
286299
* EXECUTE

‎partiql-lang/src/test/kotlin/org/partiql/lang/syntax/PartiQLParserDDLTest.kt

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,44 @@ internal class PartiQLParserDDLTest : PartiQLParserTestBase() {
9696
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
9797
context = mapOf(),
9898
),
99+
ParserErrorTestCase(
100+
description = "PIG Parser does not support OPTIONAL Attribute",
101+
query = """
102+
CREATE TABLE tbl (
103+
a OPTIONAL INT2
104+
)
105+
""".trimIndent(),
106+
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
107+
context = mapOf(),
108+
),
109+
ParserErrorTestCase(
110+
description = "PIG Parser does not support COMMENT keyword",
111+
query = """
112+
CREATE TABLE tbl (
113+
a INT2 COMMENT 'this is a comment'
114+
)
115+
""".trimIndent(),
116+
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
117+
context = mapOf(),
118+
),
119+
ParserErrorTestCase(
120+
description = "PIG Parser does not support PARTITION BY keyword",
121+
query = """
122+
CREATE TABLE tbl
123+
PARTITION BY (a, b)
124+
""".trimIndent(),
125+
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
126+
context = mapOf(),
127+
),
128+
ParserErrorTestCase(
129+
description = "PIG Parser does not support TBLPROPERTIES keyword",
130+
query = """
131+
CREATE TABLE tbl
132+
TBLPROPERTIES ('k1' = 'v1')
133+
""".trimIndent(),
134+
code = ErrorCode.PARSE_UNEXPECTED_TOKEN,
135+
context = mapOf(),
136+
),
99137

100138
// Putting those tests here are they are impacted by DDL implementation
101139
ParserErrorTestCase(

‎partiql-lang/src/test/kotlin/org/partiql/lang/syntax/PartiQLParserMatchTest.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1067,7 +1067,7 @@ class PartiQLParserMatchTest : PartiQLParserTestBase() {
10671067

10681068
@Test
10691069
fun prefilters() = assertExpression(
1070-
"SELECT u as banCandidate FROM g MATCH (p:Post Where p.isFlagged = true) <-[:createdPost]- (u:Usr WHERE u.isBanned = false AND u.karma < 20) -[:createdComment]->(c:Comment WHERE c.isFlagged = true) WHERE p.title LIKE '%considered harmful%'",
1070+
"SELECT u as banCandidate FROM g MATCH (p:Post Where p.isFlagged = true) <-[:createdPost]- (u:Usr WHERE u.isBanned = false AND u.karma < 20) -[:createdComment]->(c:\"Comment\" WHERE c.isFlagged = true) WHERE p.title LIKE '%considered harmful%'",
10711071
) {
10721072
PartiqlAst.build {
10731073
select(

‎partiql-parser/src/main/antlr/PartiQL.g4

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,14 +77,15 @@ qualifiedName : (qualifier+=symbolPrimitive PERIOD)* name=symbolPrimitive;
7777
tableName : symbolPrimitive;
7878
columnName : symbolPrimitive;
7979
constraintName : symbolPrimitive;
80+
comment : COMMENT LITERAL_STRING;
8081

8182
ddl
8283
: createCommand
8384
| dropCommand
8485
;
8586

8687
createCommand
87-
: CREATE TABLE qualifiedName ( PAREN_LEFT tableDef PAREN_RIGHT )? # CreateTable
88+
: CREATE TABLE qualifiedName ( PAREN_LEFT tableDef PAREN_RIGHT )? tableExtension* # CreateTable
8889
| CREATE INDEX ON symbolPrimitive PAREN_LEFT pathSimple ( COMMA pathSimple )* PAREN_RIGHT # CreateIndex
8990
;
9091

@@ -98,7 +99,7 @@ tableDef
9899
;
99100

100101
tableDefPart
101-
: columnName type columnConstraint* # ColumnDeclaration
102+
: columnName OPTIONAL? type columnConstraint* comment? # ColumnDeclaration
102103
| ( CONSTRAINT constraintName )? tableConstraintDef # TableConstrDeclaration
103104
;
104105

@@ -136,6 +137,20 @@ uniqueConstraintDef
136137
// but we at least can eliminate SFW query here.
137138
searchCondition : exprOr;
138139

140+
// SQL/HIVE DDL Extension, Support additional table metadatas such as partition by, tblProperties, etc.
141+
tableExtension
142+
: PARTITION BY partitionBy # TblExtensionPartition
143+
| TBLPROPERTIES PAREN_LEFT keyValuePair (COMMA keyValuePair)* PAREN_RIGHT # TblExtensionTblProperties
144+
;
145+
146+
// Limiting the scope to only allow String as valid value for now
147+
keyValuePair : key=LITERAL_STRING EQ value=LITERAL_STRING;
148+
149+
// For now: just support a list of column name
150+
// In the future, we might support common partition expression such as Hash(), Range(), etc.
151+
partitionBy
152+
: PAREN_LEFT columnName (COMMA columnName)* PAREN_RIGHT #PartitionColList
153+
;
139154
/**
140155
*
141156
* DATA MANIPULATION LANGUAGE (DML)
@@ -836,5 +851,5 @@ type
836851
;
837852

838853
structField
839-
: columnName COLON type columnConstraint*
854+
: columnName OPTIONAL? COLON type columnConstraint* comment?
840855
;

‎partiql-parser/src/main/antlr/PartiQLTokens.g4

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ COALESCE: 'COALESCE';
5959
COLLATE: 'COLLATE';
6060
COLLATION: 'COLLATION';
6161
COLUMN: 'COLUMN';
62+
COMMENT: 'COMMENT';
6263
COMMIT: 'COMMIT';
6364
CONNECT: 'CONNECT';
6465
CONNECTION: 'CONNECTION';
@@ -175,6 +176,7 @@ ON: 'ON';
175176
ONLY: 'ONLY';
176177
OPEN: 'OPEN';
177178
OPTION: 'OPTION';
179+
OPTIONAL: 'OPTIONAL';
178180
OR: 'OR';
179181
ORDER: 'ORDER';
180182
OUTER: 'OUTER';
@@ -287,6 +289,7 @@ MODIFIED: 'MODIFIED';
287289
NEW: 'NEW';
288290
OLD: 'OLD';
289291
NOTHING: 'NOTHING';
292+
TBLPROPERTIES: 'TBLPROPERTIES';
290293

291294
/**
292295
*

‎partiql-parser/src/main/kotlin/org/partiql/parser/internal/PartiQLParserDefault.kt

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import org.partiql.ast.GroupBy
4444
import org.partiql.ast.Identifier
4545
import org.partiql.ast.Let
4646
import org.partiql.ast.OnConflict
47+
import org.partiql.ast.PartitionBy
4748
import org.partiql.ast.Path
4849
import org.partiql.ast.Returning
4950
import org.partiql.ast.Select
@@ -139,6 +140,7 @@ import org.partiql.ast.onConflictActionDoUpdate
139140
import org.partiql.ast.onConflictTargetConstraint
140141
import org.partiql.ast.onConflictTargetSymbols
141142
import org.partiql.ast.orderBy
143+
import org.partiql.ast.partitionByAttrList
142144
import org.partiql.ast.path
143145
import org.partiql.ast.pathStepIndex
144146
import org.partiql.ast.pathStepSymbol
@@ -176,6 +178,7 @@ import org.partiql.ast.statementExplainTargetDomain
176178
import org.partiql.ast.statementQuery
177179
import org.partiql.ast.tableDefinition
178180
import org.partiql.ast.tableDefinitionAttribute
181+
import org.partiql.ast.tableProperty
179182
import org.partiql.ast.typeAny
180183
import org.partiql.ast.typeArray
181184
import org.partiql.ast.typeBag
@@ -610,7 +613,26 @@ internal class PartiQLParserDefault : PartiQLParser {
610613
override fun visitCreateTable(ctx: GeneratedParser.CreateTableContext) = translate(ctx) {
611614
val table = visitQualifiedName(ctx.qualifiedName())
612615
val definition = ctx.tableDef()?.let { visitTableDef(it) }
613-
ddlOpCreateTable(table, definition)
616+
val partitionBy = ctx
617+
.tableExtension()
618+
.filterIsInstance<GeneratedParser.TblExtensionPartitionContext>()
619+
.let {
620+
if (it.size > 1) throw error(ctx, "Expect one PARTITION BY clause.")
621+
it.firstOrNull()?.let { visitTblExtensionPartition(it) }
622+
}
623+
val tblProperties = ctx
624+
.tableExtension()
625+
.filterIsInstance<GeneratedParser.TblExtensionTblPropertiesContext>()
626+
.let {
627+
if (it.size > 1) throw error(ctx, "Expect one TBLPROPERTIES clause.")
628+
val tblPropertiesCtx = it.firstOrNull()
629+
tblPropertiesCtx?.keyValuePair()?.map {
630+
val key = it.key.getStringValue()
631+
val value = it.value.getStringValue()
632+
tableProperty(key, stringValue(value))
633+
} ?: emptyList()
634+
}
635+
ddlOpCreateTable(table, definition, partitionBy, tblProperties)
614636
}
615637

616638
override fun visitCreateIndex(ctx: GeneratedParser.CreateIndexContext) = translate(ctx) {
@@ -640,7 +662,12 @@ internal class PartiQLParserDefault : PartiQLParser {
640662
isValidTypeDeclarationOrThrow(it, ctx.type())
641663
}
642664
val constraints = ctx.columnConstraint().map { visitColumnConstraint(it) }
643-
tableDefinitionAttribute(name, type, constraints)
665+
val optional = when (ctx.OPTIONAL()) {
666+
null -> false
667+
else -> true
668+
}
669+
val comment = ctx.comment()?.LITERAL_STRING()?.getStringValue()
670+
tableDefinitionAttribute(name, type, constraints, optional, comment)
644671
}
645672

646673
/**
@@ -710,6 +737,13 @@ internal class PartiQLParserDefault : PartiQLParser {
710737
constraint(identifier, body)
711738
}
712739

740+
override fun visitTblExtensionPartition(ctx: GeneratedParser.TblExtensionPartitionContext) =
741+
ctx.partitionBy().accept(this) as PartitionBy
742+
743+
override fun visitPartitionColList(ctx: GeneratedParser.PartitionColListContext) = translate(ctx) {
744+
partitionByAttrList(ctx.columnName().map { visitAs<Identifier.Symbol> (it.symbolPrimitive()) })
745+
}
746+
713747
/**
714748
*
715749
* EXECUTE
@@ -2255,8 +2289,13 @@ internal class PartiQLParserDefault : PartiQLParser {
22552289
else -> throw error(it, "Only NULL Constraint and NOT NULL Constraint are allowed in Struct field")
22562290
}
22572291
}
2292+
val optional = when (structFieldCtx.OPTIONAL()) {
2293+
null -> false
2294+
else -> true
2295+
}
2296+
val comment = structFieldCtx.comment()?.LITERAL_STRING()?.getStringValue()
22582297

2259-
typeStructField(name, type, constraints)
2298+
typeStructField(name, type, constraints, optional, comment)
22602299
}
22612300
typeStruct(fields)
22622301
}

0 commit comments

Comments
 (0)