Add EXCLUDE to the parser, ast, plan, and plan schema inferencer (#1226)

Co-authored-by: John Ed Quinn <[email protected]>

Author: alancai98

CHANGELOG.md

@@ -34,6 +34,9 @@ Thank you to all who have contributed!
 - Support parsing, planning, and evaluation of Bitwise AND operator (&).
   - The Bitwise And Operator only works for integer operands.
   - The operator precedence may change based on the pending operator precedence [RFC](https://github.com/partiql/partiql-docs/issues/50).
+- **EXPERIMENTAL** Adds `EXCLUDE` to parser, ast, plan, and plan schema inferencer
+  - This feature is marked as experimental until an RFC is added https://github.com/partiql/partiql-spec/issues/39
+  - NOTE: this feature is not currently implemented in the evaluator
 
 ### Changed
 
@@ -42,6 +45,7 @@ Thank you to all who have contributed!
 ### Fixed
 - Fixes typing of scalar subqueries in the PartiQLSchemaInferencer. Note that usage of `SELECT *` in subqueries
   is not fully supported. Please make sure to handle InferenceExceptions.
+- Fixes schema inferencer behavior for ORDER BY
 
 ### Removed
 

partiql-ast/src/main/kotlin/org/partiql/ast/helpers/ToLegacyAst.kt

@@ -17,6 +17,7 @@ import com.amazon.ionelement.api.ionSymbol
 import com.amazon.ionelement.api.metaContainerOf
 import org.partiql.ast.AstNode
 import org.partiql.ast.DatetimeField
+import org.partiql.ast.Exclude
 import org.partiql.ast.Expr
 import org.partiql.ast.From
 import org.partiql.ast.GraphMatch
@@ -665,14 +666,15 @@ private class AstTranslator(val metas: Map<String, MetaContainer>) : AstBaseVisi
         }
         val project = visitSelect(node.select, ctx)
         val from = visitFrom(node.from, ctx)
+        val exclude = node.exclude?.let { visitExclude(it, ctx) }
         val fromLet = node.let?.let { visitLet(it, ctx) }
         val where = node.where?.let { visitExpr(it, ctx) }
         val groupBy = node.groupBy?.let { visitGroupBy(it, ctx) }
         val having = node.having?.let { visitExpr(it, ctx) }
         val orderBy = node.orderBy?.let { visitOrderBy(it, ctx) }
         val limit = node.limit?.let { visitExpr(it, ctx) }
         val offset = node.offset?.let { visitExpr(it, ctx) }
-        select(setq, project, from, fromLet, where, groupBy, having, orderBy, limit, offset, metas)
+        select(setq, project, exclude, from, fromLet, where, groupBy, having, orderBy, limit, offset, metas)
     }
 
     /**
@@ -750,6 +752,48 @@ private class AstTranslator(val metas: Map<String, MetaContainer>) : AstBaseVisi
         join(type, lhs, rhs, condition, metas)
     }
 
+    override fun visitExclude(node: Exclude, ctx: Ctx): PartiqlAst.ExcludeOp = translate(node) { metas ->
+        val excludeExprs = node.exprs.translate<PartiqlAst.ExcludeExpr>(ctx)
+        excludeOp(excludeExprs, metas)
+    }
+
+    override fun visitExcludeExcludeExpr(node: Exclude.ExcludeExpr, ctx: Ctx) = translate(node) { metas ->
+        val root = visitIdentifierSymbol(node.root, ctx)
+        val steps = node.steps.translate<PartiqlAst.ExcludeStep>(ctx)
+        excludeExpr(root = root, steps = steps, metas)
+    }
+
+    override fun visitExcludeStep(node: Exclude.Step, ctx: Ctx) =
+        super.visitExcludeStep(node, ctx) as PartiqlAst.ExcludeStep
+
+    override fun visitExcludeStepExcludeTupleAttr(node: Exclude.Step.ExcludeTupleAttr, ctx: Ctx) = translate(node) { metas ->
+        val attr = node.symbol.symbol
+        val case = node.symbol.caseSensitivity.toLegacyCaseSensitivity()
+        excludeTupleAttr(identifier(attr, case), metas)
+    }
+
+    override fun visitExcludeStepExcludeCollectionIndex(
+        node: Exclude.Step.ExcludeCollectionIndex,
+        ctx: Ctx
+    ) = translate(node) { metas ->
+        val index = node.index.toLong()
+        excludeCollectionIndex(index, metas)
+    }
+
+    override fun visitExcludeStepExcludeTupleWildcard(
+        node: Exclude.Step.ExcludeTupleWildcard,
+        ctx: Ctx
+    ) = translate(node) { metas ->
+        excludeTupleWildcard(metas)
+    }
+
+    override fun visitExcludeStepExcludeCollectionWildcard(
+        node: Exclude.Step.ExcludeCollectionWildcard,
+        ctx: Ctx
+    ) = translate(node) { metas ->
+        excludeCollectionWildcard(metas)
+    }
+
     override fun visitLet(node: Let, ctx: Ctx) = translate(node) { metas ->
         val bindings = node.bindings.translate<PartiqlAst.LetBinding>(ctx)
         let(bindings, metas)

partiql-ast/src/main/pig/partiql.ion

@@ -167,6 +167,7 @@ may then be further optimized by selecting better implementations of each operat
             (select
                 (setq (? set_quantifier))
                 (project projection)
+                (exclude_clause (? exclude_op))
                 (from from_source)
                 (from_let (? let))
                 (where (? expr))
@@ -197,7 +198,7 @@ may then be further optimized by selecting better implementations of each operat
         (sum path_step
             // `someRoot[<expr>]`, or `someRoot.someField` which is equivalent to `someRoot['someField']`.
             (path_expr index::expr case::case_sensitivity)
-            // `someRoot[*]`]
+            // `someRoot[*]`
             (path_wildcard)
             // `someRoot.*`
             (path_unpivot))
@@ -220,6 +221,22 @@ may then be further optimized by selecting better implementations of each operat
             // For `<expr> [AS <id>]`
             (project_expr expr::expr as_alias::(? symbol)))
 
+        // EXCLUDE clause
+        (product exclude_op exprs::(* exclude_expr 1))
+
+        (product exclude_expr root::identifier steps::(* exclude_step 1))
+
+        (sum exclude_step
+            // `someRoot.someField` case sensitivity depends on if `someField` is quoted
+            // `someRoot[<string literal>] is equivalent to `someRoot."<string literal>"` (case-sensitive)
+            (exclude_tuple_attr attr::identifier)
+            // `someRoot[<int literal>]`
+            (exclude_collection_index index::int)
+            // `someRoot[*]`]
+            (exclude_tuple_wildcard)
+            // `someRoot.*`
+            (exclude_collection_wildcard))
+
         // A list of LET bindings
         (product let let_bindings::(* let_binding 1))
 

partiql-ast/src/main/resources/partiql_ast.ion

@@ -494,6 +494,7 @@ expr::[
   // The PartiQL `<sfw>` query expression, think SQL `<query specification>`
   s_f_w::{
     select:   select,              // oneof SELECT / SELECT VALUE / PIVOT
+    exclude:  optional::exclude,
     from:     from,
     let:      optional::let,
     where:    optional::expr,
@@ -561,6 +562,22 @@ select::[
   },
 ]
 
+exclude::{
+  exprs: list::[exclude_expr],
+  _: [
+    exclude_expr::{
+      root: '.identifier.symbol',
+      steps: list::[step],
+    },
+    step::[
+      exclude_tuple_attr::{ symbol: '.identifier.symbol' },
+      exclude_collection_index::{ index: int },
+      exclude_tuple_wildcard::{},
+      exclude_collection_wildcard::{},
+    ]
+  ]
+}
+
 // PartiQL FROM Clause Variants — https://partiql.org/dql/from.html
 from::[
 

partiql-lang/src/main/kotlin/org/partiql/lang/eval/visitors/GroupByPathExpressionVisitorTransform.kt

@@ -102,6 +102,7 @@ class GroupByPathExpressionVisitorTransform(
         val projection = currentAndUnshadowedTransformer.transformExprSelect_project(node)
 
         // The scope of the expressions in the FROM clause is the same as that of the parent scope.
+        val exclude = unshadowedTransformer.transformExprSelect_excludeClause(node)
         val from = this.transformExprSelect_from(node)
         val fromLet = unshadowedTransformer.transformExprSelect_fromLet(node)
         val where = unshadowedTransformer.transformExprSelect_where(node)
@@ -116,6 +117,7 @@ class GroupByPathExpressionVisitorTransform(
             PartiqlAst.Expr.Select(
                 setq = node.setq,
                 project = projection,
+                excludeClause = exclude,
                 from = from,
                 fromLet = fromLet,
                 where = where,

partiql-lang/src/main/kotlin/org/partiql/lang/eval/visitors/VisitorTransformBase.kt

@@ -36,6 +36,7 @@ abstract class VisitorTransformBase : PartiqlAst.VisitorTransform() {
      * infinite recursion.
      */
     fun transformExprSelectEvaluationOrder(node: PartiqlAst.Expr.Select): PartiqlAst.Expr {
+        val exclude = transformExprSelect_excludeClause(node)
         val from = transformExprSelect_from(node)
         val fromLet = transformExprSelect_fromLet(node)
         val where = transformExprSelect_where(node)
@@ -51,6 +52,7 @@ abstract class VisitorTransformBase : PartiqlAst.VisitorTransform() {
             PartiqlAst.Expr.Select(
                 setq = setq,
                 project = project,
+                excludeClause = exclude,
                 from = from,
                 fromLet = fromLet,
                 where = where,

partiql-lang/src/main/kotlin/org/partiql/lang/planner/PlanningProblemDetails.kt

@@ -77,6 +77,12 @@ sealed class PlanningProblemDetails(
                     "Please use the `INSERT INTO <table> << <expr>, ... >>` form instead."
             }
         )
+
+    data class UnresolvedExcludeExprRoot(val root: String) :
+        PlanningProblemDetails(
+            ProblemSeverity.ERROR,
+            { "Exclude expression given an unresolvable root '$root'" }
+        )
 }
 
 private fun quotationHint(caseSensitive: Boolean) =