Merge pull request #1314 from partiql/partiql-eval-merge

Merges 0.14 main into partiql-eval

Author: rchowell

CHANGELOG.md

@@ -27,6 +27,24 @@ Thank you to all who have contributed!
 
 ## [Unreleased]
 
+### Added
+
+### Changed
+
+### Deprecated
+
+### Fixed
+
+### Removed
+
+### Security
+
+### Contributors
+Thank you to all who have contributed!
+- @<your-username>
+
+## [0.14.0-alpha] - 2023-12-15
+
 ### Added
 - Adds top-level IR node creation functions.
 - Adds `componentN` functions (destructuring) to IR nodes via Kotlin data classes
@@ -43,29 +61,38 @@ Thank you to all who have contributed!
 - **Breaking** The default integer literal type is now 32-bit; if the literal can not fit in a 32-bit integer, it overflows to 64-bit. 
 - **BREAKING** `PartiQLValueType` now distinguishes between Arbitrary Precision Decimal and Fixed Precision Decimal. 
 - **BREAKING** Function Signature Changes. Now Function signature has two subclasses, `Scalar` and `Aggregation`. 
+- **BREAKING** Plugin Changes. Only return one Connector.Factory, use Kotlin fields. JVM signature remains the same.
 - **BREAKING** In the produced plan: 
   - The new plan is fully resolved and typed.
   - Operators will be converted to function call. 
 - Changes the return type of `filter_distinct` to a list if input collection is list
+- Changes the `PartiQLValue` collections to implement Iterable rather than Sequence, allowing for multiple consumption.
+- **BREAKING** Moves PartiQLParserBuilder.standard().build() to be PartiQLParser.default().
+- **BREAKING** Changed modeling of `EXCLUDE` in `partiql-ast`
 
 ### Deprecated
 
 ### Fixed
 - Fixes the CLI hanging on invalid queries. See issue #1230.
 - Fixes Timestamp Type parsing issue. Previously Timestamp Type would get parsed to a Time type.
+- Fixes PIVOT parsing to assign the key and value as defined by spec section 14.
 - Fixes the physical plan compiler to return list when `DISTINCT` used with `ORDER BY`
 
 ### Removed
 - **Breaking** Removed IR factory in favor of static top-level functions. Change `Ast.foo()`
   to `foo()`
 - **Breaking** Removed `org.partiql.lang.planner.transforms.AstToPlan`. Use `org.partiql.planner.PartiQLPlanner`. 
+- **Breaking** Removed `org.partiql.lang.planner.transforms.PartiQLSchemaInferencer`. In order to achieve the same functionality, one would need to use the `org.partiql.planner.PartiQLPlanner`.
+  - To get the inferred type of the query result, one can do: `(plan.statement as Statement.Query).root.type`
 
 ### Security
 
 ### Contributors
 Thank you to all who have contributed!
 - @rchowell
 - @johnedquinn
+- @yliuuuu
+- @alancai98
 
 ## [0.13.2-alpha] - 2023-09-29
 
@@ -920,6 +947,7 @@ breaking changes if migrating from v0.9.2. The breaking changes accidentally int
 Initial alpha release of PartiQL.
 
 [Unreleased]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.13.2-alpha...HEAD
+[0.14.0-alpha]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.13.2-alpha...v0.14.0-alpha
 [0.13.2-alpha]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.13.1-alpha...v0.13.2-alpha
 [0.13.1-alpha]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.13.0-alpha...v0.13.1-alpha
 [0.13.0-alpha]: https://github.com/partiql/partiql-lang-kotlin/compare/v0.12.0-alpha...v0.13.0-alpha

README.md

@@ -31,7 +31,7 @@ This project is published to [Maven Central](https://search.maven.org/artifact/o
 
 | Group ID      | Artifact ID           | Recommended Version |
 |---------------|-----------------------|---------------------|
-| `org.partiql` | `partiql-lang-kotlin` | `0.13.2`            |
+| `org.partiql` | `partiql-lang-kotlin` | `0.14.0`            |
 
 
 For Maven builds, add the following to your `pom.xml`:

gradle.properties

@@ -1,5 +1,5 @@
 group=org.partiql
-version=0.14.0-SNAPSHOT
+version=0.14.1-SNAPSHOT
 
 ossrhUsername=EMPTY
 ossrhPassword=EMPTY

partiql-ast/src/main/kotlin/org/partiql/ast/helpers/ToLegacyAst.kt

@@ -708,9 +708,12 @@ private class AstTranslator(val metas: Map<String, MetaContainer>) : AstBaseVisi
             projectExpr(expr, alias, metas)
         }
 
+    // !!
+    // Legacy AST mislabels key and value in PIVOT, swapping the order here to recreate bug for compatibility.
+    // !!
     override fun visitSelectPivot(node: Select.Pivot, ctx: Ctx) = translate(node) { metas ->
-        val value = visitExpr(node.value, ctx)
-        val key = visitExpr(node.key, ctx)
+        val key = visitExpr(node.value, ctx) // SWAP val -> key
+        val value = visitExpr(node.key, ctx) // SWAP key -> val
         projectPivot(value, key, metas)
     }
 
@@ -753,42 +756,42 @@ private class AstTranslator(val metas: Map<String, MetaContainer>) : AstBaseVisi
     }
 
     override fun visitExclude(node: Exclude, ctx: Ctx): PartiqlAst.ExcludeOp = translate(node) { metas ->
-        val excludeExprs = node.exprs.translate<PartiqlAst.ExcludeExpr>(ctx)
+        val excludeExprs = node.items.translate<PartiqlAst.ExcludeExpr>(ctx)
         excludeOp(excludeExprs, metas)
     }
 
-    override fun visitExcludeExcludeExpr(node: Exclude.ExcludeExpr, ctx: Ctx) = translate(node) { metas ->
-        val root = visitIdentifierSymbol(node.root, ctx)
+    override fun visitExcludeItem(node: Exclude.Item, ctx: Ctx) = translate(node) { metas ->
+        val root = visitExprVar(node.root, ctx)
         val steps = node.steps.translate<PartiqlAst.ExcludeStep>(ctx)
-        excludeExpr(root = root, steps = steps, metas)
+        excludeExpr(root = identifier_(root.name, root.case), steps = steps, metas)
     }
 
     override fun visitExcludeStep(node: Exclude.Step, ctx: Ctx) =
         super.visitExcludeStep(node, ctx) as PartiqlAst.ExcludeStep
 
-    override fun visitExcludeStepExcludeTupleAttr(node: Exclude.Step.ExcludeTupleAttr, ctx: Ctx) = translate(node) { metas ->
+    override fun visitExcludeStepStructField(node: Exclude.Step.StructField, ctx: Ctx) = translate(node) { metas ->
         val attr = node.symbol.symbol
         val case = node.symbol.caseSensitivity.toLegacyCaseSensitivity()
         excludeTupleAttr(identifier(attr, case), metas)
     }
 
-    override fun visitExcludeStepExcludeCollectionIndex(
-        node: Exclude.Step.ExcludeCollectionIndex,
+    override fun visitExcludeStepCollIndex(
+        node: Exclude.Step.CollIndex,
         ctx: Ctx
     ) = translate(node) { metas ->
         val index = node.index.toLong()
         excludeCollectionIndex(index, metas)
     }
 
-    override fun visitExcludeStepExcludeTupleWildcard(
-        node: Exclude.Step.ExcludeTupleWildcard,
+    override fun visitExcludeStepStructWildcard(
+        node: Exclude.Step.StructWildcard,
         ctx: Ctx
     ) = translate(node) { metas ->
         excludeTupleWildcard(metas)
     }
 
-    override fun visitExcludeStepExcludeCollectionWildcard(
-        node: Exclude.Step.ExcludeCollectionWildcard,
+    override fun visitExcludeStepCollWildcard(
+        node: Exclude.Step.CollWildcard,
         ctx: Ctx
     ) = translate(node) { metas ->
         excludeCollectionWildcard(metas)

partiql-ast/src/main/kotlin/org/partiql/ast/sql/SqlDialect.kt

@@ -86,32 +86,32 @@ public abstract class SqlDialect : AstBaseVisitor<SqlBlock, SqlBlock>() {
     override fun visitExclude(node: Exclude, head: SqlBlock): SqlBlock {
         var h = head
         h = h concat " EXCLUDE "
-        h = h concat list(start = null, end = null) { node.exprs }
+        h = h concat list(start = null, end = null) { node.items }
         return h
     }
 
-    override fun visitExcludeExcludeExpr(node: Exclude.ExcludeExpr, head: SqlBlock): SqlBlock {
+    override fun visitExcludeItem(node: Exclude.Item, head: SqlBlock): SqlBlock {
         var h = head
-        h = h concat visitIdentifierSymbol(node.root, SqlBlock.Nil)
+        h = h concat visitExprVar(node.root, SqlBlock.Nil)
         h = h concat list(delimiter = null, start = null, end = null) { node.steps }
         return h
     }
 
-    override fun visitExcludeStepExcludeCollectionIndex(node: Exclude.Step.ExcludeCollectionIndex, head: SqlBlock): SqlBlock {
+    override fun visitExcludeStepCollIndex(node: Exclude.Step.CollIndex, head: SqlBlock): SqlBlock {
         return head concat r("[${node.index}]")
     }
 
-    override fun visitExcludeStepExcludeTupleWildcard(node: Exclude.Step.ExcludeTupleWildcard, head: SqlBlock): SqlBlock {
+    override fun visitExcludeStepStructWildcard(node: Exclude.Step.StructWildcard, head: SqlBlock): SqlBlock {
         return head concat r(".*")
     }
 
-    override fun visitExcludeStepExcludeTupleAttr(node: Exclude.Step.ExcludeTupleAttr, head: SqlBlock): SqlBlock {
+    override fun visitExcludeStepStructField(node: Exclude.Step.StructField, head: SqlBlock): SqlBlock {
         var h = head concat r(".")
         h = h concat visitIdentifierSymbol(node.symbol, SqlBlock.Nil)
         return h
     }
 
-    override fun visitExcludeStepExcludeCollectionWildcard(node: Exclude.Step.ExcludeCollectionWildcard, head: SqlBlock): SqlBlock {
+    override fun visitExcludeStepCollWildcard(node: Exclude.Step.CollWildcard, head: SqlBlock): SqlBlock {
         return head concat r("[*]")
     }
 

partiql-ast/src/main/resources/partiql_ast.ion

@@ -563,17 +563,17 @@ select::[
 ]
 
 exclude::{
-  exprs: list::[exclude_expr],
+  items: list::[item],
   _: [
-    exclude_expr::{
-      root: '.identifier.symbol',
+    item::{
+      root: '.expr.var',
       steps: list::[step],
     },
     step::[
-      exclude_tuple_attr::{ symbol: '.identifier.symbol' },
-      exclude_collection_index::{ index: int },
-      exclude_tuple_wildcard::{},
-      exclude_collection_wildcard::{},
+      struct_field::{ symbol: '.identifier.symbol' },
+      coll_index::{ index: int },
+      struct_wildcard::{},
+      coll_wildcard::{},
     ]
   ]
 }

partiql-ast/src/test/kotlin/org/partiql/ast/helpers/ToLegacyAstTest.kt

@@ -536,10 +536,14 @@ class ToLegacyAstTest {
                     }
                 }
             },
-            expect("(project_pivot (lit 1) (lit 2))") {
+            expect("(project_pivot (lit 2) (lit 1))") {
                 selectPivot {
-                    value = exprLit(int32Value(1))
+                    // PIVOT 1 AT 2
+                    //  - 1 is the VALUE
+                    //  - 2 is the KEY
+                    // In the legacy implementation these were accidentally flipped
                     key = exprLit(int32Value(2))
+                    value = exprLit(int32Value(1))
                 }
             },
             expect("(project_value (lit null))") {

partiql-ast/src/test/kotlin/org/partiql/ast/sql/SqlDialectTest.kt

@@ -1073,11 +1073,9 @@ class SqlDialectTest {
                         type = From.Value.Type.SCAN
                     }
                     exclude = exclude {
-                        exprs += excludeExcludeExpr {
-                            root = id("t", Identifier.CaseSensitivity.INSENSITIVE)
-                            steps += excludeStepExcludeTupleAttr {
-                                symbol = id("a", Identifier.CaseSensitivity.INSENSITIVE)
-                            }
+                        items += excludeItem {
+                            root = v("t")
+                            steps += insensitiveExcludeStructField("a")
                         }
                     }
                 }
@@ -1090,21 +1088,21 @@ class SqlDialectTest {
                         type = From.Value.Type.SCAN
                     }
                     exclude = exclude {
-                        exprs += excludeExcludeExpr {
-                            root = id("a", Identifier.CaseSensitivity.INSENSITIVE)
-                            steps += insensitiveExcludeTupleAttr("b")
+                        items += excludeItem {
+                            root = v("a")
+                            steps += insensitiveExcludeStructField("b")
                         }
-                        exprs += excludeExcludeExpr {
-                            root = id("c", Identifier.CaseSensitivity.INSENSITIVE)
-                            steps += insensitiveExcludeTupleAttr("d")
+                        items += excludeItem {
+                            root = v("c")
+                            steps += insensitiveExcludeStructField("d")
                         }
-                        exprs += excludeExcludeExpr {
-                            root = id("e", Identifier.CaseSensitivity.INSENSITIVE)
-                            steps += insensitiveExcludeTupleAttr("f")
+                        items += excludeItem {
+                            root = v("e")
+                            steps += insensitiveExcludeStructField("f")
                         }
-                        exprs += excludeExcludeExpr {
-                            root = id("g", Identifier.CaseSensitivity.INSENSITIVE)
-                            steps += insensitiveExcludeTupleAttr("h")
+                        items += excludeItem {
+                            root = v("g")
+                            steps += insensitiveExcludeStructField("h")
                         }
                     }
                 }
@@ -1117,37 +1115,37 @@ class SqlDialectTest {
                         type = From.Value.Type.SCAN
                     }
                     exclude = exclude {
-                        exprs += excludeExcludeExpr {
-                            root = id("t", Identifier.CaseSensitivity.INSENSITIVE)
+                        items += excludeItem {
+                            root = v("t")
                             steps += mutableListOf(
-                                insensitiveExcludeTupleAttr("a"),
-                                sensitiveExcludeTupleAttr("b"),
-                                excludeStepExcludeTupleWildcard(),
-                                excludeStepExcludeCollectionWildcard(),
-                                insensitiveExcludeTupleAttr("c"),
+                                insensitiveExcludeStructField("a"),
+                                sensitiveExcludeStructField("b"),
+                                excludeStepStructWildcard(),
+                                excludeStepCollWildcard(),
+                                insensitiveExcludeStructField("c"),
                             )
                         }
-                        exprs += excludeExcludeExpr {
-                            root = id("s", Identifier.CaseSensitivity.SENSITIVE)
+                        items += excludeItem {
+                            root = exprVar(id("s", Identifier.CaseSensitivity.SENSITIVE), Expr.Var.Scope.DEFAULT)
                             steps += mutableListOf(
-                                excludeStepExcludeCollectionIndex(0),
-                                insensitiveExcludeTupleAttr("d"),
-                                sensitiveExcludeTupleAttr("e"),
-                                excludeStepExcludeCollectionWildcard(),
-                                insensitiveExcludeTupleAttr("f"),
-                                excludeStepExcludeTupleWildcard(),
+                                excludeStepCollIndex(0),
+                                insensitiveExcludeStructField("d"),
+                                sensitiveExcludeStructField("e"),
+                                excludeStepCollWildcard(),
+                                insensitiveExcludeStructField("f"),
+                                excludeStepStructWildcard(),
                             )
                         }
                     }
                 }
             },
         )
 
-        private fun AstBuilder.insensitiveExcludeTupleAttr(str: String) = excludeStepExcludeTupleAttr {
+        private fun AstBuilder.insensitiveExcludeStructField(str: String) = excludeStepStructField {
             symbol = id(str, Identifier.CaseSensitivity.INSENSITIVE)
         }
 
-        private fun AstBuilder.sensitiveExcludeTupleAttr(str: String) = excludeStepExcludeTupleAttr {
+        private fun AstBuilder.sensitiveExcludeStructField(str: String) = excludeStepStructField {
             symbol = id(str, Identifier.CaseSensitivity.SENSITIVE)
         }
 

partiql-cli/src/main/kotlin/org/partiql/cli/Main.kt

@@ -17,19 +17,16 @@ package org.partiql.cli
 
 import AstPrinter
 import com.amazon.ion.system.IonSystemBuilder
-import com.amazon.ionelement.api.field
-import com.amazon.ionelement.api.ionString
-import com.amazon.ionelement.api.ionStructOf
 import org.partiql.cli.pico.PartiQLCommand
 import org.partiql.cli.shell.info
 import org.partiql.lang.eval.EvaluationSession
-import org.partiql.parser.PartiQLParserBuilder
+import org.partiql.parser.PartiQLParser
 import org.partiql.plan.debug.PlanPrinter
 import org.partiql.planner.PartiQLPlanner
-import org.partiql.planner.PartiQLPlannerBuilder
-import org.partiql.plugins.local.LocalPlugin
+import org.partiql.plugins.local.LocalConnector
 import picocli.CommandLine
 import java.io.PrintStream
+import java.nio.file.Paths
 import java.util.UUID
 import kotlin.system.exitProcess
 
@@ -53,15 +50,14 @@ object Debug {
 
     private const val USER_ID = "DEBUG_USER_ID"
 
-    private val plugins = listOf(LocalPlugin())
-    private val catalogs = mapOf(
-        "local" to ionStructOf(
-            field("connector_name", ionString("local")),
-        )
-    )
+    private val root = Paths.get(System.getProperty("user.home")).resolve(".partiql/local")
 
-    private val planner = PartiQLPlannerBuilder().plugins(plugins).build()
-    private val parser = PartiQLParserBuilder.standard().build()
+    private val planner = PartiQLPlanner.builder()
+        .catalogs(
+            "local" to LocalConnector.Metadata(root)
+        )
+        .build()
+    private val parser = PartiQLParser.default()
 
     // !!
     // IMPLEMENT DEBUG BEHAVIOR HERE
@@ -80,7 +76,6 @@ object Debug {
         val sess = PartiQLPlanner.Session(
             queryId = UUID.randomUUID().toString(),
             userId = "debug",
-            catalogConfig = catalogs,
         )
         val result = planner.plan(statement, sess).plan
         out.info("-- Plan ----------")