Skip to content

Commit a8f4472

Browse files
alancai98alancai98
authored
Merge 4c4062f into ff60c72 
2 parents ff60c72 + 4c4062f commit a8f4472

35 files changed

+1166
-390
lines changed

‎CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ Thank you to all who have contributed!
4646
- **BREAKING** In the produced plan:
4747
- The new plan is fully resolved and typed.
4848
- Operators will be converted to function call.
49+
- **EXPERIMENTAL** `EXCLUDE` addition to the physical plan
50+
- This is currently marked as experimental until the RFC is approved https://github.com/partiql/partiql-lang/issues/27
51+
- Evaluation of `EXCLUDE` in the `PlanCompiler`
4952
- Changes the return type of `filter_distinct` to a list if input collection is list
5053

5154
### Deprecated

‎partiql-ast/src/main/pig/partiql.ion

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -799,6 +799,9 @@ may then be further optimized by selecting better implementations of each operat
799799

800800
// For every row of `source`, adds each specified `let_binding`.
801801
(let source::bexpr bindings::(* let_binding 1))
802+
803+
// For every row of `source`, omits the values specified by `exclude_expr`s
804+
(exclude_clause source::bexpr exprs::(* exclude_expr 1))
802805
)
803806
)
804807

@@ -872,6 +875,7 @@ may then be further optimized by selecting better implementations of each operat
872875
column_component
873876
returning_mapping
874877
assignment
878+
exclude_op
875879
)
876880
)
877881
)
@@ -934,6 +938,9 @@ may then be further optimized by selecting better implementations of each operat
934938
)
935939
)
936940

941+
(exclude exclude_expr)
942+
(include (product exclude_expr root::int steps::(* exclude_step 1)))
943+
937944
// Replace statement.dml.target with statement.dml.uniqueId (the "resolved" corollary).
938945
(with statement
939946
(exclude dml)
@@ -1007,6 +1014,8 @@ may then be further optimized by selecting better implementations of each operat
10071014
// Notice that the physical window operator contains a list of window expression
10081015
// That is because, we want to combine the window functions that are operating on the same window to a single window operator
10091016
(window i::impl source:: bexpr window_specification:: over window_expression_list:: (* window_expression 1))
1017+
1018+
(exclude_clause i::impl source::bexpr exprs::(* exclude_expr 1))
10101019
)
10111020
)
10121021
)

‎partiql-eval/src/main/kotlin/org/partiql/lang/compiler/PartiQLCompilerBuilder.kt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import org.partiql.lang.eval.builtins.storedprocedure.StoredProcedure
2424
import org.partiql.lang.eval.internal.builtins.SCALAR_BUILTINS_DEFAULT
2525
import org.partiql.lang.eval.internal.builtins.definitionalBuiltins
2626
import org.partiql.lang.eval.physical.operators.AggregateOperatorFactoryDefault
27+
import org.partiql.lang.eval.physical.operators.ExcludeRelationalOperatorFactoryDefault
2728
import org.partiql.lang.eval.physical.operators.FilterRelationalOperatorFactoryDefault
2829
import org.partiql.lang.eval.physical.operators.JoinRelationalOperatorFactoryDefault
2930
import org.partiql.lang.eval.physical.operators.LetRelationalOperatorFactoryDefault
@@ -87,6 +88,7 @@ class PartiQLCompilerBuilder private constructor() {
8788
// Notice here we will not propagate the optin requirement to the user
8889
@OptIn(ExperimentalWindowFunctions::class)
8990
WindowRelationalOperatorFactoryDefault,
91+
ExcludeRelationalOperatorFactoryDefault,
9092
)
9193

9294
@JvmStatic

‎partiql-eval/src/main/kotlin/org/partiql/lang/eval/internal/AnyOfCastTable.kt

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,9 @@ import org.partiql.lang.ast.sourceLocation
88
import org.partiql.lang.eval.EvaluationException
99
import org.partiql.lang.eval.ExprValue
1010
import org.partiql.lang.eval.ExprValueType
11-
import org.partiql.lang.eval.StructOrdering
1211
import org.partiql.lang.eval.internal.ext.isUnknown
13-
import org.partiql.lang.eval.name
14-
import org.partiql.lang.eval.namedValue
12+
import org.partiql.lang.eval.internal.ext.name
13+
import org.partiql.lang.eval.internal.ext.namedValue
1514
import org.partiql.lang.types.StaticTypeUtils
1615
import org.partiql.types.AnyOfType
1716
import org.partiql.types.AnyType
@@ -228,19 +227,19 @@ internal class AnyOfCastTable(
228227
val children = source.asSequence().map { cast(it) }
229228

230229
when (targetType) {
231-
ExprValueType.LIST -> ExprValue.newList(children)
232-
ExprValueType.SEXP -> ExprValue.newSexp(children)
233-
ExprValueType.BAG -> ExprValue.newBag(children)
230+
ExprValueType.LIST -> ListExprValue(children)
231+
ExprValueType.SEXP -> SexpExprValue(children)
232+
ExprValueType.BAG -> BagExprValue(children)
234233
ExprValueType.STRUCT -> {
235234
if (source.type != ExprValueType.STRUCT) {
236235
// Should not be possible
237236
throw IllegalStateException("Cannot cast from non-struct to struct")
238237
}
239-
ExprValue.Companion.newStruct(
240-
children.zip(source.asSequence()).map { (child, original) ->
238+
StructExprValue(
239+
sequence = children.zip(source.asSequence()).map { (child, original) ->
241240
child.namedValue(original.name!!)
242241
},
243-
StructOrdering.UNORDERED
242+
ordering = StructOrdering.UNORDERED
244243
)
245244
}
246245
else -> throw IllegalStateException("Invalid collection target type: $targetType")

‎partiql-eval/src/main/kotlin/org/partiql/lang/eval/internal/IonStructBindings.kt

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
* Copyright 2019 Amazon.com, Inc. or its affiliates. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at:
7+
*
8+
* http://aws.amazon.com/apache2.0/
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific
12+
* language governing permissions and limitations under the License.
13+
*/
14+
15+
package org.partiql.lang.eval.internal
16+
17+
import com.amazon.ion.IonStruct
18+
import com.amazon.ion.IonValue
19+
import org.partiql.errors.ErrorCode
20+
import org.partiql.errors.Property
21+
import org.partiql.lang.eval.BindingCase
22+
import org.partiql.lang.eval.BindingName
23+
import org.partiql.lang.eval.Bindings
24+
import org.partiql.lang.eval.EvaluationException
25+
import org.partiql.lang.eval.ExprValue
26+
import org.partiql.lang.eval.internal.ext.namedValue
27+
import org.partiql.lang.util.propertyValueMapOf
28+
import org.partiql.lang.util.to
29+
30+
internal fun errAmbiguousBinding(bindingName: String, matchingNames: List<String>): Nothing {
31+
err(
32+
"Multiple matches were found for the specified identifier",
33+
ErrorCode.EVALUATOR_AMBIGUOUS_BINDING,
34+
propertyValueMapOf(
35+
Property.BINDING_NAME to bindingName,
36+
Property.BINDING_NAME_MATCHES to matchingNames.joinToString(", ")
37+
),
38+
internal = false
39+
)
40+
}
41+
42+
/**
43+
* Custom implementation of [Bindings] that lazily computes case sensitive or insensitive hash tables which
44+
* will speed up the lookup of bindings within structs.
45+
*
46+
* The key difference in behavior between this and other [Bindings] implementations is that it
47+
* can throw an ambiguous binding [EvaluationException] even for case-sensitive lookups as it is
48+
* entirely possible that fields with identical names can appear within [IonStruct]s.
49+
*
50+
* Important: this class is critical to performance for many queries. Change with caution.
51+
*/
52+
internal class IonStructBindings(private val myStruct: IonStruct) : Bindings<ExprValue> {
53+
54+
private val caseInsensitiveFieldMap by lazy {
55+
HashMap<String, ArrayList<IonValue>>().apply {
56+
for (field in myStruct) {
57+
val entries = getOrPut(field.fieldName.lowercase()) { ArrayList(1) }
58+
entries.add(field)
59+
}
60+
}
61+
}
62+
63+
private val caseSensitiveFieldMap by lazy {
64+
HashMap<String, ArrayList<IonValue>>().apply {
65+
for (field in myStruct) {
66+
val entries = getOrPut(field.fieldName) { ArrayList(1) }
67+
entries.add(field)
68+
}
69+
}
70+
}
71+
72+
private fun caseSensitiveLookup(fieldName: String): IonValue? =
73+
caseSensitiveFieldMap[fieldName]?.let { entries -> handleMatches(entries, fieldName) }
74+
75+
private fun caseInsensitiveLookup(fieldName: String): IonValue? =
76+
caseInsensitiveFieldMap[fieldName.lowercase()]?.let { entries -> handleMatches(entries, fieldName) }
77+
78+
private fun handleMatches(entries: List<IonValue>, fieldName: String): IonValue? =
79+
when (entries.size) {
80+
0 -> null
81+
1 -> entries[0]
82+
else ->
83+
errAmbiguousBinding(fieldName, entries.map { it.fieldName })
84+
}
85+
86+
override operator fun get(bindingName: BindingName): ExprValue? =
87+
when (bindingName.bindingCase) {
88+
BindingCase.SENSITIVE -> caseSensitiveLookup(bindingName.name)
89+
BindingCase.INSENSITIVE -> caseInsensitiveLookup(bindingName.name)
90+
}?.let {
91+
ionValueToExprValue(it).namedValue(ExprValue.newString(it.fieldName))
92+
}
93+
}

‎partiql-eval/src/main/kotlin/org/partiql/lang/eval/internal/IonValueToExprValue.kt

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
package org.partiql.lang.eval.internal
2+
3+
import com.amazon.ion.IonBlob
4+
import com.amazon.ion.IonBool
5+
import com.amazon.ion.IonClob
6+
import com.amazon.ion.IonDatagram
7+
import com.amazon.ion.IonDecimal
8+
import com.amazon.ion.IonFloat
9+
import com.amazon.ion.IonInt
10+
import com.amazon.ion.IonList
11+
import com.amazon.ion.IonSexp
12+
import com.amazon.ion.IonString
13+
import com.amazon.ion.IonStruct
14+
import com.amazon.ion.IonSymbol
15+
import com.amazon.ion.IonTimestamp
16+
import com.amazon.ion.IonValue
17+
import org.partiql.lang.eval.BAG_ANNOTATION
18+
import org.partiql.lang.eval.Bindings
19+
import org.partiql.lang.eval.DATE_ANNOTATION
20+
import org.partiql.lang.eval.ExprValue
21+
import org.partiql.lang.eval.GRAPH_ANNOTATION
22+
import org.partiql.lang.eval.MISSING_ANNOTATION
23+
import org.partiql.lang.eval.TIME_ANNOTATION
24+
import org.partiql.lang.eval.internal.ext.namedValue
25+
import org.partiql.lang.eval.time.Time
26+
import org.partiql.lang.graph.ExternalGraphReader
27+
import org.partiql.lang.util.bytesValue
28+
import java.math.BigDecimal
29+
30+
/**
31+
* Creates a new [ExprValue] instance from any Ion value.
32+
*
33+
* If possible, prefer the use of the other methods instead because they might return [ExprValue] instances
34+
* that are better optimized for their specific data type (depending on implementation).
35+
*/
36+
internal fun ionValueToExprValue(value: IonValue): ExprValue {
37+
return when {
38+
value.isNullValue && value.hasTypeAnnotation(MISSING_ANNOTATION) -> ExprValue.missingValue // MISSING
39+
value.isNullValue -> ExprValue.newNull(value.type) // NULL
40+
value is IonBool -> ExprValue.newBoolean(value.booleanValue()) // BOOL
41+
value is IonInt -> ExprValue.newInt(value.longValue()) // INT
42+
value is IonFloat -> ExprValue.newFloat(value.doubleValue()) // FLOAT
43+
value is IonDecimal -> ExprValue.newDecimal(value.decimalValue()) // DECIMAL
44+
value is IonTimestamp && value.hasTypeAnnotation(DATE_ANNOTATION) -> { // DATE
45+
val timestampValue = value.timestampValue()
46+
ExprValue.newDate(timestampValue.year, timestampValue.month, timestampValue.day)
47+
}
48+
value is IonTimestamp -> ExprValue.newTimestamp(value.timestampValue()) // TIMESTAMP
49+
value is IonStruct && value.hasTypeAnnotation(TIME_ANNOTATION) -> { // TIME
50+
val hourValue = (value["hour"] as IonInt).intValue()
51+
val minuteValue = (value["minute"] as IonInt).intValue()
52+
val secondInDecimal = (value["second"] as IonDecimal).decimalValue()
53+
val secondValue = secondInDecimal.toInt()
54+
val nanoValue = secondInDecimal.remainder(BigDecimal.ONE).multiply(NANOS_PER_SECOND.toBigDecimal()).toInt()
55+
val timeZoneHourValue = (value["timezone_hour"] as IonInt).intValue()
56+
val timeZoneMinuteValue = (value["timezone_minute"] as IonInt).intValue()
57+
ExprValue.newTime(
58+
Time.of(
59+
hourValue,
60+
minuteValue,
61+
secondValue,
62+
nanoValue,
63+
secondInDecimal.scale(),
64+
timeZoneHourValue * 60 + timeZoneMinuteValue
65+
)
66+
)
67+
}
68+
value is IonStruct && value.hasTypeAnnotation(GRAPH_ANNOTATION) -> // GRAPH
69+
ExprValue.newGraph(ExternalGraphReader.read(value))
70+
value is IonSymbol -> ExprValue.newSymbol(value.stringValue()) // SYMBOL
71+
value is IonString -> ExprValue.newString(value.stringValue()) // STRING
72+
value is IonClob -> ExprValue.newClob(value.bytesValue()) // CLOB
73+
value is IonBlob -> ExprValue.newBlob(value.bytesValue()) // BLOB
74+
value is IonList && value.hasTypeAnnotation(BAG_ANNOTATION) -> BagExprValue(value.map { ionValueToExprValue(it) }) // BAG
75+
value is IonList -> ListExprValue(value.map { ionValueToExprValue(it) }) // LIST
76+
value is IonSexp -> SexpExprValue(value.map { ionValueToExprValue(it) }) // SEXP
77+
value is IonStruct -> IonStructExprValue(value) // STRUCT
78+
value is IonDatagram -> BagExprValue(value.map { ionValueToExprValue(it) }) // DATAGRAM represented as BAG ExprValue
79+
else -> error("Unrecognized IonValue to transform to ExprValue: $value")
80+
}
81+
}
82+
83+
private class IonStructExprValue(
84+
ionStruct: IonStruct
85+
) : StructExprValue(
86+
StructOrdering.UNORDERED,
87+
ionStruct.asSequence().map { ionValueToExprValue(it).namedValue(ExprValue.newString(it.fieldName)) }
88+
) {
89+
override val bindings: Bindings<ExprValue> =
90+
IonStructBindings(ionStruct)
91+
}

‎partiql-eval/src/main/kotlin/org/partiql/lang/eval/internal/NamedExprValue.kt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package org.partiql.lang.eval.internal
2+
3+
import org.partiql.lang.eval.ExprValue
4+
import org.partiql.lang.eval.Named
5+
import org.partiql.lang.eval.stringify
6+
import org.partiql.lang.util.downcast
7+
8+
/**
9+
* An [ExprValue] that also implements [Named].
10+
*/
11+
internal class NamedExprValue(override val name: ExprValue, val value: ExprValue) : ExprValue by value, Named {
12+
override fun <T : Any?> asFacet(type: Class<T>?): T? = downcast(type) ?: value.asFacet(type)
13+
14+
override fun toString(): String = stringify()
15+
}

‎partiql-eval/src/main/kotlin/org/partiql/lang/eval/internal/SequenceExprValue.kt

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package org.partiql.lang.eval.internal
2+
3+
import org.partiql.lang.eval.BaseExprValue
4+
import org.partiql.lang.eval.ExprValue
5+
import org.partiql.lang.eval.ExprValueType
6+
import org.partiql.lang.eval.OrdinalBindings
7+
import org.partiql.lang.eval.internal.ext.namedValue
8+
9+
internal class ListExprValue(val values: Sequence<ExprValue>) : BaseExprValue() {
10+
override val type = ExprValueType.LIST
11+
override val ordinalBindings by lazy { OrdinalBindings.ofList(toList()) }
12+
override fun iterator() = values.mapIndexed { i, v -> v.namedValue(ExprValue.newInt(i)) }.iterator()
13+
14+
constructor(values: List<ExprValue>) : this(values.asSequence())
15+
}
16+
17+
internal class BagExprValue(val values: Sequence<ExprValue>) : BaseExprValue() {
18+
override val type = ExprValueType.BAG
19+
override val ordinalBindings = OrdinalBindings.EMPTY
20+
override fun iterator() = values.iterator()
21+
22+
constructor(values: List<ExprValue>) : this(values.asSequence())
23+
}
24+
25+
internal class SexpExprValue(val values: Sequence<ExprValue>) : BaseExprValue() {
26+
override val type = ExprValueType.SEXP
27+
override val ordinalBindings by lazy { OrdinalBindings.ofList(toList()) }
28+
override fun iterator() = values.mapIndexed { i, v -> v.namedValue(ExprValue.newInt(i)) }.iterator()
29+
30+
constructor(values: List<ExprValue>) : this(values.asSequence())
31+
}
32+
33+
/**
34+
* Returns an [ExprValue] created from a sequence of [seq]. Requires [type] to be a sequence type
35+
* (i.e. [ExprValueType.isSequence] == true).
36+
*/
37+
internal fun newSequenceExprValue(type: ExprValueType, seq: Sequence<ExprValue>): ExprValue {
38+
return when (type) {
39+
ExprValueType.LIST -> ListExprValue(seq)
40+
ExprValueType.BAG -> BagExprValue(seq)
41+
ExprValueType.SEXP -> SexpExprValue(seq)
42+
else -> error("Sequence type required")
43+
}
44+
}

‎partiql-eval/src/main/kotlin/org/partiql/lang/eval/internal/StructExprValue.kt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,10 @@ internal enum class StructOrdering {
3434
* Provides a [ExprValueType.STRUCT] implementation lazily backed by a sequence.
3535
*/
3636
internal open class StructExprValue(
37-
private val ordering: StructOrdering,
37+
internal val ordering: StructOrdering,
3838
private val sequence: Sequence<ExprValue>
3939
) : BaseExprValue() {
40+
constructor(fields: List<ExprValue>, ordering: StructOrdering) : this(ordering, fields.asSequence())
4041

4142
override val type = ExprValueType.STRUCT
4243

‎partiql-eval/src/main/kotlin/org/partiql/lang/eval/internal/builtins/Accumulator.kt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import org.partiql.lang.eval.ExprValue
2020
import org.partiql.lang.eval.ExprValueType
2121
import org.partiql.lang.eval.NaturalExprValueComparators
2222
import org.partiql.lang.eval.booleanValue
23+
import org.partiql.lang.eval.internal.BagExprValue
2324
import org.partiql.lang.eval.internal.ExprAggregator
2425
import org.partiql.lang.eval.internal.errNoContext
2526
import org.partiql.lang.eval.internal.ext.bigDecimalOf
@@ -173,7 +174,7 @@ internal class AccumulatorGroupAs(
173174
exprValues.add(value)
174175
}
175176

176-
override fun compute(): ExprValue = ExprValue.newBag(exprValues)
177+
override fun compute(): ExprValue = BagExprValue(exprValues)
177178
}
178179

179180
private fun comparisonAccumulator(comparator: NaturalExprValueComparators): (ExprValue?, ExprValue) -> ExprValue =

0 commit comments

Comments
 (0)