Skip to content

Commit e5a1e24

Browse files
johnedquinnjohnedquinn
authored
Merge pull request #1379 from partiql/partiql-eval-perf
Adds performance optimizations for materialization
2 parents f2369ba + 5a92d2b commit e5a1e24

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+658
-356
lines changed

‎partiql-eval/src/main/kotlin/org/partiql/eval/PartiQLEngineDefault.kt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
package org.partiql.eval
22

33
import org.partiql.eval.internal.Compiler
4-
import org.partiql.eval.internal.Record
4+
import org.partiql.eval.internal.Environment
55
import org.partiql.eval.internal.Symbols
66
import org.partiql.plan.PartiQLPlan
77
import org.partiql.value.PartiQLValue
@@ -19,7 +19,7 @@ internal class PartiQLEngineDefault : PartiQLEngine {
1919
val expression = compiler.compile()
2020
return object : PartiQLStatement.Query {
2121
override fun execute(): PartiQLValue {
22-
return expression.eval(Record.empty)
22+
return expression.eval(Environment.empty)
2323
}
2424
}
2525
} catch (ex: Exception) {

‎partiql-eval/src/main/kotlin/org/partiql/eval/internal/Compiler.kt

Lines changed: 22 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -59,20 +59,6 @@ internal class Compiler(
5959
private val symbols: Symbols
6060
) : PlanBaseVisitor<Operator, StaticType?>() {
6161

62-
/**
63-
* The variables environment
64-
*/
65-
private val env: Environment = Environment()
66-
67-
/**
68-
* Aids in determining the index by which we will query [Environment.get] for [Rex.Op.Var.depth].
69-
*
70-
* @see scope
71-
* @see Environment
72-
* @see Rex.Op.Var
73-
*/
74-
private var scopeSize = 0
75-
7662
fun compile(): Operator.Expr {
7763
return visitPartiQLPlan(plan, null)
7864
}
@@ -93,12 +79,10 @@ internal class Compiler(
9379
throw IllegalStateException(node.message)
9480
}
9581

96-
// TODO: Re-look at
9782
override fun visitPartiQLPlan(node: PartiQLPlan, ctx: StaticType?): Operator.Expr {
9883
return visitStatement(node.statement, ctx) as Operator.Expr
9984
}
10085

101-
// TODO: Re-look at
10286
override fun visitStatementQuery(node: Statement.Query, ctx: StaticType?): Operator.Expr {
10387
return visitRex(node.root, ctx).modeHandled()
10488
}
@@ -122,35 +106,29 @@ internal class Compiler(
122106
return ExprStruct(fields)
123107
}
124108

125-
override fun visitRexOpSelect(node: Rex.Op.Select, ctx: StaticType?): Operator.Expr {
126-
return scope {
127-
val rel = visitRel(node.rel, ctx)
128-
val ordered = node.rel.type.props.contains(Rel.Prop.ORDERED)
129-
val constructor = visitRex(node.constructor, ctx).modeHandled()
130-
ExprSelect(rel, constructor, ordered, env)
131-
}
109+
override fun visitRexOpSelect(node: Rex.Op.Select, ctx: StaticType?): Operator {
110+
val rel = visitRel(node.rel, ctx)
111+
val ordered = node.rel.type.props.contains(Rel.Prop.ORDERED)
112+
val constructor = visitRex(node.constructor, ctx).modeHandled()
113+
return ExprSelect(rel, constructor, ordered)
132114
}
133115

134116
override fun visitRexOpSubquery(node: Rex.Op.Subquery, ctx: StaticType?): Operator {
135-
return scope {
136-
val constructor = visitRex(node.constructor, ctx)
137-
val input = visitRel(node.rel, ctx)
138-
when (node.coercion) {
139-
Rex.Op.Subquery.Coercion.SCALAR -> ExprSubquery.Scalar(constructor, input, env)
140-
Rex.Op.Subquery.Coercion.ROW -> ExprSubquery.Row(constructor, input, env)
141-
}
117+
val constructor = visitRex(node.constructor, ctx)
118+
val input = visitRel(node.rel, ctx)
119+
return when (node.coercion) {
120+
Rex.Op.Subquery.Coercion.SCALAR -> ExprSubquery.Scalar(constructor, input)
121+
Rex.Op.Subquery.Coercion.ROW -> ExprSubquery.Row(constructor, input)
142122
}
143123
}
144124

145125
override fun visitRexOpPivot(node: Rex.Op.Pivot, ctx: StaticType?): Operator {
146-
return scope {
147-
val rel = visitRel(node.rel, ctx)
148-
val key = visitRex(node.key, ctx)
149-
val value = visitRex(node.value, ctx)
150-
when (session.mode) {
151-
PartiQLEngine.Mode.PERMISSIVE -> ExprPivotPermissive(rel, key, value)
152-
PartiQLEngine.Mode.STRICT -> ExprPivot(rel, key, value)
153-
}
126+
val rel = visitRel(node.rel, ctx)
127+
val key = visitRex(node.key, ctx)
128+
val value = visitRex(node.value, ctx)
129+
return when (session.mode) {
130+
PartiQLEngine.Mode.PERMISSIVE -> ExprPivotPermissive(rel, key, value)
131+
PartiQLEngine.Mode.STRICT -> ExprPivot(rel, key, value)
154132
}
155133
}
156134

@@ -164,8 +142,7 @@ internal class Compiler(
164142
return when (node.depth) {
165143
0 -> ExprVarLocal(node.ref)
166144
else -> {
167-
val index = scopeSize - node.depth
168-
ExprVarOuter(index, node.ref, env)
145+
ExprVarOuter(node.depth, node.ref)
169146
}
170147
}
171148
}
@@ -297,13 +274,13 @@ internal class Compiler(
297274

298275
override fun visitRelOpJoin(node: Rel.Op.Join, ctx: StaticType?): Operator {
299276
val lhs = visitRel(node.lhs, ctx)
300-
val rhs = scope { visitRel(node.rhs, ctx) }
277+
val rhs = visitRel(node.rhs, ctx)
301278
val condition = visitRex(node.rex, ctx)
302279
return when (node.type) {
303-
Rel.Op.Join.Type.INNER -> RelJoinInner(lhs, rhs, condition, env)
304-
Rel.Op.Join.Type.LEFT -> RelJoinLeft(lhs, rhs, condition, env)
305-
Rel.Op.Join.Type.RIGHT -> RelJoinRight(lhs, rhs, condition, env)
306-
Rel.Op.Join.Type.FULL -> RelJoinOuterFull(lhs, rhs, condition, env)
280+
Rel.Op.Join.Type.INNER -> RelJoinInner(lhs, rhs, condition)
281+
Rel.Op.Join.Type.LEFT -> RelJoinLeft(lhs, rhs, condition)
282+
Rel.Op.Join.Type.RIGHT -> RelJoinRight(lhs, rhs, condition)
283+
Rel.Op.Join.Type.FULL -> RelJoinOuterFull(lhs, rhs, condition)
307284
}
308285
}
309286

@@ -368,20 +345,4 @@ internal class Compiler(
368345
}
369346
return item
370347
}
371-
372-
/**
373-
* Figuratively creates a new scope by incrementing/decrementing the [scopeSize] before/after the [block] invocation.
374-
*
375-
* @see scopeSize
376-
* @see Environment
377-
* @see Rex.Op.Var
378-
*/
379-
private inline fun <T> scope(block: () -> T): T {
380-
scopeSize++
381-
try {
382-
return block.invoke()
383-
} finally {
384-
scopeSize--
385-
}
386-
}
387348
}

‎partiql-eval/src/main/kotlin/org/partiql/eval/internal/Environment.kt

Lines changed: 48 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,61 @@
11
package org.partiql.eval.internal
22

3-
import java.util.Stack
3+
import org.partiql.value.PartiQLValue
4+
import org.partiql.value.PartiQLValueExperimental
45

56
/**
6-
* This class represents the Variables Environment defined in the PartiQL Specification. It differs slightly as it does
7-
* not hold the "current" [Record]. The reason for this is that the PartiQL Maintainers have opted to use the Volcano
8-
* Model for query execution (see [org.partiql.eval.internal.operator.Operator.Relation.next] and
9-
* [org.partiql.eval.internal.operator.Operator.Expr.eval]), however, the use of the [Environment] is to provide the
10-
* functionality defined in the PartiQL Specification. It accomplishes this by wrapping the "outer" variables
11-
* environments (or [scopes]).
7+
* This class represents the Variables Environment defined in the PartiQL Specification.
128
*/
13-
internal class Environment {
9+
internal class Environment(
10+
private val bindings: Record,
11+
private val parent: Environment? = null
12+
) {
1413

15-
private val scopes: Stack<Record> = Stack<Record>()
14+
companion object {
15+
@JvmStatic
16+
val empty: Environment = Environment(Record.empty, null)
17+
}
1618

17-
/**
18-
* Creates a new scope using the [record] to execute the [block]. Pops the [record] once the [block] is done executing.
19-
*/
20-
internal inline fun <T> scope(record: Record, block: () -> T): T {
21-
scopes.push(record)
22-
try {
23-
return block.invoke()
24-
} catch (t: Throwable) {
25-
throw t
26-
} finally {
27-
scopes.pop()
28-
}
19+
@OptIn(PartiQLValueExperimental::class)
20+
operator fun get(index: Int): PartiQLValue {
21+
return this.bindings[index]
22+
}
23+
24+
@OptIn(PartiQLValueExperimental::class)
25+
fun getOrNull(index: Int): PartiQLValue? {
26+
return this.bindings.values.getOrNull(index)
27+
}
28+
29+
internal fun next(): Environment? {
30+
return this.parent
2931
}
3032

3133
/**
32-
* Gets the scope/record/variables-environment at the requested [index].
34+
* Returns a new [Environment] that contains the [record] and encloses the current [Environment]. This is used to:
35+
* 1. Pass on a [Record] from a Rel to a Rex. Consider `SELECT a + 1 FROM t`. The PROJECT would likely grab the input
36+
* record from the SCAN, [push] it into the current environment, and pass it to the Expr representing `a + 1`.
37+
* 2. Create a nested scope. Consider `SELECT 1 + (SELECT t1.a + t2.b FROM t2 LIMIT 1) FROM t1`. Since the inner
38+
* SELECT (ExprSubquery) is creating a "nested scope", it would invoke [push].
39+
*
40+
* Here are the general rules to follow:
41+
* 1. When evaluating Expressions from within a Relation, one should always use [push] to "push" onto the stack.
42+
* 2. When evaluating Relations from within an Expression, one should always use [push] to "push" onto the stack.
43+
* 3. When evaluating Expressions from within a Relation, there is no need to use [push].
44+
* 4. When evaluating Relations from within a Relation, one **might** want to use [push]. Consider the LATERAL JOIN, for instance.
45+
*
46+
* @see [org.partiql.eval.internal.operator.Operator.Expr]
47+
* @see [org.partiql.eval.internal.operator.Operator.Relation]
48+
* @see [org.partiql.eval.internal.operator.rex.ExprSubquery]
3349
*/
34-
operator fun get(index: Int): Record {
35-
return scopes[index]
50+
internal fun push(record: Record): Environment = Environment(
51+
record,
52+
this
53+
)
54+
55+
override fun toString(): String {
56+
return when (parent) {
57+
null -> bindings.toString()
58+
else -> "$bindings --> $parent"
59+
}
3660
}
3761
}

‎partiql-eval/src/main/kotlin/org/partiql/eval/internal/Record.kt

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,17 @@ internal data class Record(val values: Array<PartiQLValue>) {
3333
public operator fun get(index: Int): PartiQLValue {
3434
return this.values[index]
3535
}
36+
37+
override fun toString(): String {
38+
return buildString {
39+
append("< ")
40+
values.forEachIndexed { index, value ->
41+
append("$index: $value")
42+
if (index != values.lastIndex) {
43+
append(", ")
44+
}
45+
}
46+
append(" >")
47+
}
48+
}
3649
}

‎partiql-eval/src/main/kotlin/org/partiql/eval/internal/helpers/RecordValueIterator.kt

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package org.partiql.eval.internal.helpers
2+
3+
import org.partiql.eval.internal.Record
4+
import org.partiql.value.CollectionValue
5+
import org.partiql.value.PartiQLValueExperimental
6+
7+
/**
8+
* An [Iterator] over a [CollectionValue] lazily producing [Record]s as you call [next].
9+
*/
10+
@OptIn(PartiQLValueExperimental::class)
11+
internal class RecordValueIterator(
12+
collectionValue: CollectionValue<*>
13+
) : Iterator<Record> {
14+
15+
private val collectionIter = collectionValue.iterator()
16+
17+
override fun hasNext(): Boolean = collectionIter.hasNext()
18+
19+
override fun next(): Record = Record(Array(1) { collectionIter.next() })
20+
}

‎partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/Operator.kt

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.partiql.eval.internal.operator
22

3+
import org.partiql.eval.internal.Environment
34
import org.partiql.eval.internal.Record
45
import org.partiql.spi.fn.Agg
56
import org.partiql.spi.fn.FnExperimental
@@ -14,17 +15,15 @@ internal sealed interface Operator {
1415
interface Expr : Operator {
1516

1617
@OptIn(PartiQLValueExperimental::class)
17-
fun eval(record: Record): PartiQLValue
18+
fun eval(env: Environment): PartiQLValue
1819
}
1920

2021
/**
2122
* Relation operator represents an evaluable collection of binding tuples.
2223
*/
23-
interface Relation : Operator, AutoCloseable {
24+
interface Relation : Operator, AutoCloseable, Iterator<Record> {
2425

25-
fun open()
26-
27-
fun next(): Record?
26+
fun open(env: Environment)
2827

2928
override fun close()
3029
}

‎partiql-eval/src/main/kotlin/org/partiql/eval/internal/operator/rel/RelAggregate.kt

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.partiql.eval.internal.operator.rel
22

3+
import org.partiql.eval.internal.Environment
34
import org.partiql.eval.internal.Record
45
import org.partiql.eval.internal.operator.Operator
56
import org.partiql.spi.fn.Agg
@@ -56,13 +57,12 @@ internal class RelAggregate(
5657
)
5758

5859
@OptIn(PartiQLValueExperimental::class, FnExperimental::class)
59-
override fun open() {
60-
input.open()
61-
var inputRecord = input.next()
62-
while (inputRecord != null) {
60+
override fun open(env: Environment) {
61+
input.open(env)
62+
for (inputRecord in input) {
6363
// Initialize the AggregationMap
6464
val evaluatedGroupByKeys = keys.map {
65-
val key = it.eval(inputRecord!!)
65+
val key = it.eval(env.push(inputRecord))
6666
when (key.type == PartiQLValueType.MISSING) {
6767
true -> nullValue()
6868
false -> key
@@ -83,7 +83,7 @@ internal class RelAggregate(
8383

8484
// Aggregate Values in Aggregation State
8585
accumulators.forEachIndexed { index, function ->
86-
val valueToAggregate = function.args.map { it.eval(inputRecord!!) }
86+
val valueToAggregate = function.args.map { it.eval(env.push(inputRecord)) }
8787
// Skip over aggregation if NULL/MISSING
8888
if (valueToAggregate.any { it.type == PartiQLValueType.MISSING || it.isNull }) {
8989
return@forEachIndexed
@@ -94,7 +94,6 @@ internal class RelAggregate(
9494
}
9595
accumulators[index].delegate.next(valueToAggregate.toTypedArray())
9696
}
97-
inputRecord = input.next()
9897
}
9998

10099
// No Aggregations Created
@@ -116,12 +115,12 @@ internal class RelAggregate(
116115
}
117116
}
118117

119-
override fun next(): Record? {
120-
return if (records.hasNext()) {
121-
records.next()
122-
} else {
123-
null
124-
}
118+
override fun hasNext(): Boolean {
119+
return records.hasNext()
120+
}
121+
122+
override fun next(): Record {
123+
return records.next()
125124
}
126125

127126
@OptIn(PartiQLValueExperimental::class)

0 commit comments

Comments
 (0)