diff --git a/interactive_engine/compiler/Makefile b/interactive_engine/compiler/Makefile index 53fc79b55c36..e26c14678a5f 100644 --- a/interactive_engine/compiler/Makefile +++ b/interactive_engine/compiler/Makefile @@ -20,6 +20,9 @@ target.revision:=0.0.1-SNAPSHOT QUIET_OPT := --quiet +query:= +physical:= + build: cd $(CUR_DIR)/.. && \ mvn clean install -DskipTests -Drevision=${target.revision} -Pexperimental ${QUIET_OPT} && \ @@ -57,6 +60,14 @@ run: -Dpegasus.hosts=${pegasus.hosts} \ com.alibaba.graphscope.gremlin.service.GraphServiceMain +# make physical_plan query='' physical='' +physical_plan: + cd $(CUR_DIR) && $(java) \ + -cp ".:./target/libs/*:./target/compiler-0.0.1-SNAPSHOT.jar" \ + -Djna.library.path=../executor/ir/target/release \ + -Dgraph.schema=${graph.schema} \ + com.alibaba.graphscope.common.ir.tools.GraphPlanner "$(query)" "$(physical)" + # start rpc server # make run graph.schema:=../executor/ir/core/resource/ldbc_schema.json ldbc_test: diff --git a/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 b/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 index 8008a294810d..f824138d9440 100644 --- a/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 +++ b/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 @@ -26,6 +26,9 @@ oC_Statement : oC_Query ; oC_Query + : oC_RegularQuery ; + +oC_RegularQuery : oC_Match ( SP? oC_With )* ( SP oC_Return ) ; oC_Match diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java index a50865aa66a1..2766f40e311c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/IrPlan.java @@ -599,7 +599,7 @@ public Pointer createParams(QueryParams params) { } public IrPlan(IrMeta meta, InterOpCollection opCollection) { - irCoreLib.setSchema(meta.getSchemaJson()); + irCoreLib.setSchema(meta.getSchema().schemaJson()); this.ptrPlan = irCoreLib.initLogicalPlan(); // add snapshot to QueryParams for (InterOpBase op : opCollection.unmodifiableCollection()) { @@ -611,9 +611,10 @@ public IrPlan(IrMeta meta, InterOpCollection opCollection) { } else if (op instanceof GetVOp && ((GetVOp) op).getParams().isPresent()) { params = ((GetVOp) op).getParams().get(); } - if (params != null && meta.isAcquireSnapshot()) { + if (params != null && meta.getSnapshotId().isAcquired()) { params.addExtraParams( - QueryParams.SNAPSHOT_CONFIG_NAME, String.valueOf(meta.getSnapshotId())); + QueryParams.SNAPSHOT_CONFIG_NAME, + String.valueOf(meta.getSnapshotId().getId())); } } appendInterOpCollection(-1, opCollection); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/antlr4/Antlr4Parser.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/antlr4/Antlr4Parser.java new file mode 100644 index 000000000000..0a707f40f8c8 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/antlr4/Antlr4Parser.java @@ -0,0 +1,26 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.antlr4; + +import org.antlr.v4.runtime.tree.ParseTree; + +/** + * parse DSL statement to antlr tree + */ +public interface Antlr4Parser { + ParseTree parse(String statement); +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/LogicalPlanConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/LogicalPlanConverter.java deleted file mode 100644 index af8ef3b878fa..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/LogicalPlanConverter.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.ir.runtime; - -import com.alibaba.graphscope.common.ir.runtime.type.LogicalNode; -import com.alibaba.graphscope.common.ir.runtime.type.LogicalPlan; - -import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.RelShuttle; -import org.apache.calcite.rel.RelVisitor; -import org.apache.calcite.rel.logical.LogicalValues; -import org.checkerframework.checker.nullness.qual.Nullable; - -import java.util.Objects; - -/** - * convert logical plan in calcite to the structures in ir_core - * @param type of each node (i.e. {@link com.sun.jna.Pointer} for FFI interfaces) - * @param type of physical plan (i.e. {@link com.alibaba.graphscope.common.jna.type.FfiData for FFI interfaces}) - */ -public class LogicalPlanConverter extends RelVisitor { - private final LogicalPlan logicalPlan; - private final RelShuttle rexShuttle; - - public LogicalPlanConverter(RelShuttle relShuttle, LogicalPlan logicalPlan) { - this.rexShuttle = Objects.requireNonNull(relShuttle); - this.logicalPlan = Objects.requireNonNull(logicalPlan); - } - - @Override - public void visit(RelNode node, int ordinal, @Nullable RelNode parent) { - super.visit(node, ordinal, parent); - if (node instanceof LogicalValues) { - this.logicalPlan.setReturnEmpty(true); - } - if (!this.logicalPlan.isReturnEmpty()) { - this.logicalPlan.appendNode((LogicalNode) node.accept(rexShuttle)); - } - } - - @Override - public LogicalPlan go(RelNode p) { - replaceRoot(p); - visit(p, 0, null); - return this.logicalPlan; - } -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/RegularPhysicalBuilder.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/RegularPhysicalBuilder.java new file mode 100644 index 000000000000..5adcd5d09f70 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/RegularPhysicalBuilder.java @@ -0,0 +1,60 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.ir.runtime; + +import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode; +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelShuttle; +import org.apache.calcite.rel.RelVisitor; +import org.checkerframework.checker.nullness.qual.Nullable; + +/** + * build physical plan from logical plan of a regular query + * @param + * @param + */ +public abstract class RegularPhysicalBuilder extends PhysicalBuilder { + protected RelShuttle relShuttle; + + protected RegularPhysicalBuilder(LogicalPlan logicalPlan, RelShuttle relShuttle) { + super(logicalPlan); + this.relShuttle = relShuttle; + } + + protected void initialize() { + if (this.logicalPlan.getRegularQuery() != null && !this.logicalPlan.isReturnEmpty()) { + RelNode regularQuery = this.logicalPlan.getRegularQuery(); + RelVisitor relVisitor = + new RelVisitor() { + @Override + public void visit(RelNode node, int ordinal, @Nullable RelNode parent) { + super.visit(node, ordinal, parent); + appendNode((PhysicalNode) node.accept(relShuttle)); + } + }; + relVisitor.go(regularQuery); + } + } + + /** + * append {@code PhysicalNode} to the physical plan + * @param node + */ + protected abstract void appendNode(PhysicalNode node); +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiLogicalPlan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java similarity index 84% rename from interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiLogicalPlan.java rename to interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java index 346e7ed31398..1e5d2afbe611 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiLogicalPlan.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/FfiPhysicalBuilder.java @@ -16,10 +16,13 @@ package com.alibaba.graphscope.common.ir.runtime.ffi; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.PegasusConfig; import com.alibaba.graphscope.common.intermediate.ArgUtils; import com.alibaba.graphscope.common.ir.rel.GraphLogicalAggregate; import com.alibaba.graphscope.common.ir.rel.GraphLogicalProject; import com.alibaba.graphscope.common.ir.rel.GraphLogicalSort; +import com.alibaba.graphscope.common.ir.rel.GraphRelShuttleWrapper; import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalExpand; import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalGetV; import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalPathExpand; @@ -28,53 +31,55 @@ import com.alibaba.graphscope.common.ir.rel.graph.match.GraphLogicalSingleMatch; import com.alibaba.graphscope.common.ir.rel.type.group.GraphGroupKeys; import com.alibaba.graphscope.common.ir.rex.RexGraphVariable; +import com.alibaba.graphscope.common.ir.runtime.RegularPhysicalBuilder; import com.alibaba.graphscope.common.ir.runtime.proto.RexToProtoConverter; -import com.alibaba.graphscope.common.ir.runtime.type.LogicalNode; -import com.alibaba.graphscope.common.ir.runtime.type.LogicalPlan; +import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode; import com.alibaba.graphscope.common.ir.tools.AliasInference; +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.alibaba.graphscope.common.jna.IrCoreLibrary; -import com.alibaba.graphscope.common.jna.type.*; +import com.alibaba.graphscope.common.jna.type.FfiData; +import com.alibaba.graphscope.common.jna.type.FfiPbPointer; +import com.alibaba.graphscope.common.jna.type.FfiResult; +import com.alibaba.graphscope.common.jna.type.ResultCode; import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.gaia.proto.OuterExpression; import com.google.common.base.Preconditions; import com.sun.jna.Pointer; import com.sun.jna.ptr.IntByReference; -import org.apache.calcite.plan.RelOptCluster; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexVariable; -import org.apache.commons.lang3.StringUtils; import java.util.List; /** - * build ffi logical plan in ir core by jna invocation + * build physical plan from logical plan of a regular query, the physical plan is actually denoted by ir core structure (FFI Pointer) */ -public class FfiLogicalPlan extends LogicalPlan { +public class FfiPhysicalBuilder extends RegularPhysicalBuilder { private static final IrCoreLibrary LIB = IrCoreLibrary.INSTANCE; - + private final IrMeta irMeta; + private final Configs graphConfig; private final Pointer ptrPlan; - private int lastIdx; - public FfiLogicalPlan(RelOptCluster cluster, IrMeta irMeta, List hints) { - super(cluster, hints); - checkFfiResult(LIB.setSchema(irMeta.getSchemaJson())); + public FfiPhysicalBuilder(Configs graphConfig, IrMeta irMeta, LogicalPlan logicalPlan) { + super( + logicalPlan, + new GraphRelShuttleWrapper(new RelToFfiConverter(irMeta.getSchema().isColumnId()))); + this.graphConfig = graphConfig; + this.irMeta = irMeta; + checkFfiResult(LIB.setSchema(irMeta.getSchema().schemaJson())); this.ptrPlan = LIB.initLogicalPlan(); this.lastIdx = -1; + initialize(); } @Override - public void appendNode(LogicalNode node) { - if (isReturnEmpty()) { - throw new IllegalArgumentException( - "should not append any node to the logical pb if returnEmpty is set to true"); - } + public void appendNode(PhysicalNode node) { IntByReference oprIdx = new IntByReference(this.lastIdx); RelNode original = node.getOriginal(); if (original instanceof GraphLogicalSource) { @@ -126,7 +131,6 @@ public void appendNode(LogicalNode node) { @Override public String explain() { - if (isReturnEmpty()) return StringUtils.EMPTY; FfiResult res = LIB.printPlanAsJson(this.ptrPlan); if (res == null || res.code != ResultCode.Success) { throw new IllegalStateException("print plan in ir core fail, msg : %s" + res, null); @@ -135,11 +139,10 @@ public String explain() { } @Override - public byte[] toPhysical() { - if (isReturnEmpty()) return null; + public byte[] build() { appendSink(new IntByReference(this.lastIdx)); - int servers = Integer.valueOf(hints.get(0).kvOptions.get("servers")); - int workers = Integer.valueOf(hints.get(0).kvOptions.get("workers")); + int servers = PegasusConfig.PEGASUS_HOSTS.get(graphConfig).split(",").length; + int workers = PegasusConfig.PEGASUS_WORKER_NUM.get(graphConfig); FfiData.ByValue ffiData = LIB.buildPhysicalPlan(ptrPlan, workers, servers); checkFfiResult(ffiData.error); byte[] bytes = ffiData.getBytes(); @@ -162,10 +165,10 @@ private void checkFfiResult(FfiResult res) { } private boolean isColumnId() { - return Boolean.valueOf(hints.get(0).kvOptions.get("isColumnId")); + return this.irMeta.getSchema().isColumnId(); } - private void appendMatch(LogicalNode node, IntByReference oprIdx) { + private void appendMatch(PhysicalNode node, IntByReference oprIdx) { // append dummy source Pointer ptrScan = LIB.initScanOperator(Utils.ffiScanOpt(GraphOpt.Source.VERTEX)); checkFfiResult(LIB.appendScanOperator(ptrPlan, ptrScan, oprIdx.getValue(), oprIdx)); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java index badd4919798e..d5396fd3d3a1 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/ffi/RelToFfiConverter.java @@ -28,7 +28,7 @@ import com.alibaba.graphscope.common.ir.rel.type.order.GraphFieldCollation; import com.alibaba.graphscope.common.ir.rex.RexGraphVariable; import com.alibaba.graphscope.common.ir.runtime.proto.RexToProtoConverter; -import com.alibaba.graphscope.common.ir.runtime.type.LogicalNode; +import com.alibaba.graphscope.common.ir.runtime.type.PhysicalNode; import com.alibaba.graphscope.common.ir.tools.AliasInference; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.alibaba.graphscope.common.ir.type.GraphLabelType; @@ -85,7 +85,7 @@ public RelNode visit(GraphLogicalSource source) { if (source.getAliasId() != AliasInference.DEFAULT_ID) { checkFfiResult(LIB.setScanAlias(ptrScan, ArgUtils.asAlias(source.getAliasId()))); } - return new LogicalNode(source, ptrScan); + return new PhysicalNode(source, ptrScan); } @Override @@ -96,7 +96,7 @@ public RelNode visit(GraphLogicalExpand expand) { if (expand.getAliasId() != AliasInference.DEFAULT_ID) { checkFfiResult(LIB.setEdgexpdAlias(ptrExpand, ArgUtils.asAlias(expand.getAliasId()))); } - return new LogicalNode(expand, ptrExpand); + return new PhysicalNode(expand, ptrExpand); } @Override @@ -106,13 +106,13 @@ public RelNode visit(GraphLogicalGetV getV) { if (getV.getAliasId() != AliasInference.DEFAULT_ID) { checkFfiResult(LIB.setGetvAlias(ptrGetV, ArgUtils.asAlias(getV.getAliasId()))); } - return new LogicalNode(getV, ptrGetV); + return new PhysicalNode(getV, ptrGetV); } @Override public RelNode visit(GraphLogicalPathExpand pxd) { - LogicalNode expand = (LogicalNode) visit((GraphLogicalExpand) pxd.getExpand()); - LogicalNode getV = (LogicalNode) visit((GraphLogicalGetV) pxd.getGetV()); + PhysicalNode expand = (PhysicalNode) visit((GraphLogicalExpand) pxd.getExpand()); + PhysicalNode getV = (PhysicalNode) visit((GraphLogicalGetV) pxd.getGetV()); Pointer ptrPxd = LIB.initPathxpdOperatorWithExpandBase( (Pointer) expand.getNode(), @@ -124,7 +124,7 @@ public RelNode visit(GraphLogicalPathExpand pxd) { if (pxd.getAliasId() != AliasInference.DEFAULT_ID) { checkFfiResult(LIB.setPathxpdAlias(ptrPxd, ArgUtils.asAlias(pxd.getAliasId()))); } - return new LogicalNode(pxd, ptrPxd); + return new PhysicalNode(pxd, ptrPxd); } @Override @@ -135,7 +135,7 @@ public RelNode visit(GraphLogicalSingleMatch match) { Pointer ptrSentence = LIB.initPatternSentence(FfiJoinKind.Inner); addFfiBinder(ptrSentence, match.getSentence(), true); checkFfiResult(LIB.addPatternSentence(ptrPattern, ptrSentence)); - return new LogicalNode(match, ptrPattern); + return new PhysicalNode(match, ptrPattern); case OPTIONAL: case ANTI: default: @@ -151,7 +151,7 @@ public RelNode visit(GraphLogicalMultiMatch match) { addFfiBinder(ptrSentence, sentence, true); checkFfiResult(LIB.addPatternSentence(ptrPattern, ptrSentence)); } - return new LogicalNode(match, ptrPattern); + return new PhysicalNode(match, ptrPattern); } @Override @@ -162,11 +162,11 @@ public RelNode visit(LogicalFilter logicalFilter) { checkFfiResult( LIB.setSelectPredicatePb( ptrFilter, new FfiPbPointer.ByValue(exprProto.toByteArray()))); - return new LogicalNode(logicalFilter, ptrFilter); + return new PhysicalNode(logicalFilter, ptrFilter); } @Override - public LogicalNode visit(GraphLogicalProject project) { + public PhysicalNode visit(GraphLogicalProject project) { Pointer ptrProject = LIB.initProjectOperator(project.isAppend()); List fields = project.getRowType().getFieldList(); for (int i = 0; i < project.getProjects().size(); ++i) { @@ -183,14 +183,14 @@ public LogicalNode visit(GraphLogicalProject project) { new FfiPbPointer.ByValue(expression.toByteArray()), ffiAlias)); } - return new LogicalNode(project, ptrProject); + return new PhysicalNode(project, ptrProject); } @Override - public LogicalNode visit(GraphLogicalAggregate aggregate) { + public PhysicalNode visit(GraphLogicalAggregate aggregate) { List groupCalls = aggregate.getAggCalls(); if (groupCalls.isEmpty()) { // transform to project + dedup by keys - return new LogicalNode(aggregate, null); + return new PhysicalNode(aggregate, null); } Pointer ptrGroup = LIB.initGroupbyOperator(); List fields = aggregate.getRowType().getFieldList(); @@ -249,11 +249,11 @@ public LogicalNode visit(GraphLogicalAggregate aggregate) { ffiAggOpt, ffiAlias)); } - return new LogicalNode(aggregate, ptrGroup); + return new PhysicalNode(aggregate, ptrGroup); } @Override - public LogicalNode visit(GraphLogicalSort sort) { + public PhysicalNode visit(GraphLogicalSort sort) { List collations = sort.getCollation().getFieldCollations(); Pointer ptrNode = null; if (!collations.isEmpty()) { @@ -280,7 +280,7 @@ public LogicalNode visit(GraphLogicalSort sort) { checkFfiResult(LIB.setOrderbyLimit(ptrNode, limitRange.get(0), limitRange.get(1))); } } - return new LogicalNode(sort, ptrNode); + return new PhysicalNode(sort, ptrNode); } private Pointer ffiQueryParams(AbstractBindableTableScan tableScan) { @@ -381,12 +381,12 @@ private void addFfiBinder(Pointer ptrSentence, RelNode binder, boolean isTail) { addFfiBinder(ptrSentence, binder.getInput(0), false); if (binder instanceof AbstractBindableTableScan) { if (binder instanceof GraphLogicalExpand) { - LogicalNode node = (LogicalNode) visit((GraphLogicalExpand) binder); + PhysicalNode node = (PhysicalNode) visit((GraphLogicalExpand) binder); checkFfiResult( LIB.addSentenceBinder( ptrSentence, (Pointer) node.getNode(), FfiBinderOpt.Edge)); } else { // getV - LogicalNode node = (LogicalNode) visit((GraphLogicalGetV) binder); + PhysicalNode node = (PhysicalNode) visit((GraphLogicalGetV) binder); checkFfiResult( LIB.addSentenceBinder( ptrSentence, (Pointer) node.getNode(), FfiBinderOpt.Vertex)); @@ -396,7 +396,7 @@ private void addFfiBinder(Pointer ptrSentence, RelNode binder, boolean isTail) { checkFfiResult(LIB.setSentenceEnd(ptrSentence, ArgUtils.asNameOrId(aliasId))); } } else if (binder instanceof GraphLogicalPathExpand) { // path expand - LogicalNode node = (LogicalNode) visit((GraphLogicalPathExpand) binder); + PhysicalNode node = (PhysicalNode) visit((GraphLogicalPathExpand) binder); checkFfiResult( LIB.addSentenceBinder( ptrSentence, (Pointer) node.getNode(), FfiBinderOpt.Path)); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/type/LogicalPlan.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/type/LogicalPlan.java deleted file mode 100644 index c7ec1eca3c5c..000000000000 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/type/LogicalPlan.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.alibaba.graphscope.common.ir.runtime.type; - -import org.apache.calcite.plan.RelOptCluster; -import org.apache.calcite.plan.RelTraitSet; -import org.apache.calcite.rel.AbstractRelNode; -import org.apache.calcite.rel.hint.RelHint; - -import java.util.List; - -/** - * define interfaces to build a logical plan, {@link com.alibaba.graphscope.common.ir.runtime.ffi.FfiLogicalPlan} is one of implementations - * @param - * @param - */ -public abstract class LogicalPlan extends AbstractRelNode implements AutoCloseable { - protected final List hints; - protected boolean returnEmpty; - - protected LogicalPlan(RelOptCluster cluster, List hints) { - super(cluster, RelTraitSet.createEmpty()); - this.hints = hints; - } - - /** - * append {@code LogicalNode} to the plan - * @param node - */ - public abstract void appendNode(LogicalNode node); - - /** - * output logical plan - */ - public abstract String explain(); - - /** - * convert logical plan to physical plan - * @return - */ - public abstract R toPhysical(); - - public void setReturnEmpty(boolean returnEmpty) { - this.returnEmpty = returnEmpty; - } - - public boolean isReturnEmpty() { - return returnEmpty; - } -} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/type/LogicalNode.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/type/PhysicalNode.java similarity index 78% rename from interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/type/LogicalNode.java rename to interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/type/PhysicalNode.java index 6decacd72a41..17a3856dca9a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/type/LogicalNode.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/type/PhysicalNode.java @@ -21,21 +21,23 @@ import org.apache.calcite.rel.AbstractRelNode; import org.apache.calcite.rel.RelNode; +import java.util.Objects; + /** * a wrapper of node (i.e. {@link com.sun.jna.Pointer} for FFI interfaces) in ir_core * @param */ -public class LogicalNode extends AbstractRelNode { +public class PhysicalNode extends AbstractRelNode { private final RelNode original; private final T node; - protected LogicalNode(RelOptCluster cluster, RelTraitSet traitSet, RelNode original, T node) { + protected PhysicalNode(RelOptCluster cluster, RelTraitSet traitSet, RelNode original, T node) { super(cluster, traitSet); - this.original = original; - this.node = node; + this.original = Objects.requireNonNull(original); + this.node = Objects.requireNonNull(node); } - public LogicalNode(RelNode original, T node) { + public PhysicalNode(RelNode original, T node) { this(original.getCluster(), RelTraitSet.createEmpty(), original, node); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/schema/GraphSchemaWrapper.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/schema/GraphSchemaWrapper.java index 4884ea0e4e60..0ab1d44288a3 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/schema/GraphSchemaWrapper.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/schema/GraphSchemaWrapper.java @@ -37,10 +37,12 @@ */ public class GraphSchemaWrapper implements StatisticSchema { private final GraphSchema graphSchema; + private final String schemeJson; private final boolean isColumnId; - public GraphSchemaWrapper(GraphSchema graphSchema, boolean isColumnId) { + public GraphSchemaWrapper(GraphSchema graphSchema, String schemaJson, boolean isColumnId) { this.graphSchema = graphSchema; + this.schemeJson = schemaJson; this.isColumnId = isColumnId; } @@ -60,6 +62,11 @@ public boolean isColumnId() { return this.isColumnId; } + @Override + public String schemaJson() { + return this.schemeJson; + } + @Override public GraphElement getElement(String s) throws GraphElementNotFoundException { return this.graphSchema.getElement(s); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/schema/StatisticSchema.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/schema/StatisticSchema.java index 1fb1dcab3103..34e59d4c5b7c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/schema/StatisticSchema.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/schema/StatisticSchema.java @@ -31,4 +31,7 @@ public interface StatisticSchema extends GraphSchema { // if the property name need to be converted to id boolean isColumnId(); + + // schema json for ir core + String schemaJson(); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java new file mode 100644 index 000000000000..3c35bc6eea9e --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/tools/GraphPlanner.java @@ -0,0 +1,203 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.ir.tools; + +import com.alibaba.graphscope.common.antlr4.Antlr4Parser; +import com.alibaba.graphscope.common.config.Configs; +import com.alibaba.graphscope.common.config.FileLoadType; +import com.alibaba.graphscope.common.config.PlannerConfig; +import com.alibaba.graphscope.common.ir.planner.rules.FilterMatchRule; +import com.alibaba.graphscope.common.ir.runtime.PhysicalBuilder; +import com.alibaba.graphscope.common.ir.runtime.ffi.FfiPhysicalBuilder; +import com.alibaba.graphscope.common.ir.schema.GraphOptSchema; +import com.alibaba.graphscope.common.ir.schema.StatisticSchema; +import com.alibaba.graphscope.common.store.ExperimentalMetaFetcher; +import com.alibaba.graphscope.common.store.IrMeta; +import com.alibaba.graphscope.cypher.antlr4.parser.CypherAntlr4Parser; +import com.alibaba.graphscope.cypher.antlr4.visitor.LogicalPlanVisitor; + +import org.antlr.v4.runtime.tree.ParseTree; +import org.apache.calcite.jdbc.JavaTypeFactoryImpl; +import org.apache.calcite.plan.GraphOptCluster; +import org.apache.calcite.plan.RelOptPlanner; +import org.apache.calcite.plan.hep.HepPlanner; +import org.apache.calcite.plan.hep.HepProgram; +import org.apache.calcite.plan.hep.HepProgramBuilder; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rex.RexBuilder; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.NotImplementedException; +import org.checkerframework.checker.nullness.qual.Nullable; + +import java.io.File; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; + +/** + * A unified structure to build {@link PlannerInstance} which can further build logical and physical plan from an antlr tree + */ +public class GraphPlanner { + private final Configs graphConfig; + private final PlannerConfig plannerConfig; + private final RelOptPlanner optPlanner; + private final RexBuilder rexBuilder; + private final AtomicLong idGenerator; + + public GraphPlanner(Configs graphConfig) { + this.graphConfig = graphConfig; + this.plannerConfig = PlannerConfig.create(this.graphConfig); + this.optPlanner = createRelOptPlanner(this.plannerConfig); + this.rexBuilder = new RexBuilder(new JavaTypeFactoryImpl()); + this.idGenerator = new AtomicLong(0l); + } + + public PlannerInstance instance(ParseTree parsedQuery, IrMeta irMeta) { + long id = idGenerator.getAndIncrement(); + String name = "ir_plan_" + id; + GraphOptCluster optCluster = GraphOptCluster.create(this.optPlanner, this.rexBuilder); + return new PlannerInstance(id, name, parsedQuery, optCluster, irMeta); + } + + public class PlannerInstance { + private final long id; + private final String name; + private final ParseTree parsedQuery; + private final GraphOptCluster optCluster; + private final IrMeta irMeta; + + public PlannerInstance( + long id, + String name, + ParseTree parsedQuery, + GraphOptCluster optCluster, + IrMeta irMeta) { + this.id = id; + this.name = name; + this.parsedQuery = parsedQuery; + this.optCluster = optCluster; + this.irMeta = irMeta; + } + + public Summary plan() { + // build logical plan from parsed query + StatisticSchema schema = irMeta.getSchema(); + GraphBuilder graphBuilder = + GraphBuilder.create( + null, this.optCluster, new GraphOptSchema(this.optCluster, schema)); + LogicalPlan logicalPlan = + new LogicalPlanVisitor(graphBuilder, this.irMeta).visit(this.parsedQuery); + // apply optimizations + if (plannerConfig.isOn() + && logicalPlan.getRegularQuery() != null + && !logicalPlan.isReturnEmpty()) { + RelNode regularQuery = logicalPlan.getRegularQuery(); + RelOptPlanner planner = this.optCluster.getPlanner(); + planner.setRoot(regularQuery); + logicalPlan = new LogicalPlan(planner.findBestExp(), logicalPlan.isReturnEmpty()); + } + // build physical plan from logical plan + PhysicalBuilder physicalBuilder; + if (logicalPlan.isReturnEmpty()) { + physicalBuilder = PhysicalBuilder.createEmpty(logicalPlan); + } else if (logicalPlan.getRegularQuery() != null) { + physicalBuilder = new FfiPhysicalBuilder(graphConfig, irMeta, logicalPlan); + } else { + throw new NotImplementedException("procedure call is unimplemented yet"); + } + return new Summary(this.id, this.name, logicalPlan, physicalBuilder); + } + } + + public static class Summary { + private final long id; + private final String name; + private final LogicalPlan logicalPlan; + private final PhysicalBuilder physicalBuilder; + + public Summary( + long id, String name, LogicalPlan logicalPlan, PhysicalBuilder physicalBuilder) { + this.id = id; + this.name = name; + this.logicalPlan = Objects.requireNonNull(logicalPlan); + this.physicalBuilder = Objects.requireNonNull(physicalBuilder); + } + + public long getId() { + return id; + } + + public String getName() { + return name; + } + + public LogicalPlan getLogicalPlan() { + return logicalPlan; + } + + public @Nullable PhysicalBuilder getPhysicalBuilder() { + return physicalBuilder; + } + } + + private RelOptPlanner createRelOptPlanner(PlannerConfig plannerConfig) { + if (plannerConfig.isOn()) { + PlannerConfig.Opt opt = plannerConfig.getOpt(); + switch (opt) { + case RBO: + HepProgramBuilder hepBuilder = HepProgram.builder(); + plannerConfig + .getRules() + .forEach( + k -> { + if (k.equals(FilterMatchRule.class.getSimpleName())) { + hepBuilder.addRuleInstance( + FilterMatchRule.Config.DEFAULT.toRule()); + } else { + // todo: add more rules + } + }); + return new HepPlanner(hepBuilder.build()); + case CBO: + default: + throw new UnsupportedOperationException( + "planner type " + opt.name() + " is unsupported yet"); + } + } else { + // return HepPlanner with empty rules if optimization is turned off + return new HepPlanner(HepProgram.builder().build()); + } + } + + public static void main(String[] args) throws Exception { + Configs configs = new Configs("conf/ir.compiler.properties", FileLoadType.RELATIVE_PATH); + ExperimentalMetaFetcher metaFetcher = new ExperimentalMetaFetcher(configs); + if (args.length < 2 || args[0].isEmpty() || args[1].isEmpty()) { + throw new IllegalArgumentException( + "usage: make physical_plan query='' physical=''"); + } + String query = args[0]; + GraphPlanner planner = new GraphPlanner(configs); + Antlr4Parser cypherParser = new CypherAntlr4Parser(); + PlannerInstance instance = + planner.instance(cypherParser.parse(query), metaFetcher.fetch().get()); + Summary summary = instance.plan(); + try (PhysicalBuilder physicalBuilder = summary.getPhysicalBuilder()) { + FileUtils.writeByteArrayToFile(new File(args[1]), physicalBuilder.build()); + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/ExperimentalMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/ExperimentalMetaFetcher.java index c25f45d7bc80..2a0b3da53c1b 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/ExperimentalMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/ExperimentalMetaFetcher.java @@ -35,8 +35,8 @@ public ExperimentalMetaFetcher(Configs configs) throws IOException { new GraphSchemaWrapper( com.alibaba.graphscope.common.ir.schema.Utils.buildSchemaFromJson( schemaJson), - false), - schemaJson); + schemaJson, + false)); } @Override diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMeta.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMeta.java index 3ef0d504d89a..4409b6fd04bb 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMeta.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/IrMeta.java @@ -21,37 +21,23 @@ import java.util.Objects; public class IrMeta { + private final SnapshotId snapshotId; private StatisticSchema schema; - private String schemaJson; - private long snapshotId; - private boolean acquireSnapshot; - - public IrMeta(StatisticSchema schema, String schemaJson) { - this.schema = Objects.requireNonNull(schema); - this.schemaJson = Objects.requireNonNull(schemaJson); + public IrMeta(StatisticSchema schema) { + this(SnapshotId.createEmpty(), schema); } - public IrMeta(StatisticSchema schema, String schemaJson, long snapshotId) { - this.schema = schema; - this.schemaJson = schemaJson; - this.snapshotId = snapshotId; - this.acquireSnapshot = true; + public IrMeta(SnapshotId snapshotId, StatisticSchema schema) { + this.snapshotId = Objects.requireNonNull(snapshotId); + this.schema = Objects.requireNonNull(schema); } public StatisticSchema getSchema() { return schema; } - public String getSchemaJson() { - return schemaJson; - } - - public long getSnapshotId() { + public SnapshotId getSnapshotId() { return snapshotId; } - - public boolean isAcquireSnapshot() { - return acquireSnapshot; - } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/SnapshotId.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/SnapshotId.java new file mode 100644 index 000000000000..095143d4f942 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/store/SnapshotId.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.store; + +public class SnapshotId { + public static final SnapshotId createEmpty() { + return new SnapshotId(false, -1); + } + + private final boolean acquired; + private final long id; + + public SnapshotId(boolean acquired, long id) { + this.acquired = acquired; + this.id = id; + } + + public boolean isAcquired() { + return acquired; + } + + public long getId() { + return id; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/parser/CypherAntlr4Parser.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/parser/CypherAntlr4Parser.java new file mode 100644 index 000000000000..44dd8aa3c824 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/parser/CypherAntlr4Parser.java @@ -0,0 +1,49 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.cypher.antlr4.parser; + +import com.alibaba.graphscope.common.antlr4.Antlr4Parser; +import com.alibaba.graphscope.common.antlr4.SyntaxErrorListener; +import com.alibaba.graphscope.grammar.CypherGSLexer; +import com.alibaba.graphscope.grammar.CypherGSParser; + +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.tree.ParseTree; + +/** + * parse cypher DSL to antlr tree + */ +public class CypherAntlr4Parser implements Antlr4Parser { + @Override + public ParseTree parse(String statement) { + CypherGSLexer lexer = new CypherGSLexer(CharStreams.fromString(statement)); + // reset error listeners on lexer + lexer.removeErrorListeners(); + lexer.addErrorListener(new SyntaxErrorListener()); + final CypherGSParser parser = new CypherGSParser(new CommonTokenStream(lexer)); + // setup error handler on parser + parser.setErrorHandler(new DefaultErrorStrategy()); + // reset error listeners on parser + parser.removeErrorListeners(); + parser.addErrorListener(new SyntaxErrorListener()); + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + return parser.oC_Cypher(); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ExpressionVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ExpressionVisitor.java index 6aaa35732a2c..6bd05ca81d72 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ExpressionVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/ExpressionVisitor.java @@ -20,7 +20,7 @@ import com.alibaba.graphscope.common.ir.rex.RexTmpVariable; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; -import com.alibaba.graphscope.cypher.antlr4.type.ExprVisitorResult; +import com.alibaba.graphscope.cypher.antlr4.visitor.type.ExprVisitorResult; import com.alibaba.graphscope.grammar.CypherGSBaseVisitor; import com.alibaba.graphscope.grammar.CypherGSParser; import com.google.common.collect.ImmutableList; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java index ff9d675b03c7..3f7da8cc75d4 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java @@ -19,8 +19,8 @@ import com.alibaba.graphscope.common.ir.rel.type.group.GraphAggCall; import com.alibaba.graphscope.common.ir.rex.RexTmpVariableConverter; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; -import com.alibaba.graphscope.common.ir.tools.config.*; -import com.alibaba.graphscope.cypher.antlr4.type.ExprVisitorResult; +import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; +import com.alibaba.graphscope.cypher.antlr4.visitor.type.ExprVisitorResult; import com.alibaba.graphscope.grammar.CypherGSBaseVisitor; import com.alibaba.graphscope.grammar.CypherGSParser; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/LogicalPlanVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/LogicalPlanVisitor.java new file mode 100644 index 000000000000..e99de3dbefe8 --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/LogicalPlanVisitor.java @@ -0,0 +1,70 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.cypher.antlr4.visitor; + +import com.alibaba.graphscope.common.ir.tools.GraphBuilder; +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; +import com.alibaba.graphscope.common.store.IrMeta; +import com.alibaba.graphscope.grammar.CypherGSBaseVisitor; +import com.alibaba.graphscope.grammar.CypherGSParser; +import com.google.common.collect.Lists; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.logical.LogicalValues; +import org.apache.commons.lang3.NotImplementedException; + +import java.util.List; + +public class LogicalPlanVisitor extends CypherGSBaseVisitor { + private final GraphBuilder builder; + private final IrMeta irMeta; + + public LogicalPlanVisitor(GraphBuilder builder, IrMeta irMeta) { + this.builder = builder; + this.irMeta = irMeta; + } + + @Override + public LogicalPlan visitOC_Cypher(CypherGSParser.OC_CypherContext ctx) { + return visitOC_Statement(ctx.oC_Statement()); + } + + @Override + public LogicalPlan visitOC_Query(CypherGSParser.OC_QueryContext ctx) { + if (ctx.oC_RegularQuery() != null) { + RelNode regularQuery = + new GraphBuilderVisitor(this.builder) + .visitOC_RegularQuery(ctx.oC_RegularQuery()) + .build(); + return new LogicalPlan(regularQuery, returnEmpty(regularQuery)); + } else { + throw new NotImplementedException("procedure call is unimplemented yet"); + } + } + + private boolean returnEmpty(RelNode relNode) { + List inputs = Lists.newArrayList(relNode); + while (!inputs.isEmpty()) { + RelNode cur = inputs.remove(0); + if (cur instanceof LogicalValues) { + return true; + } + inputs.addAll(cur.getInputs()); + } + return false; + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/type/ExprVisitorResult.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/type/ExprVisitorResult.java similarity index 97% rename from interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/type/ExprVisitorResult.java rename to interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/type/ExprVisitorResult.java index 18dd477d4d54..cc09bae281a7 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/type/ExprVisitorResult.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/type/ExprVisitorResult.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package com.alibaba.graphscope.cypher.antlr4.type; +package com.alibaba.graphscope.cypher.antlr4.visitor.type; import com.google.common.collect.ImmutableList; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java index f108d242a7aa..22c0ec79975d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/integration/processor/IrTestOpProcessor.java @@ -20,7 +20,6 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; import com.alibaba.graphscope.common.store.IrMeta; -import com.alibaba.graphscope.common.store.IrMetaFetcher; import com.alibaba.graphscope.gremlin.integration.result.GraphProperties; import com.alibaba.graphscope.gremlin.integration.result.GremlinTestResultProcessor; import com.alibaba.graphscope.gremlin.plugin.processor.IrStandardOpProcessor; @@ -57,13 +56,12 @@ public class IrTestOpProcessor extends IrStandardOpProcessor { public IrTestOpProcessor( Configs configs, - IrMetaFetcher irMetaFetcher, ChannelFetcher fetcher, IrMetaQueryCallback metaQueryCallback, Graph graph, GraphTraversalSource g, GraphProperties testGraph) { - super(configs, irMetaFetcher, fetcher, metaQueryCallback, graph, g); + super(configs, fetcher, metaQueryCallback, graph, g); this.context = new SimpleScriptContext(); Bindings globalBindings = new SimpleBindings(); globalBindings.put("g", g); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java index 58cb8d66e5bc..d21a23f92231 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/IrStandardOpProcessor.java @@ -29,20 +29,12 @@ import com.alibaba.graphscope.common.client.channel.ChannelFetcher; import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.PegasusConfig; -import com.alibaba.graphscope.common.config.PlannerConfig; import com.alibaba.graphscope.common.intermediate.InterOpCollection; -import com.alibaba.graphscope.common.ir.planner.rules.FilterMatchRule; -import com.alibaba.graphscope.common.ir.rel.GraphRelShuttleWrapper; -import com.alibaba.graphscope.common.ir.runtime.LogicalPlanConverter; -import com.alibaba.graphscope.common.ir.runtime.ffi.FfiLogicalPlan; -import com.alibaba.graphscope.common.ir.runtime.ffi.RelToFfiConverter; -import com.alibaba.graphscope.common.ir.runtime.type.LogicalPlan; -import com.alibaba.graphscope.common.ir.schema.GraphOptSchema; -import com.alibaba.graphscope.common.ir.schema.StatisticSchema; -import com.alibaba.graphscope.common.ir.tools.GraphBuilder; +import com.alibaba.graphscope.common.ir.runtime.PhysicalBuilder; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; import com.alibaba.graphscope.common.store.IrMeta; -import com.alibaba.graphscope.common.store.IrMetaFetcher; import com.alibaba.graphscope.gremlin.InterOpCollectionBuilder; import com.alibaba.graphscope.gremlin.Utils; import com.alibaba.graphscope.gremlin.plugin.script.AntlrCypherScriptEngineFactory; @@ -57,17 +49,9 @@ import com.alibaba.pegasus.service.protocol.PegasusClient; import com.google.common.collect.ImmutableList; import com.google.protobuf.InvalidProtocolBufferException; -import com.sun.jna.Pointer; -import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.plan.GraphOptCluster; -import org.apache.calcite.plan.RelOptPlanner; -import org.apache.calcite.plan.hep.HepPlanner; -import org.apache.calcite.plan.hep.HepProgram; -import org.apache.calcite.plan.hep.HepProgramBuilder; -import org.apache.calcite.rel.RelNode; +import org.antlr.v4.runtime.tree.ParseTree; import org.apache.calcite.rel.hint.RelHint; -import org.apache.calcite.rex.RexBuilder; import org.apache.tinkerpop.gremlin.driver.message.RequestMessage; import org.apache.tinkerpop.gremlin.driver.message.ResponseMessage; import org.apache.tinkerpop.gremlin.driver.message.ResponseStatusCode; @@ -90,12 +74,14 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.*; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicLong; -import java.util.function.Function; import java.util.function.Supplier; import javax.script.SimpleBindings; @@ -108,20 +94,16 @@ public class IrStandardOpProcessor extends StandardOpProcessor { protected Graph graph; protected GraphTraversalSource g; protected Configs configs; - protected PlannerConfig plannerConfig; /** * todo: replace with {@link com.alibaba.graphscope.common.client.ExecutionClient} after unifying Gremlin into the Calcite stack */ protected RpcClient rpcClient; - protected IrMetaFetcher irMetaFetcher; protected IrMetaQueryCallback metaQueryCallback; - - protected final Function graphBuilderGenerator; + protected final GraphPlanner graphPlanner; public IrStandardOpProcessor( Configs configs, - IrMetaFetcher irMetaFetcher, ChannelFetcher fetcher, IrMetaQueryCallback metaQueryCallback, Graph graph, @@ -129,21 +111,10 @@ public IrStandardOpProcessor( this.graph = graph; this.g = g; this.configs = configs; - this.plannerConfig = PlannerConfig.create(this.configs); - this.irMetaFetcher = irMetaFetcher; this.rpcClient = new RpcClient(PegasusConfig.PEGASUS_GRPC_TIMEOUT.get(configs), fetcher.fetch()); this.metaQueryCallback = metaQueryCallback; - - RexBuilder rexBuilder = new RexBuilder(new JavaTypeFactoryImpl()); - this.graphBuilderGenerator = - (StatisticSchema schema) -> { - Objects.requireNonNull(schema); - GraphOptCluster optCluster = - GraphOptCluster.create(getRelOptPlanner(), rexBuilder); - return GraphBuilder.create( - null, optCluster, new GraphOptSchema(optCluster, schema)); - }; + this.graphPlanner = new GraphPlanner(configs); } @Override @@ -334,15 +305,6 @@ protected GremlinExecutor.LifeCycle createLifeCycle( b.putAll(bindingsSupplier.get()); b.put("graph", graph); b.put("g", g); - if (language.equals( - AntlrGremlinScriptEngineFactory.LANGUAGE_NAME)) { - // todo: prepare configs to parse gremlin - } else if (language.equals( - AntlrCypherScriptEngineFactory.LANGUAGE_NAME)) { - b.put( - "graph.builder", - graphBuilderGenerator.apply(irMeta.getSchema())); - } } catch (OpProcessorException ope) { throw new RuntimeException(ope); } @@ -365,26 +327,18 @@ protected GremlinExecutor.LifeCycle createLifeCycle( jobId, script, irMeta); - } else if (o != null && o instanceof GraphBuilder) { - GraphBuilder builder = (GraphBuilder) o; - GraphOptCluster optCluster = - (GraphOptCluster) builder.getCluster(); - RelNode topNode = builder.build(); - // apply optimizations - if (this.plannerConfig.isOn()) { - RelOptPlanner planner = optCluster.getPlanner(); - planner.setRoot(topNode); - topNode = planner.findBestExp(); - } + } else if (o != null && o instanceof ParseTree) { + GraphPlanner.PlannerInstance instance = + graphPlanner.instance((ParseTree) o, irMeta); + GraphPlanner.Summary summary = instance.plan(); if (language.equals( AntlrGremlinScriptEngineFactory.LANGUAGE_NAME)) { // todo: handle gremlin results } else if (language.equals( AntlrCypherScriptEngineFactory.LANGUAGE_NAME)) { processRelNode( - topNode, - optCluster, - new CypherResultProcessor(ctx, topNode), + summary, + new CypherResultProcessor(ctx, summary), jobId, script, irMeta, @@ -440,39 +394,34 @@ protected void processTraversal( } protected void processRelNode( - RelNode topNode, - GraphOptCluster optCluster, + GraphPlanner.Summary summary, ResultProcessor resultProcessor, long jobId, String script, IrMeta irMeta, Context ctx) throws Exception { - try (LogicalPlan logicalPlan = - new LogicalPlanConverter<>( - new GraphRelShuttleWrapper( - new RelToFfiConverter(irMeta.getSchema().isColumnId())), - new FfiLogicalPlan(optCluster, irMeta, getPlanHints(irMeta))) - .go(topNode)) { - String jobName = "ir_plan_" + jobId; - if (logicalPlan.isReturnEmpty()) { - logger.info( - "gremlin query \"{}\", job conf name \"{}\", relNode plan\n {}", - script, - jobName, - topNode.explain()); - // return empty results to the client - RequestMessage msg = ctx.getRequestMessage(); - ctx.writeAndFlush( - ResponseMessage.build(msg).code(ResponseStatusCode.NO_CONTENT).create()); - } else { - byte[] physicalPlanBytes = logicalPlan.toPhysical(); + String jobName = "ir_plan_" + jobId; + LogicalPlan logicalPlan = summary.getLogicalPlan(); + if (logicalPlan.isReturnEmpty()) { + logger.info( + "gremlin query \"{}\", job conf name \"{}\", logical plan\n {}", + script, + jobName, + logicalPlan.explain()); + // return empty results to the client + RequestMessage msg = ctx.getRequestMessage(); + ctx.writeAndFlush( + ResponseMessage.build(msg).code(ResponseStatusCode.NO_CONTENT).create()); + } else { + try (PhysicalBuilder physicalBuilder = summary.getPhysicalBuilder()) { + byte[] physicalPlanBytes = physicalBuilder.build(); // print script and jobName with ir plan logger.info( - "gremlin query \"{}\", job conf name \"{}\", ir plan {}", + "gremlin query \"{}\", job conf name \"{}\", ir core plan {}", script, jobName, - logicalPlan.explain()); + physicalBuilder.explain()); PegasusClient.JobRequest request = PegasusClient.JobRequest.parseFrom(physicalPlanBytes); PegasusClient.JobConfig jobConfig = @@ -520,35 +469,6 @@ public static void applyStrategies(Traversal traversal) { traversal.asAdmin().applyStrategies(); } - private RelOptPlanner getRelOptPlanner() { - if (this.plannerConfig.isOn()) { - PlannerConfig.Opt opt = this.plannerConfig.getOpt(); - switch (opt) { - case RBO: - HepProgramBuilder hepBuilder = HepProgram.builder(); - this.plannerConfig - .getRules() - .forEach( - k -> { - if (k.equals(FilterMatchRule.class.getSimpleName())) { - hepBuilder.addRuleInstance( - FilterMatchRule.Config.DEFAULT.toRule()); - } else { - // todo: add more rules - } - }); - return new HepPlanner(hepBuilder.build()); - case CBO: - default: - throw new UnsupportedOperationException( - "planner type " + opt.name() + " is unsupported yet"); - } - } else { - // return HepPlanner with empty rules if optimization is turned off - return new HepPlanner(HepProgram.builder().build()); - } - } - @Override public void close() throws Exception { if (this.rpcClient != null) { diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrCypherScriptEngine.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrCypherScriptEngine.java index 0b161b0b92b1..e356bffaf38c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrCypherScriptEngine.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/script/AntlrCypherScriptEngine.java @@ -16,14 +16,8 @@ package com.alibaba.graphscope.gremlin.plugin.script; -import com.alibaba.graphscope.common.antlr4.SyntaxErrorListener; -import com.alibaba.graphscope.common.ir.tools.GraphBuilder; -import com.alibaba.graphscope.cypher.antlr4.visitor.GraphBuilderVisitor; -import com.alibaba.graphscope.grammar.CypherGSLexer; -import com.alibaba.graphscope.grammar.CypherGSParser; +import com.alibaba.graphscope.cypher.antlr4.parser.CypherAntlr4Parser; -import org.antlr.v4.runtime.*; -import org.antlr.v4.runtime.atn.PredictionMode; import org.apache.commons.lang3.NotImplementedException; import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine; import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngineFactory; @@ -33,7 +27,6 @@ import org.slf4j.LoggerFactory; import java.io.Reader; -import java.util.Objects; import javax.script.AbstractScriptEngine; import javax.script.Bindings; @@ -43,27 +36,16 @@ public class AntlrCypherScriptEngine extends AbstractScriptEngine implements GremlinScriptEngine { private static final Logger logger = LoggerFactory.getLogger(AntlrCypherScriptEngine.class); private volatile AntlrCypherScriptEngineFactory factory; + private final CypherAntlr4Parser cypherParser; + + public AntlrCypherScriptEngine() { + this.cypherParser = new CypherAntlr4Parser(); + } @Override public Object eval(String script, ScriptContext ctx) { logger.debug("antlr-cypher start to eval \"{}\"", script); - Bindings globalBindings = ctx.getBindings(ScriptContext.ENGINE_SCOPE); - GraphBuilder graphBuilder = - Objects.requireNonNull((GraphBuilder) globalBindings.get("graph.builder")); - GraphBuilderVisitor visitor = new GraphBuilderVisitor(graphBuilder); - - CypherGSLexer lexer = new CypherGSLexer(CharStreams.fromString(script)); - // reset error listeners on lexer - lexer.removeErrorListeners(); - lexer.addErrorListener(new SyntaxErrorListener()); - final CypherGSParser parser = new CypherGSParser(new CommonTokenStream(lexer)); - // setup error handler on parser - parser.setErrorHandler(new DefaultErrorStrategy()); - // reset error listeners on parser - parser.removeErrorListeners(); - parser.addErrorListener(new SyntaxErrorListener()); - parser.getInterpreter().setPredictionMode(PredictionMode.LL); - return visitor.visit(parser.oC_Cypher()); + return this.cypherParser.parse(script); } @Override diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/processor/CypherResultProcessor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/processor/CypherResultProcessor.java index 613102b637a8..c9bffdd43cc2 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/processor/CypherResultProcessor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/result/processor/CypherResultProcessor.java @@ -17,6 +17,8 @@ package com.alibaba.graphscope.gremlin.result.processor; import com.alibaba.graphscope.common.ir.tools.AliasInference; +import com.alibaba.graphscope.common.ir.tools.GraphPlanner; +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; import com.alibaba.graphscope.gremlin.result.parser.CypherResultParser; import com.google.common.collect.Lists; @@ -29,11 +31,12 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; public class CypherResultProcessor extends AbstractResultProcessor { - public CypherResultProcessor(Context context, RelNode topNode) { - super(context, new CypherResultParser(getOutputDataType(topNode))); + public CypherResultProcessor(Context context, GraphPlanner.Summary summary) { + super(context, new CypherResultParser(getOutputType(summary.getLogicalPlan()))); } @Override @@ -41,17 +44,23 @@ protected void aggregateResults() { // do nothing } - private static RelDataType getOutputDataType(RelNode topNode) { - List inputQueue = Lists.newArrayList(topNode); - List outputFields = new ArrayList<>(); - while (!inputQueue.isEmpty()) { - RelNode cur = inputQueue.remove(0); - outputFields.addAll(cur.getRowType().getFieldList()); - if (AliasInference.removeAlias(cur)) { - break; + private static RelDataType getOutputType(LogicalPlan logicalPlan) { + if (logicalPlan.getRegularQuery() != null) { + List inputs = Lists.newArrayList(logicalPlan.getRegularQuery()); + List outputFields = new ArrayList<>(); + while (!inputs.isEmpty()) { + RelNode cur = inputs.remove(0); + outputFields.addAll(cur.getRowType().getFieldList()); + if (AliasInference.removeAlias(cur)) { + break; + } + inputs.addAll(cur.getInputs()); } - inputQueue.addAll(cur.getInputs()); + return new RelRecordType( + StructKind.FULLY_QUALIFIED, + outputFields.stream().distinct().collect(Collectors.toList())); + } else { + return logicalPlan.getProcedureCall().getType(); } - return new RelRecordType(StructKind.FULLY_QUALIFIED, outputFields); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/GraphServiceMain.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/GraphServiceMain.java index 0aa6c49f7529..b80fb9539bfb 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/GraphServiceMain.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/GraphServiceMain.java @@ -18,7 +18,6 @@ import com.alibaba.graphscope.common.config.GraphConfig; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; import com.alibaba.graphscope.common.store.ExperimentalMetaFetcher; -import com.alibaba.graphscope.common.store.IrMetaFetcher; import com.alibaba.graphscope.gremlin.integration.result.TestGraphFactory; public class GraphServiceMain { @@ -27,25 +26,15 @@ public class GraphServiceMain { public static void main(String[] args) throws Exception { Configs configs = new Configs("conf/ir.compiler.properties", FileLoadType.RELATIVE_PATH); - IrMetaFetcher irMetaFetcher = new ExperimentalMetaFetcher(configs); + IrMetaQueryCallback metaQueryCallback = + new IrMetaQueryCallback(new ExperimentalMetaFetcher(configs)); ChannelFetcher fetcher = new HostsRpcChannelFetcher(configs); - IrGremlinServer server = new IrGremlinServer(); String storeType = GraphConfig.GRAPH_STORE.get(configs); if (storeType.equals(EXPERIMENTAL)) { - server.start( - configs, - irMetaFetcher, - fetcher, - new IrMetaQueryCallback(irMetaFetcher), - TestGraphFactory.EXPERIMENTAL); + server.start(configs, fetcher, metaQueryCallback, TestGraphFactory.EXPERIMENTAL); } else if (storeType.equals(CSR)) { - server.start( - configs, - irMetaFetcher, - fetcher, - new IrMetaQueryCallback(irMetaFetcher), - TestGraphFactory.MCSR); + server.start(configs, fetcher, metaQueryCallback, TestGraphFactory.MCSR); } else { throw new UnsupportedOperationException( "store type " + storeType + " is unsupported yet"); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrGremlinServer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrGremlinServer.java index 656f4e1440e4..de118be05bb3 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrGremlinServer.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/service/IrGremlinServer.java @@ -19,7 +19,6 @@ import com.alibaba.graphscope.common.client.channel.ChannelFetcher; import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; -import com.alibaba.graphscope.common.store.IrMetaFetcher; import com.alibaba.graphscope.gremlin.Utils; import com.alibaba.graphscope.gremlin.auth.AuthManager; import com.alibaba.graphscope.gremlin.auth.AuthManagerReference; @@ -68,18 +67,15 @@ public IrGremlinServer(int gremlinPort) { public void start( Configs configs, - IrMetaFetcher irMetaFetcher, ChannelFetcher fetcher, IrMetaQueryCallback metaQueryCallback, GraphProperties testGraph) throws Exception { AbstractOpProcessor standardProcessor = - new IrStandardOpProcessor( - configs, irMetaFetcher, fetcher, metaQueryCallback, graph, g); + new IrStandardOpProcessor(configs, fetcher, metaQueryCallback, graph, g); IrOpLoader.addProcessor(standardProcessor.getName(), standardProcessor); AbstractOpProcessor testProcessor = - new IrTestOpProcessor( - configs, irMetaFetcher, fetcher, metaQueryCallback, graph, g, testGraph); + new IrTestOpProcessor(configs, fetcher, metaQueryCallback, graph, g, testGraph); IrOpLoader.addProcessor(testProcessor.getName(), testProcessor); AuthManager authManager = new DefaultAuthManager(configs); diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java index 6fce99ec723d..50be56a1aa23 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/runtime/FfiLogicalPlanTest.java @@ -16,26 +16,20 @@ package com.alibaba.graphscope.common.ir.runtime; +import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.ir.Utils; -import com.alibaba.graphscope.common.ir.rel.GraphRelShuttleWrapper; -import com.alibaba.graphscope.common.ir.runtime.ffi.FfiLogicalPlan; -import com.alibaba.graphscope.common.ir.runtime.ffi.RelToFfiConverter; -import com.alibaba.graphscope.common.ir.runtime.type.LogicalPlan; +import com.alibaba.graphscope.common.ir.runtime.ffi.FfiPhysicalBuilder; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.GraphStdOperatorTable; +import com.alibaba.graphscope.common.ir.tools.LogicalPlan; import com.alibaba.graphscope.common.ir.tools.config.*; -import com.alibaba.graphscope.common.jna.type.FfiData; import com.alibaba.graphscope.common.utils.FileUtils; -import com.google.common.collect.ImmutableList; -import com.sun.jna.Pointer; +import com.google.common.collect.ImmutableMap; import org.apache.calcite.rel.RelNode; -import org.apache.calcite.rel.hint.RelHint; import org.junit.Assert; import org.junit.Test; -import java.util.List; - public class FfiLogicalPlanTest { // Match (x:person)-[:knows*1..3]->(:person {age: 10}) @Test @@ -89,22 +83,17 @@ public void logical_plan_test() throws Exception { + " alias=[x], opt=[VERTEX])\n" + "], matchOpt=[INNER])", aggregate.explain().trim()); - try (LogicalPlan ffiPlan = - new LogicalPlanConverter( - new GraphRelShuttleWrapper(new RelToFfiConverter(true)), - new FfiLogicalPlan( - builder.getCluster(), Utils.schemaMeta, getMockPlanHints())) - .go(aggregate)) { + try (PhysicalBuilder ffiBuilder = + new FfiPhysicalBuilder( + getMockGraphConfig(), + Utils.schemaMeta, + new LogicalPlan(aggregate, false))) { Assert.assertEquals( - FileUtils.readJsonFromResource("ffi_logical_plan.json"), ffiPlan.explain()); + FileUtils.readJsonFromResource("ffi_logical_plan.json"), ffiBuilder.explain()); } } - private List getMockPlanHints() { - return ImmutableList.of( - RelHint.builder("plan") - .hintOption("servers", "1") - .hintOption("workers", "1") - .build()); + private Configs getMockGraphConfig() { + return new Configs(ImmutableMap.of("servers", "1", "workers", "1")); } } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/Utils.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/Utils.java index 7a86a34910cc..46195a8aabd2 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/Utils.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/Utils.java @@ -17,8 +17,11 @@ package com.alibaba.graphscope.cypher.antlr4; import com.alibaba.graphscope.common.ir.tools.GraphBuilder; +import com.alibaba.graphscope.cypher.antlr4.visitor.GraphBuilderVisitor; import com.alibaba.graphscope.gremlin.plugin.script.AntlrCypherScriptEngine; +import org.antlr.v4.runtime.tree.ParseTree; + import javax.script.Bindings; import javax.script.ScriptContext; import javax.script.SimpleBindings; @@ -28,10 +31,10 @@ public abstract class Utils { public static final GraphBuilder eval(String query) { AntlrCypherScriptEngine scriptEngine = new AntlrCypherScriptEngine(); Bindings globalBindings = new SimpleBindings(); - globalBindings.put( - "graph.builder", com.alibaba.graphscope.common.ir.Utils.mockGraphBuilder()); ScriptContext context = new SimpleScriptContext(); context.setBindings(globalBindings, ScriptContext.ENGINE_SCOPE); - return (GraphBuilder) scriptEngine.eval(query, context); + ParseTree parseTree = (ParseTree) scriptEngine.eval(query, context); + return new GraphBuilderVisitor(com.alibaba.graphscope.common.ir.Utils.mockGraphBuilder()) + .visit(parseTree); } } diff --git a/interactive_engine/compiler/src/test/resources/ffi_logical_plan.json b/interactive_engine/compiler/src/test/resources/ffi_logical_plan.json index 7b92cf53fe08..9d69f1245a8d 100644 --- a/interactive_engine/compiler/src/test/resources/ffi_logical_plan.json +++ b/interactive_engine/compiler/src/test/resources/ffi_logical_plan.json @@ -291,7 +291,7 @@ { "prop_id": { "item": { - "Id": -1 + "Name": "id" } }, "type": 2 @@ -299,7 +299,7 @@ { "prop_id": { "item": { - "Id": -1 + "Name": "name" } }, "type": 4 @@ -307,7 +307,7 @@ { "prop_id": { "item": { - "Id": -1 + "Name": "age" } }, "type": 1 diff --git a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/Frontend.java b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/Frontend.java index 85a3b8da8e02..b6ebb577ebcc 100644 --- a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/Frontend.java +++ b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/Frontend.java @@ -38,8 +38,7 @@ public void start() throws Exception { int port = FrontendConfig.FRONTEND_SERVICE_PORT.get(configs); IrMetaQueryCallback queryCallback = new IrMetaQueryCallback(irMetaFetcher); server = new IrGremlinServer(port); - server.start( - configs, irMetaFetcher, channelFetcher, queryCallback, TestGraphFactory.VINEYARD); + server.start(configs, channelFetcher, queryCallback, TestGraphFactory.VINEYARD); } @Override diff --git a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardMetaFetcher.java b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardMetaFetcher.java index 544c1a33e488..a6ea6576b4e8 100644 --- a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardMetaFetcher.java +++ b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardMetaFetcher.java @@ -33,7 +33,7 @@ public VineyardMetaFetcher(String schemaPath) { JsonFileSchemaFetcher fetcher = new JsonFileSchemaFetcher(schemaPath); GraphSchema graphSchema = fetcher.getSchemaSnapshotPair().getLeft(); this.irMeta = - new IrMeta(new GraphSchemaWrapper(graphSchema, true), parser.parse(graphSchema)); + new IrMeta(new GraphSchemaWrapper(graphSchema, parser.parse(graphSchema), true)); } @Override diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java index 88999724125a..c38770ff3260 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java @@ -82,7 +82,7 @@ public void stop() { public synchronized IrMeta beforeExec() { try { IrMeta irMeta = super.beforeExec(); - QueryStatus status = new QueryStatus(irMeta.getSnapshotId()); + QueryStatus status = new QueryStatus(irMeta.getSnapshotId().getId()); queryQueue.put(status); return irMeta; } catch (InterruptedException e) { @@ -93,7 +93,7 @@ public synchronized IrMeta beforeExec() { // set the QueryStatus as done after the execution of the query @Override public synchronized void afterExec(IrMeta irMeta) { - long snapshotId = irMeta.getSnapshotId(); + long snapshotId = irMeta.getSnapshotId().getId(); queryQueue.forEach( k -> { if (k.snapshotId == snapshotId) { @@ -111,7 +111,7 @@ public void run() { queryQueue.remove(); } if (queryQueue.isEmpty()) { - minSnapshotId = fetcher.fetch().get().getSnapshotId(); + minSnapshotId = fetcher.fetch().get().getSnapshotId().getId(); } else { minSnapshotId = queryQueue.peek().snapshotId; } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootMetaFetcher.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootMetaFetcher.java index 76d45919425d..c27494daacd1 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootMetaFetcher.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootMetaFetcher.java @@ -19,6 +19,7 @@ import com.alibaba.graphscope.common.ir.schema.GraphSchemaWrapper; import com.alibaba.graphscope.common.store.IrMeta; import com.alibaba.graphscope.common.store.IrMetaFetcher; +import com.alibaba.graphscope.common.store.SnapshotId; import com.alibaba.graphscope.compiler.api.schema.*; import com.alibaba.graphscope.groot.common.util.IrSchemaParser; @@ -45,9 +46,8 @@ public Optional fetch() { && (snapshotId = pair.getRight()) != null) { return Optional.of( new IrMeta( - new GraphSchemaWrapper(schema, true), - parser.parse(schema), - snapshotId)); + new SnapshotId(true, snapshotId), + new GraphSchemaWrapper(schema, parser.parse(schema), true))); } else { return Optional.empty(); } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java index c636dfbd05d1..30a63bf479cd 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java @@ -74,11 +74,7 @@ public void start() { try { logger.info("Starting Gremlin service at port {}", port); irGremlinServer.start( - irConfigs, - irMetaFetcher, - channelFetcher, - queryManager, - TestGraphFactory.GROOT); + irConfigs, channelFetcher, queryManager, TestGraphFactory.GROOT); queryManager.start(); } catch (Exception e) {