From c1449b409594e75212275128069dc9a11b92790c Mon Sep 17 00:00:00 2001 From: shirly121 Date: Mon, 23 Sep 2024 15:46:40 +0800 Subject: [PATCH 1/3] [GIE Compiler] support call subquery in union --- .../compiler/src/main/antlr4/CypherGS.g4 | 13 ++++- .../common/ir/planner/rules/FlatJoinRule.java | 0 .../common/ir/type/GraphTypeFactoryImpl.java | 43 ++++++++++++++- .../antlr4/visitor/CallSubQueryVisitor.java | 55 +++++++++++++++++++ .../antlr4/visitor/GraphBuilderVisitor.java | 19 +++++++ 5 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinRule.java create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/CallSubQueryVisitor.java diff --git a/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 b/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 index d4edbfacd7b9..f041655c01b0 100644 --- a/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 +++ b/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 @@ -52,7 +52,18 @@ CALL : ( 'C' | 'c' ) ( 'A' | 'a' ) ( 'L' | 'l' ) ( 'L' | 'l' ) ; YIELD : ( 'Y' | 'y' ) ( 'I' | 'i' ) ( 'E' | 'e' ) ( 'L' | 'l' ) ( 'D' | 'd' ) ; oC_RegularQuery - : oC_Match ( SP? ( oC_Match | oC_With | oC_Unwind ) )* ( SP oC_Return ) ; + : oC_Match ( SP? ( oC_Match | oC_With | oC_Unwind | oC_UnionCallSubQuery ) )* ( SP oC_Return ) ; + +oC_SubQuery + : ( ( oC_Match | oC_With | oC_Unwind ) SP? )* ( SP? oC_Return ) ; + +oC_CallSubQuery + : CALL SP? '{' SP? oC_SubQuery SP? '}'; + +oC_UnionCallSubQuery + : oC_CallSubQuery ( SP? UNION SP? oC_CallSubQuery )* ; + +UNION : ( 'U' | 'u' ) ( 'N' | 'n' ) ( 'I' | 'i' ) ( 'O' | 'o' ) ( 'N' | 'n' ) ; oC_Match : ( OPTIONAL SP )? MATCH SP? oC_Pattern ( SP? oC_Where )? ; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinRule.java new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java index 15f13bc03d6e..b2a5c470bcf7 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeFactoryImpl.java @@ -22,8 +22,9 @@ import com.google.common.collect.Maps; import org.apache.calcite.jdbc.JavaTypeFactoryImpl; -import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.*; import org.apache.calcite.rex.RexNode; +import org.apache.calcite.util.Util; import org.checkerframework.checker.nullness.qual.Nullable; import java.nio.charset.Charset; @@ -97,6 +98,46 @@ public RelDataType createArbitraryMapType( return super.leastRestrictive(types); } + /** + * reimplement the {@link RelDataTypeFactoryImpl#leastRestrictiveStructuredType(List)} method + * to maintain the original alias id in the input types + * @param types + * @return + */ + @Override + protected @Nullable RelDataType leastRestrictiveStructuredType(final List types) { + final RelDataType type0 = types.get(0); + if (!type0.isStruct()) { + return null; + } + final int fieldCount = type0.getFieldCount(); + boolean isNullable = false; + for (RelDataType type : types) { + if (!type.isStruct()) { + return null; + } + if (type.getFieldList().size() != fieldCount) { + return null; + } + isNullable |= type.isNullable(); + } + List fields = Lists.newArrayList(); + for (int j = 0; j < fieldCount; ++j) { + final int k = j; + RelDataType type = + leastRestrictive(Util.transform(types, t -> t.getFieldList().get(k).getType())); + if (type == null) { + return null; + } + fields.add( + new RelDataTypeFieldImpl( + type0.getFieldList().get(j).getName(), + type0.getFieldList().get(j).getIndex(), + type)); + } + return new RelRecordType(StructKind.FULLY_QUALIFIED, fields, isNullable); + } + // re-implement lease-restrictive type inference for arbitrary map types // for each key type and value type, check if they have a least-restrictive type, otherwise // return null diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/CallSubQueryVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/CallSubQueryVisitor.java new file mode 100644 index 000000000000..b627a9858e2b --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/CallSubQueryVisitor.java @@ -0,0 +1,55 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package com.alibaba.graphscope.cypher.antlr4.visitor; + +import com.alibaba.graphscope.common.ir.meta.schema.CommonOptTable; +import com.alibaba.graphscope.common.ir.rel.CommonTableScan; +import com.alibaba.graphscope.common.ir.tools.GraphBuilder; +import com.alibaba.graphscope.grammar.CypherGSBaseVisitor; +import com.alibaba.graphscope.grammar.CypherGSParser; +import org.apache.calcite.plan.GraphOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; + +public class CallSubQueryVisitor extends CypherGSBaseVisitor { + private final GraphBuilder parentBuilder; + private final GraphBuilder nestedBuilder; + + public CallSubQueryVisitor(GraphBuilder parentBuilder) { + this.parentBuilder = parentBuilder; + this.nestedBuilder = + GraphBuilder.create( + this.parentBuilder.getContext(), + (GraphOptCluster) this.parentBuilder.getCluster(), + this.parentBuilder.getRelOptSchema()); + } + + @Override + public RelNode visitOC_CallSubQuery(CypherGSParser.OC_CallSubQueryContext ctx) { + if (parentBuilder.size() > 0) { + RelNode commonRel = parentBuilder.peek(); + RelOptTable commonTable = new CommonOptTable(commonRel); + commonRel = + new CommonTableScan( + commonRel.getCluster(), commonRel.getTraitSet(), commonTable); + nestedBuilder.push(commonRel); + } + return new GraphBuilderVisitor(nestedBuilder).visit(ctx.oC_SubQuery()).build(); + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java index ffa4485ec732..cc22c7f8de53 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java @@ -58,6 +58,25 @@ public GraphBuilderVisitor(GraphBuilder builder, ExprUniqueAliasInfer aliasInfer this.expressionVisitor = new ExpressionVisitor(this); } + @Override + public GraphBuilder visitOC_UnionCallSubQuery(CypherGSParser.OC_UnionCallSubQueryContext ctx) { + List branches = Lists.newArrayList(); + for (int i = 0; i < ctx.oC_CallSubQuery().size(); ++i) { + CypherGSParser.OC_CallSubQueryContext callSubQuery = ctx.oC_CallSubQuery(i); + if (callSubQuery != null) { + branches.add( + new CallSubQueryVisitor(this.builder).visit(callSubQuery)); + } + } + Preconditions.checkArgument( + branches.size() > 1, "union should have at least two branches in cypher queries"); + builder.build(); + for (RelNode branch : branches) { + builder.push(branch); + } + return (GraphBuilder) builder.union(true, branches.size()); + } + @Override public GraphBuilder visitOC_Cypher(CypherGSParser.OC_CypherContext ctx) { return visitOC_Statement(ctx.oC_Statement()); From ff7693f5d78a8f63a162b67dc6b7000b0954026a Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 17 Oct 2024 20:05:47 +0800 Subject: [PATCH 2/3] minor fix --- .../common/ir/planner/PlannerGroup.java | 3 + .../common/ir/planner/rules/FlatJoinRule.java | 467 +++++++++++++ .../planner/rules/FlatJoinToExpandRule.java | 113 ++++ .../ir/rel/graph/GraphLogicalExpand.java | 24 +- .../common/ir/rel/graph/GraphLogicalGetV.java | 20 +- .../ir/rel/graph/GraphLogicalSource.java | 22 +- .../common/ir/planner/cbo/LdbcTest.java | 631 +++++++++++------- 7 files changed, 1029 insertions(+), 251 deletions(-) create mode 100644 interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinToExpandRule.java diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroup.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroup.java index 110c615ab82a..3c3e787752a9 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroup.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/PlannerGroup.java @@ -58,6 +58,9 @@ public synchronized RelNode optimize(RelNode before, GraphIOProcessor ioProcesso // apply rules of 'FilterPushDown' before the match optimization relPlanner.setRoot(before); RelNode relOptimized = relPlanner.findBestExp(); + if (config.getRules().contains(FlatJoinToExpandRule.class.getSimpleName())) { + relOptimized = relOptimized.accept(new FlatJoinToExpandRule(config)); + } if (config.getOpt() == PlannerConfig.Opt.CBO) { relOptimized = relOptimized.accept( diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinRule.java index e69de29bb2d1..a4711340019b 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinRule.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinRule.java @@ -0,0 +1,467 @@ +/* + * Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.graphscope.common.ir.planner.rules; + +import com.alibaba.graphscope.common.ir.meta.schema.CommonOptTable; +import com.alibaba.graphscope.common.ir.rel.CommonTableScan; +import com.alibaba.graphscope.common.ir.rel.GraphShuttle; +import com.alibaba.graphscope.common.ir.rel.graph.*; +import com.alibaba.graphscope.common.ir.rel.graph.match.GraphLogicalSingleMatch; +import com.alibaba.graphscope.common.ir.rel.type.AliasNameWithId; +import com.alibaba.graphscope.common.ir.rex.RexGraphVariable; +import com.alibaba.graphscope.common.ir.tools.AliasInference; +import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; +import com.alibaba.graphscope.common.ir.type.GraphSchemaType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import org.apache.calcite.plan.GraphOptCluster; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlKind; +import org.apache.commons.lang3.ObjectUtils; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Map; + +public abstract class FlatJoinRule extends GraphShuttle { + private static final Logger logger = LoggerFactory.getLogger(FlatJoinRule.class); + + protected final Map optimizedCommonMap; + + protected abstract boolean matches(LogicalJoin join); + + protected abstract RelNode perform(LogicalJoin join); + + public FlatJoinRule() { + this.optimizedCommonMap = Maps.newHashMap(); + } + + @Override + public RelNode visit(LogicalJoin join) { + join = (LogicalJoin) visitChildren(join); + return matches(join) ? perform(join) : join; + } + + @Override + public RelNode visit(CommonTableScan tableScan) { + CommonOptTable optTable = (CommonOptTable) tableScan.getTable(); + String tableName = optTable.getQualifiedName().get(0); + RelNode optimized = optimizedCommonMap.get(tableName); + if (optimized == null) { + optimized = optTable.getCommon().accept(this); + } + return new CommonTableScan( + tableScan.getCluster(), tableScan.getTraitSet(), new CommonOptTable(optimized)); + } + + static @Nullable RexGraphVariable joinByOneColumn(RexNode condition, List others) { + List joinVars = Lists.newArrayList(); + classifyJoinCondition(condition, joinVars, others); + return joinVars.size() == 1 ? joinVars.get(0) : null; + } + + /** + * analyze the join condition, separate the join condition by the same tag and other conditions + * @param joinVars + * @param others + */ + static void classifyJoinCondition( + RexNode condition, List joinVars, List others) { + List conditions = RelOptUtil.conjunctions(condition); + conditions.forEach( + c -> { + boolean added = false; + if (c.getKind() == SqlKind.EQUALS) { + List operands = ((RexCall) c).getOperands(); + if (operands.size() == 2 + && operands.get(0) instanceof RexGraphVariable + && operands.get(1) instanceof RexGraphVariable) { + RexGraphVariable var1 = (RexGraphVariable) operands.get(0); + RexGraphVariable var2 = (RexGraphVariable) operands.get(1); + if (var1.getName().equals(var2.getName())) { + joinVars.add(var1); + added = true; + } + } + } + if (!added) { + others.add(c); + } + }); + } + + static boolean hasNodeEqualFilter(RelNode top) { + if (top instanceof GraphLogicalSource) { + GraphLogicalSource source = (GraphLogicalSource) top; + if (source.getUniqueKeyFilters() != null) return true; + } + if (top instanceof GraphLogicalSource || top instanceof GraphLogicalGetV) { + List filters = ((AbstractBindableTableScan) top).getFilters(); + if (filters != null && filters.size() == 1) { + RexNode filter = filters.get(0); + if (filter instanceof RexCall + && ((RexCall) filter).getOperator().getKind() == SqlKind.EQUALS) + return true; + } + } + return top.getInputs().stream().anyMatch(k -> hasNodeEqualFilter(k)); + } + + static boolean hasPxdWithUntil(RelNode top) { + if ((top instanceof GraphLogicalPathExpand) + && ((GraphLogicalPathExpand) top).getUntilCondition() != null) { + return true; + } + return top.getInputs().stream().anyMatch(k -> hasPxdWithUntil(k)); + } + + static GraphLogicalSource getSource(RelNode top) { + if (top instanceof GraphLogicalSource) { + return (GraphLogicalSource) top; + } + if (top.getInputs().isEmpty()) return null; + return getSource(top.getInput(0)); + } + + static RelNode setStartAlias(RelNode top, AliasNameWithId startAlias) { + if (top instanceof GraphLogicalExpand) { + GraphLogicalExpand expand = (GraphLogicalExpand) top; + return GraphLogicalExpand.create( + (GraphOptCluster) expand.getCluster(), + expand.getHints(), + expand.getInput(0), + expand.getOpt(), + expand.getTableConfig(), + expand.getAliasName(), + startAlias, + expand.isOptional(), + expand.getFilters(), + (GraphSchemaType) expand.getRowType().getFieldList().get(0).getType()); + } else if (top instanceof GraphLogicalPathExpand) { + GraphLogicalPathExpand pxd = (GraphLogicalPathExpand) top; + return GraphLogicalPathExpand.create( + (GraphOptCluster) pxd.getCluster(), + ImmutableList.of(), + pxd.getInput(), + pxd.getExpand(), + pxd.getGetV(), + pxd.getOffset(), + pxd.getFetch(), + pxd.getResultOpt(), + pxd.getPathOpt(), + pxd.getUntilCondition(), + pxd.getAliasName(), + startAlias, + pxd.isOptional()); + } else if (top instanceof GraphLogicalGetV) { + GraphLogicalGetV getV = (GraphLogicalGetV) top; + return GraphLogicalGetV.create( + (GraphOptCluster) getV.getCluster(), + getV.getHints(), + getV.getInput(0), + getV.getOpt(), + getV.getTableConfig(), + getV.getAliasName(), + startAlias, + getV.getFilters()); + } + logger.warn("unable to set start alias of the rel = [" + top + "]"); + return top; + } + + static GraphOpt.Expand getExpandOpt(RelNode top) { + if (top instanceof GraphLogicalExpand) { + return ((GraphLogicalExpand) top).getOpt(); + } + if (top instanceof GraphLogicalPathExpand) { + return getExpandOpt(((GraphLogicalPathExpand) top).getExpand()); + } + throw new IllegalArgumentException("unable to get expand opt from rel = [" + top + "]"); + } + + static RelNode toSource(RelNode top) { + if (top instanceof GraphLogicalGetV) { + return GraphLogicalSource.create( + (GraphOptCluster) top.getCluster(), + ((GraphLogicalGetV) top).getHints(), + GraphOpt.Source.VERTEX, + ((GraphLogicalGetV) top).getTableConfig(), + ((GraphLogicalGetV) top).getAliasName(), + null, + ((GraphLogicalGetV) top).getFilters()); + } + throw new IllegalArgumentException("unable to convert rel = [" + top + "] to source"); + } + + static RelNode reverse(RelNode top, @Nullable RelNode reversed) { + String startAliasName = getAliasName(reversed); + Integer startAliasId = getAliasId(reversed); + AliasNameWithId startAlias = + new AliasNameWithId( + startAliasName == null ? AliasInference.DEFAULT_NAME : startAliasName, + startAliasId == null ? AliasInference.DEFAULT_ID : startAliasId); + if (top instanceof GraphLogicalSource) { + GraphOpt.Expand expandOpt = getExpandOpt(reversed); + GraphOpt.GetV reversedOpt; + switch (expandOpt) { + case BOTH: + reversedOpt = GraphOpt.GetV.OTHER; + break; + case OUT: + reversedOpt = GraphOpt.GetV.END; + break; + case IN: + default: + reversedOpt = GraphOpt.GetV.START; + } + ImmutableList.Builder filters = ImmutableList.builder(); + if (((GraphLogicalSource) top).getUniqueKeyFilters() != null) { + filters.add(((GraphLogicalSource) top).getUniqueKeyFilters()); + } + if (ObjectUtils.isNotEmpty(((GraphLogicalSource) top).getFilters())) { + filters.addAll(((GraphLogicalSource) top).getFilters()); + } + return GraphLogicalGetV.create( + (GraphOptCluster) top.getCluster(), + ((GraphLogicalSource) top).getHints(), + reversed, + reversedOpt, + ((GraphLogicalSource) top).getTableConfig(), + ((GraphLogicalSource) top).getAliasName(), + startAlias, + filters.build()); + } + if (top instanceof GraphLogicalExpand) { + GraphLogicalExpand expand = (GraphLogicalExpand) top; + GraphOpt.Expand reversedOpt; + switch (expand.getOpt()) { + case BOTH: + reversedOpt = GraphOpt.Expand.BOTH; + break; + case OUT: + reversedOpt = GraphOpt.Expand.IN; + break; + case IN: + default: + reversedOpt = GraphOpt.Expand.OUT; + } + return GraphLogicalExpand.create( + (GraphOptCluster) top.getCluster(), + ((GraphLogicalExpand) top).getHints(), + reversed, + reversedOpt, + ((GraphLogicalExpand) top).getTableConfig(), + ((GraphLogicalExpand) top).getAliasName(), + startAlias, + ((GraphLogicalExpand) top).isOptional(), + ((GraphLogicalExpand) top).getFilters(), + (GraphSchemaType) top.getRowType().getFieldList().get(0).getType()); + } + if (top instanceof GraphLogicalPathExpand) { + return GraphLogicalPathExpand.create( + (GraphOptCluster) top.getCluster(), + ImmutableList.of(), + reversed, + reverse(((GraphLogicalPathExpand) top).getExpand(), null), + reverse(((GraphLogicalPathExpand) top).getGetV(), null), + ((GraphLogicalPathExpand) top).getOffset(), + ((GraphLogicalPathExpand) top).getFetch(), + ((GraphLogicalPathExpand) top).getResultOpt(), + ((GraphLogicalPathExpand) top).getPathOpt(), + ((GraphLogicalPathExpand) top) + .getUntilCondition(), // todo: path expand with until condition cannot + // be reversed + ((GraphLogicalPathExpand) top).getAliasName(), + startAlias, + ((GraphLogicalPathExpand) top).isOptional()); + } + if (top instanceof GraphLogicalGetV) { + GraphLogicalGetV getV = (GraphLogicalGetV) top; + GraphOpt.GetV reversedOpt; + switch (getV.getOpt()) { + case OTHER: + reversedOpt = GraphOpt.GetV.OTHER; + break; + case BOTH: + reversedOpt = GraphOpt.GetV.BOTH; + break; + case END: + reversedOpt = GraphOpt.GetV.START; + break; + case START: + default: + reversedOpt = GraphOpt.GetV.END; + } + return GraphLogicalGetV.create( + (GraphOptCluster) top.getCluster(), + ((GraphLogicalGetV) top).getHints(), + reversed, + reversedOpt, + ((GraphLogicalGetV) top).getTableConfig(), + ((GraphLogicalGetV) top).getAliasName(), + startAlias, + ((GraphLogicalGetV) top).getFilters()); + } + logger.warn("unable to reverse the rel = [" + top + "]"); + return top; + } + + static void getMatchBeforeJoin(RelNode top, List matches) { + if (top instanceof GraphLogicalSingleMatch) { + matches.add(0, (GraphLogicalSingleMatch) top); + } + if (top.getInputs().size() != 1) return; + getMatchBeforeJoin(top.getInput(0), matches); + } + + static @Nullable Integer getAliasId(@Nullable RelNode rel) { + if (rel instanceof AbstractBindableTableScan) { + return ((AbstractBindableTableScan) rel).getAliasId(); + } + if (rel instanceof GraphLogicalPathExpand) { + return ((GraphLogicalPathExpand) rel).getAliasId(); + } + return null; + } + + static @Nullable String getAliasName(@Nullable RelNode rel) { + if (rel instanceof AbstractBindableTableScan) { + return ((AbstractBindableTableScan) rel).getAliasName(); + } + if (rel instanceof GraphLogicalPathExpand) { + return ((GraphLogicalPathExpand) rel).getAliasName(); + } + return null; + } + + static class SetOptional extends GraphShuttle { + @Override + public RelNode visit(GraphLogicalExpand expand) { + expand = (GraphLogicalExpand) visitChildren(expand); + RelDataType originalType = expand.getRowType().getFieldList().get(0).getType(); + GraphSchemaType schemaTypeNullable = + (GraphSchemaType) + expand.getCluster() + .getTypeFactory() + .createTypeWithNullability(originalType, true); + return GraphLogicalExpand.create( + (GraphOptCluster) expand.getCluster(), + expand.getHints(), + expand.getInput(0), + expand.getOpt(), + expand.getTableConfig(), + expand.getAliasName(), + expand.getStartAlias(), + true, + expand.getFilters(), + schemaTypeNullable); + } + + @Override + public RelNode visit(GraphLogicalPathExpand pxd) { + pxd = (GraphLogicalPathExpand) visitChildren(pxd); + return GraphLogicalPathExpand.create( + (GraphOptCluster) pxd.getCluster(), + ImmutableList.of(), + pxd.getInput(), + pxd.getExpand(), + pxd.getGetV(), + pxd.getOffset(), + pxd.getFetch(), + pxd.getResultOpt(), + pxd.getPathOpt(), + pxd.getUntilCondition(), + pxd.getAliasName(), + pxd.getStartAlias(), + true); + } + } + + static class Reverse extends GraphShuttle { + private final RelNode top; + private RelNode reversed; + + public Reverse(RelNode top) { + this.top = top; + } + + @Override + public RelNode visit(GraphLogicalSource source) { + if (top == source) return source; + return reverse(source, reversed); + } + + @Override + public RelNode visit(GraphLogicalExpand expand) { + reversed = reverse(expand, reversed); + return expand.getInput(0).accept(this); + } + + @Override + public RelNode visit(GraphLogicalGetV getV) { + reversed = (getV == top) ? toSource(getV) : reverse(getV, reversed); + return getV.getInput(0).accept(this); + } + + @Override + public RelNode visit(GraphLogicalPathExpand pxd) { + reversed = reverse(pxd, reversed); + return pxd.getInput(0).accept(this); + } + } + + static class ReplaceInput extends GraphShuttle { + private final AliasNameWithId inputAlias; + private final RelNode input; + + public ReplaceInput(AliasNameWithId inputAlias, RelNode input) { + this.inputAlias = inputAlias; + this.input = input; + } + + @Override + public RelNode visit(GraphLogicalExpand expand) { + boolean resetAlias = + !expand.getInputs().isEmpty() + && expand.getInput(0) instanceof GraphLogicalSource; + expand = (GraphLogicalExpand) visitChildren(expand); + return resetAlias ? setStartAlias(expand, inputAlias) : expand; + } + + @Override + public RelNode visit(GraphLogicalPathExpand pxd) { + boolean resetAlias = + !pxd.getInputs().isEmpty() && pxd.getInput(0) instanceof GraphLogicalSource; + pxd = (GraphLogicalPathExpand) visitChildren(pxd); + return resetAlias ? setStartAlias(pxd, inputAlias) : pxd; + } + + @Override + public RelNode visit(GraphLogicalSource source) { + return input; + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinToExpandRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinToExpandRule.java new file mode 100644 index 000000000000..193d3d86cb0f --- /dev/null +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinToExpandRule.java @@ -0,0 +1,113 @@ +/* + * + * * Copyright 2020 Alibaba Group Holding Limited. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ +package com.alibaba.graphscope.common.ir.planner.rules; + +import com.alibaba.graphscope.common.config.PlannerConfig; +import com.alibaba.graphscope.common.ir.rel.GraphShuttle; +import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalSource; +import com.alibaba.graphscope.common.ir.rel.graph.match.GraphLogicalSingleMatch; +import com.alibaba.graphscope.common.ir.rel.type.AliasNameWithId; +import com.alibaba.graphscope.common.ir.rex.RexGraphVariable; +import com.google.common.collect.Lists; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.JoinRelType; +import org.apache.calcite.rel.logical.LogicalFilter; +import org.apache.calcite.rel.logical.LogicalJoin; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexUtil; + +import java.util.List; + +public class FlatJoinToExpandRule extends FlatJoinRule { + // left plan of the join + private RelNode left; + // alias of the left key in the join condition + private AliasNameWithId leftAlias; + // whether the 'MatchOpt' in the right plan is optional + private boolean optional; + private List otherJoinConditions; + + private final PlannerConfig config; + + public FlatJoinToExpandRule(PlannerConfig config) { + this.config = config; + } + + @Override + protected boolean matches(LogicalJoin join) { + RexNode condition = join.getCondition(); + List others = Lists.newArrayList(); + RexGraphVariable var = joinByOneColumn(condition, others); + if (var == null) return false; + List matches = Lists.newArrayList(); + getMatchBeforeJoin(join.getRight(), matches); + if (matches.size() != 1) return false; + RelNode sentence = matches.get(0).getSentence(); + if (hasPxdWithUntil(sentence)) return false; + if (hasNodeEqualFilter(sentence)) return false; + GraphLogicalSource source = getSource(sentence); + List startEndAliasIds = + Lists.newArrayList(getAliasId(source), getAliasId(sentence)); + // check whether the sentence starts or ends with the alias id in the join condition + boolean contains = startEndAliasIds.contains(var.getAliasId()); + if (contains) { + left = join.getLeft(); + leftAlias = new AliasNameWithId(var.getName().split("\\.")[0], var.getAliasId()); + optional = join.getJoinType() == JoinRelType.LEFT; + otherJoinConditions = others; + } + return contains; + } + + @Override + protected RelNode perform(LogicalJoin join) { + return join.getRight().accept(new ExpandFlatter()); + } + + // perform the transformation from the join plan to the expand plan + // replace the source operator of the right plan with the left plan, + // i.e. join(getV1->expand1->source1, getV2->expand2->source2) => + // getV2->expand2->getV1->expand1->source1 + private class ExpandFlatter extends GraphShuttle { + @Override + public RelNode visit(GraphLogicalSingleMatch match) { + RelNode sentence = match.getSentence(); + // reverse the sentence if the expand order does not start from the 'leftAlias' + RelNode source = getSource(sentence); + if (getAliasId(source) != leftAlias.getAliasId()) { + sentence = sentence.accept(new Reverse(sentence)); + } + // set expand operators in the right plan as optional if 'optional' = true + if (optional) { + sentence = sentence.accept(new SetOptional()); + } + // replace the source operator of the sentence with the left plan + RelNode expand = sentence.accept(new ReplaceInput(leftAlias, left)); + // if there are other join conditions, add a filter operator on top of the expand + // operator + if (!otherJoinConditions.isEmpty()) { + RexNode otherJoinCondition = + RexUtil.composeConjunction( + match.getCluster().getRexBuilder(), otherJoinConditions); + LogicalFilter filter = LogicalFilter.create(expand, otherJoinCondition); + expand = filter; + } + return expand; + } + } +} diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java index d203a8122790..4b10f028779b 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java @@ -22,7 +22,7 @@ import com.alibaba.graphscope.common.ir.rel.type.TableConfig; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; - +import com.google.common.collect.ImmutableList; import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; @@ -30,6 +30,7 @@ import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexNode; import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; @@ -77,6 +78,27 @@ public static GraphLogicalExpand create( cluster, hints, input, opt, tableConfig, alias, startAlias, optional); } + public static GraphLogicalExpand create( + GraphOptCluster cluster, + List hints, + RelNode input, + GraphOpt.Expand opt, + TableConfig tableConfig, + @Nullable String alias, + AliasNameWithId startAlias, + boolean optional, + ImmutableList filters, + GraphSchemaType schemaType) { + GraphLogicalExpand expand = + GraphLogicalExpand.create( + cluster, hints, input, opt, tableConfig, alias, startAlias, optional); + if (ObjectUtils.isNotEmpty(filters)) { + expand.setFilters(filters); + } + expand.setSchemaType(schemaType); + return expand; + } + public GraphOpt.Expand getOpt() { return this.opt; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java index 7bb48bfe2d9c..cbebf7388fb0 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java @@ -21,7 +21,7 @@ import com.alibaba.graphscope.common.ir.rel.type.AliasNameWithId; import com.alibaba.graphscope.common.ir.rel.type.TableConfig; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; - +import com.google.common.collect.ImmutableList; import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; @@ -29,6 +29,7 @@ import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rel.metadata.RelMetadataQuery; +import org.apache.calcite.rex.RexNode; import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; @@ -60,6 +61,23 @@ public static GraphLogicalGetV create( return new GraphLogicalGetV(cluster, hints, input, opt, tableConfig, alias, startAlias); } + public static GraphLogicalGetV create( + GraphOptCluster cluster, + List hints, + RelNode input, + GraphOpt.GetV opt, + TableConfig tableConfig, + @Nullable String alias, + AliasNameWithId startAlias, + ImmutableList filters) { + GraphLogicalGetV getV = + GraphLogicalGetV.create(cluster, hints, input, opt, tableConfig, alias, startAlias); + if (ObjectUtils.isNotEmpty(filters)) { + getV.setFilters(filters); + } + return getV; + } + public GraphOpt.GetV getOpt() { return this.opt; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java index 321b4109cc2f..70c6a3d9f8bc 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java @@ -19,13 +19,14 @@ import com.alibaba.graphscope.common.ir.rel.GraphShuttle; import com.alibaba.graphscope.common.ir.rel.type.TableConfig; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; - +import com.google.common.collect.ImmutableList; import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelShuttle; import org.apache.calcite.rel.RelWriter; import org.apache.calcite.rel.hint.RelHint; import org.apache.calcite.rex.RexNode; +import org.apache.commons.lang3.ObjectUtils; import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; @@ -54,6 +55,25 @@ public static GraphLogicalSource create( return new GraphLogicalSource(cluster, hints, opt, tableConfig, alias); } + public static GraphLogicalSource create( + GraphOptCluster cluster, + List hints, + GraphOpt.Source opt, + TableConfig tableConfig, + @Nullable String alias, + RexNode uniqueKeyFilters, + ImmutableList filters) { + GraphLogicalSource source = + GraphLogicalSource.create(cluster, hints, opt, tableConfig, alias); + if (uniqueKeyFilters != null) { + source.setUniqueKeyFilters(uniqueKeyFilters); + } + if (ObjectUtils.isNotEmpty(filters)) { + source.setFilters(filters); + } + return source; + } + public GraphOpt.Source getOpt() { return this.opt; } diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java index ea08d443639f..8f4c380f8ca7 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java @@ -28,7 +28,7 @@ public static void beforeClass() { "graph.planner.opt", "CBO", "graph.planner.rules", - "NotMatchToAntiJoinRule, FilterIntoJoinRule, FilterMatchRule," + "NotMatchToAntiJoinRule, FilterIntoJoinRule, FilterMatchRule, FlatJoinToExpandRule," + " ExtendIntersectRule, ExpandGetVFusionRule")); optimizer = new GraphRelOptimizer(configs); irMeta = @@ -43,108 +43,92 @@ public void ldbc1_test() { GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); RelNode before = com.alibaba.graphscope.cypher.antlr4.Utils.eval( - "MATCH (p: PERSON{id: $personId}) -[k:KNOWS*1..4]-(f: PERSON" - + " {firstName: $firstName})\n" - + "OPTIONAL MATCH (f:" - + " PERSON)-[workAt:WORKAT]->(company:ORGANISATION)-[:ISLOCATEDIN]->(country:PLACE)\n" - + "OPTIONAL MATCH (f:" - + " PERSON)-[studyAt:STUDYAT]->(university)-[:ISLOCATEDIN]->(universityCity:PLACE)\n" - + "MATCH (f:PERSON)-[:ISLOCATEDIN]->(locationCity:PLACE)\n" - + "WHERE p <> f\n" - + "with f AS f, company, university, workAt, country, studyAt," - + " universityCity, locationCity, length(k) as len\n" - + "with f AS f, company, university, workAt, country, studyAt," - + " universityCity, locationCity, min(len) as distance\n" + "MATCH shortestPath(p: PERSON {id : $personId})" + + " -[k:KNOWS*1..4]-(f:PERSON {firstName : $firstName})\n" + + "where f <> p\n" + + "WITH f, length(k) as distance\n" + "ORDER BY distance ASC, f.lastName ASC, f.id ASC\n" + "LIMIT 20\n" + "\n" + + "\n" + + "OPTIONAL MATCH (f:" + + " PERSON)-[workAt:WORKAT]->(company:ORGANISATION)-[:ISLOCATEDIN]->(country:PLACE)\n" + "WITH \n" - + " f, distance, locationCity,\n" + + " f, distance,\n" + " CASE\n" + " WHEN company is null Then null\n" + " ELSE [company.name, workAt.workFrom, country.name]\n" - + " END as companies,\n" - + " CASE \n" - + "\t\t WHEN university is null Then null\n" - + "\t\t ELSE [university.name, studyAt.classYear," + + " END as companies\n" + + "WITH f, collect(companies) as company_info, distance\n" + + "\n" + + "OPTIONAL MATCH (f:" + + " PERSON)-[studyAt:STUDYAT]->(university)-[:ISLOCATEDIN]->(universityCity:PLACE)\n" + + "WITH\n" + + " f, company_info, distance,\n" + + "\tCASE \n" + + "\t\tWHEN university is null Then null\n" + + "\t\tELSE [university.name, studyAt.classYear," + " universityCity.name]\n" - + "\t END as universities\n" - + "WITH f, distance, locationCity, collect(companies) as" - + " company_info, collect(universities) as university_info\n" + + "\tEND as universities\n" + + "WITH f, collect(universities) as university_info ," + + " company_info, distance\n" + "\n" - + "return f.id AS friendId,\n" - + " f.lastName AS friendLastName,\n" - + " distance AS distanceFromPerson,\n" - + " f.birthday AS friendBirthday,\n" - + " f.creationDate AS friendCreationDate,\n" - + " f.gender AS friendGender,\n" - + " f.browserUsed AS friendBrowserUsed,\n" - + " f.locationIP AS friendLocationIp,\n" - + " locationCity.name AS friendCityName,\n" - + " university_info AS friendUniversities,\n" - + " company_info AS friendCompanies;", + + "MATCH (f:PERSON)-[:ISLOCATEDIN]->(locationCity:PLACE)\n" + + "\n" + + "return f.id AS friendId;", builder) .build(); RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); Assert.assertEquals( - "GraphLogicalProject(friendId=[f.id], friendLastName=[f.lastName]," - + " distanceFromPerson=[distance], friendBirthday=[f.birthday]," - + " friendCreationDate=[f.creationDate], friendGender=[f.gender]," - + " friendBrowserUsed=[f.browserUsed], friendLocationIp=[f.locationIP]," - + " friendCityName=[locationCity.name], friendUniversities=[university_info]," - + " friendCompanies=[company_info], isAppend=[false])\n" - + " GraphLogicalAggregate(keys=[{variables=[f, distance, locationCity]," - + " aliases=[f, distance, locationCity]}], values=[[{operands=[companies]," - + " aggFunction=COLLECT, alias='company_info', distinct=false}," - + " {operands=[universities], aggFunction=COLLECT, alias='university_info'," - + " distinct=false}]])\n" - + " GraphLogicalProject(f=[f], distance=[distance]," - + " locationCity=[locationCity], companies=[CASE(IS NULL(company), null:NULL," - + " ARRAY(company.name, workAt.workFrom, country.name))], universities=[CASE(IS" - + " NULL(university), null:NULL, ARRAY(university.name, studyAt.classYear," - + " universityCity.name))], isAppend=[false])\n" - + " GraphLogicalSort(sort0=[distance], sort1=[f.lastName], sort2=[f.id]," - + " dir0=[ASC], dir1=[ASC], dir2=[ASC], fetch=[20])\n" - + " GraphLogicalAggregate(keys=[{variables=[f, company, university," - + " workAt, country, studyAt, universityCity, locationCity], aliases=[f," - + " company, university, workAt, country, studyAt, universityCity," - + " locationCity]}], values=[[{operands=[len], aggFunction=MIN," - + " alias='distance', distinct=false}]])\n" - + " GraphLogicalProject(f=[f], company=[company]," - + " university=[university], workAt=[workAt], country=[country]," - + " studyAt=[studyAt], universityCity=[universityCity]," - + " locationCity=[locationCity], len=[k.~len], isAppend=[false])\n" - + " LogicalFilter(condition=[<>(p, f)])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN," - + " ORGANISATION, PLACE)]], alias=[country], startAlias=[company], opt=[OUT]," + "GraphLogicalProject(friendId=[f.id], isAppend=[false])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN, PERSON, PLACE)]]," + + " alias=[locationCity], startAlias=[f], opt=[OUT], physicalOpt=[VERTEX])\n" + + " GraphLogicalAggregate(keys=[{variables=[f, company_info, distance]," + + " aliases=[f, company_info, distance]}], values=[[{operands=[universities]," + + " aggFunction=COLLECT, alias='university_info', distinct=false}]])\n" + + " GraphLogicalProject(f=[f], company_info=[company_info]," + + " distance=[distance], universities=[CASE(IS NULL(university), null:NULL," + + " ARRAY(university.name, studyAt.classYear, universityCity.name))]," + + " isAppend=[false])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN," + + " ORGANISATION, PLACE)]], alias=[universityCity], opt=[OUT]," + " physicalOpt=[VERTEX], optional=[true])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false," - + " tables=[ORGANISATION]}], alias=[company], opt=[END])\n" - + " GraphLogicalExpand(tableConfig=[{isAll=false," - + " tables=[WORKAT]}], alias=[workAt], startAlias=[f], opt=[OUT]," - + " optional=[true])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN," - + " ORGANISATION, PLACE)]], alias=[universityCity], startAlias=[university]," - + " opt=[OUT], physicalOpt=[VERTEX], optional=[true])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false," + + " GraphLogicalGetV(tableConfig=[{isAll=false," + " tables=[ORGANISATION]}], alias=[university], opt=[END])\n" - + " GraphLogicalExpand(tableConfig=[{isAll=false," + + " GraphLogicalExpand(tableConfig=[{isAll=false," + " tables=[STUDYAT]}], alias=[studyAt], startAlias=[f], opt=[OUT]," + " optional=[true])\n" - + " " - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN, PERSON, PLACE)]]," - + " alias=[locationCity], startAlias=[f], opt=[OUT], physicalOpt=[VERTEX])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false," + + " GraphLogicalAggregate(keys=[{variables=[f, distance]," + + " aliases=[f, distance]}], values=[[{operands=[companies]," + + " aggFunction=COLLECT, alias='company_info', distinct=false}]])\n" + + " GraphLogicalProject(f=[f], distance=[distance]," + + " companies=[CASE(IS NULL(company), null:NULL, ARRAY(company.name," + + " workAt.workFrom, country.name))], isAppend=[false])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN," + + " ORGANISATION, PLACE)]], alias=[country], opt=[OUT], physicalOpt=[VERTEX]," + + " optional=[true])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=false," + + " tables=[ORGANISATION]}], alias=[company], opt=[END])\n" + + " GraphLogicalExpand(tableConfig=[{isAll=false," + + " tables=[WORKAT]}], alias=[workAt], startAlias=[f], opt=[OUT]," + + " optional=[true])\n" + + " GraphLogicalSort(sort0=[distance]," + + " sort1=[f.lastName], sort2=[f.id], dir0=[ASC], dir1=[ASC], dir2=[ASC]," + + " fetch=[20])\n" + + " GraphLogicalProject(f=[f], distance=[k.~len]," + + " isAppend=[false])\n" + + " LogicalFilter(condition=[<>(f, p)])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=false," + " tables=[PERSON]}], alias=[f], fusedFilter=[[=(_.firstName, ?1)]]," + " opt=[END])\n" - + " " + + " " + " GraphLogicalPathExpand(expand=[GraphLogicalExpand(tableConfig=[{isAll=false," + " tables=[KNOWS]}], alias=[_], opt=[BOTH])\n" + "], getV=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," + " alias=[_], opt=[OTHER])\n" - + "], offset=[1], fetch=[3], path_opt=[ARBITRARY], result_opt=[ALL_V_E]," + + "], offset=[1], fetch=[3], path_opt=[ANY_SHORTEST], result_opt=[ALL_V_E]," + " alias=[k], start_alias=[p])\n" - + " " + + " " + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}], alias=[p]," + " opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])", after.explain().trim()); @@ -355,64 +339,87 @@ public void ldbc5_test() { GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); RelNode before = com.alibaba.graphscope.cypher.antlr4.Utils.eval( - "MATCH (person:PERSON { id: $personId })-[:KNOWS*1..2]-(friend)\n" - + "MATCH (friend)<-[membership:HASMEMBER]-(forum)\n" - + "WHERE membership.joinDate > $minDate\n" - + "OPTIONAL MATCH" - + " (friend)<-[:HASCREATOR]-(post)<-[:CONTAINEROF]-(forum)\n" - + "WHERE\n" - + " NOT person=friend\n" - + "WITH\n" - + " forum,\n" - + " count(distinct post) AS postCount\n" - + "ORDER BY\n" - + " postCount DESC,\n" - + " forum.id ASC\n" - + "LIMIT 20\n" - + "RETURN\n" - + " forum.title AS forumName,\n" - + " postCount;", + "MATCH (person:PERSON { id: $personId })-[:KNOWS*1..3]-(friend)\n" + + "WITH DISTINCT friend\n" + + "WHERE friend.id <> $personId\n" + + "MATCH (friend)<-[membership:HASMEMBER]-(forum)\n" + + "WHERE membership.joinDate > $minDate\n" + + "CALL {\n" + + " WITH forum\n" + + " RETURN forum, 0 AS postCount\n" + + " ORDER BY forum.id ASC\n" + + " LIMIT 20\n" + + "}\n" + + "UNION\n" + + "CALL {\n" + + " WITH friend, collect(forum) AS forums\n" + + " MATCH" + + " (friend)<-[:HASCREATOR]-(post)<-[:CONTAINEROF]-(forum)\n" + + " WHERE forum IN forums\n" + + " WITH forum, count(post) AS postCount\n" + + " RETURN forum, postCount\n" + + " ORDER BY postCount DESC, forum.id ASC\n" + + " LIMIT 20\n" + + "}\n" + + "WITH forum, max(postCount) AS postCount\n" + + "RETURN forum, postCount\n" + + "ORDER BY postCount DESC, forum.id ASC\n" + + "LIMIT 20;", builder) .build(); RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); Assert.assertEquals( "root:\n" - + "GraphLogicalProject(forumName=[forum.title], postCount=[postCount]," - + " isAppend=[false])\n" - + " GraphLogicalSort(sort0=[postCount], sort1=[forum.id], dir0=[DESC]," - + " dir1=[ASC], fetch=[20])\n" - + " GraphLogicalAggregate(keys=[{variables=[forum], aliases=[forum]}]," - + " values=[[{operands=[post], aggFunction=COUNT, alias='postCount'," - + " distinct=true}]])\n" - + " LogicalFilter(condition=[<>(person, friend)])\n" - + " MultiJoin(joinFilter=[=(post, post)], isFullOuterJoin=[false]," - + " joinTypes=[[INNER, INNER]], outerJoinConditions=[[NULL, NULL]]," - + " projFields=[[ALL, ALL]])\n" - + " GraphPhysicalExpand(tableConfig=[{isAll=false," - + " tables=[CONTAINEROF]}], alias=[post], startAlias=[forum], opt=[OUT]," - + " physicalOpt=[VERTEX], optional=[true])\n" - + " CommonTableScan(table=[[common#391831169]])\n" - + " GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}]," - + " alias=[post], opt=[START], physicalOpt=[ITSELF])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST," - + " PERSON)]], alias=[_], startAlias=[friend], opt=[IN], physicalOpt=[VERTEX]," - + " optional=[true])\n" - + " CommonTableScan(table=[[common#391831169]])\n" - + "common#391831169:\n" - + "GraphLogicalGetV(tableConfig=[{isAll=false, tables=[FORUM]}], alias=[forum]," - + " opt=[START])\n" - + " GraphLogicalExpand(tableConfig=[{isAll=false, tables=[HASMEMBER]}]," - + " alias=[membership], startAlias=[friend], fusedFilter=[[>(_.joinDate, ?1)]]," - + " opt=[IN])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," - + " alias=[friend], opt=[END])\n" - + " " - + " GraphLogicalPathExpand(fused=[GraphPhysicalExpand(tableConfig=[{isAll=false," - + " tables=[KNOWS]}], alias=[_], opt=[BOTH], physicalOpt=[VERTEX])\n" - + "], offset=[1], fetch=[1], path_opt=[ARBITRARY], result_opt=[END_V]," - + " alias=[_], start_alias=[person])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}]," - + " alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])", + + "GraphLogicalSort(sort0=[postCount], sort1=[forum.id], dir0=[DESC]," + + " dir1=[ASC], fetch=[20])\n" + + " GraphLogicalProject(forum=[forum], postCount=[postCount]," + + " isAppend=[false])\n" + + " GraphLogicalAggregate(keys=[{variables=[forum], aliases=[forum]}]," + + " values=[[{operands=[postCount], aggFunction=MAX, alias='postCount'," + + " distinct=false}]])\n" + + " LogicalUnion(all=[true])\n" + + " GraphLogicalSort(sort0=[forum.id], dir0=[ASC], fetch=[20])\n" + + " GraphLogicalProject(forum=[forum], postCount=[0]," + + " isAppend=[false])\n" + + " GraphLogicalProject(forum=[forum], isAppend=[false])\n" + + " CommonTableScan(table=[[common#1874145243]])\n" + + " GraphLogicalSort(sort0=[postCount], sort1=[forum.id], dir0=[DESC]," + + " dir1=[ASC], fetch=[20])\n" + + " GraphLogicalProject(forum=[forum], postCount=[postCount]," + + " isAppend=[false])\n" + + " GraphLogicalAggregate(keys=[{variables=[forum]," + + " aliases=[forum]}], values=[[{operands=[post], aggFunction=COUNT," + + " alias='postCount', distinct=false}]])\n" + + " LogicalFilter(condition=[IN(forum, forums)])\n" + + " GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[CONTAINEROF]}], alias=[forum], opt=[IN], physicalOpt=[VERTEX])\n" + + " GraphPhysicalGetV(tableConfig=[{isAll=false," + + " tables=[POST]}], alias=[post], opt=[START], physicalOpt=[ITSELF])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR," + + " POST, PERSON)]], alias=[_], startAlias=[friend], opt=[IN]," + + " physicalOpt=[VERTEX])\n" + + " GraphLogicalAggregate(keys=[{variables=[friend]," + + " aliases=[friend]}], values=[[{operands=[forum], aggFunction=COLLECT," + + " alias='forums', distinct=false}]])\n" + + " CommonTableScan(table=[[common#1874145243]])\n" + + "common#1874145243:\n" + + "GraphLogicalGetV(tableConfig=[{isAll=false, tables=[FORUM]}], alias=[forum]," + + " opt=[START])\n" + + " GraphLogicalExpand(tableConfig=[{isAll=false, tables=[HASMEMBER]}]," + + " alias=[membership], startAlias=[friend], fusedFilter=[[>(_.joinDate, ?1)]]," + + " opt=[IN])\n" + + " LogicalFilter(condition=[<>(friend.id, ?0)])\n" + + " GraphLogicalAggregate(keys=[{variables=[friend], aliases=[friend]}]," + + " values=[[]])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," + + " alias=[friend], opt=[END])\n" + + " " + + " GraphLogicalPathExpand(fused=[GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[KNOWS]}], alias=[_], opt=[BOTH], physicalOpt=[VERTEX])\n" + + "], offset=[1], fetch=[2], path_opt=[ARBITRARY], result_opt=[END_V]," + + " alias=[_], start_alias=[person])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}]," + + " alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])", com.alibaba.graphscope.common.ir.tools.Utils.toString(after).trim()); } @@ -463,6 +470,34 @@ public void ldbc6_test() { after.explain().trim()); } + @Test + public void ldbc6_2_test() { + GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); + RelNode before = + com.alibaba.graphscope.cypher.antlr4.Utils.eval( + "MATCH (p_:PERSON {id: $personId})-[:KNOWS*1..3]-(other:PERSON)\n" + + "WITH distinct other\n" + + "WHERE other.id <> $personId\n" + + "\n" + + "MATCH (other)<-[:HASCREATOR]-(p:POST)-[:HASTAG]->(t:TAG" + + " {name: $tagName})\n" + + "\n" + + "Match (p:POST)-[:HASTAG]->(otherTag:TAG)\n" + + "WHERE \n" + + " otherTag <> t \n" + + "RETURN\n" + + " otherTag.name as name,\n" + + " count(distinct p) as postCnt \n" + + "ORDER BY \n" + + " postCnt desc, \n" + + " name asc \n" + + "LIMIT 10;", + builder) + .build(); + RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); + // System.out.println(after.explain()); + } + // todo: fix issues in ldbc7: expand (with alias) + getV cannot be fused thus causing the // execution errors of extend intersect @Test @@ -588,20 +623,26 @@ public void ldbc9_test() { GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); RelNode before = com.alibaba.graphscope.cypher.antlr4.Utils.eval( - "MATCH (person:PERSON {id:" - + " 2199023382370})-[:KNOWS*1..3]-(friend:PERSON)<-[:HASCREATOR]-(message)\n" - + "WHERE friend <> person\n" - + " and message.creationDate < 20130301000000000\n" - + "RETURN\n" - + " friend.id AS personId,\n" - + " friend.firstName AS personFirstName,\n" - + " friend.lastName AS personLastName,\n" - + " message.id AS commentOrPostId,\n" - + " message.creationDate AS commentOrPostCreationDate\n" - + "ORDER BY\n" - + " commentOrPostCreationDate DESC,\n" - + " commentOrPostId ASC\n" - + "LIMIT 20;", + "MATCH (p:PERSON {id: $personId})-[:KNOWS*1..3]-(friend:PERSON)\n" + + "WITH distinct friend\n" + + "where friend.id <> $personId\n" + + "MATCH " + + " (message:POST|COMMENT)-[:HASCREATOR]->(friend:PERSON)\n" + + "where message.creationDate < $maxDate\n" + + "WITH friend, message\n" + + "\n" + + "RETURN \n" + + " friend.id AS personId, \n" + + " friend.firstName AS personFirstName, \n" + + " friend.lastName AS personLastName, \n" + + " message.id AS commentOrPostId, \n" + + " message.content AS messageContent, \n" + + " message.imageFile AS messageImageFile, \n" + + " message.creationDate AS commentOrPostCreationDate\n" + + "ORDER BY \n" + + " commentOrPostCreationDate DESC, \n" + + " commentOrPostId ASC \n" + + "LIMIT 20", builder) .build(); RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); @@ -610,25 +651,30 @@ public void ldbc9_test() { + " dir0=[DESC], dir1=[ASC], fetch=[20])\n" + " GraphLogicalProject(personId=[friend.id]," + " personFirstName=[friend.firstName], personLastName=[friend.lastName]," - + " commentOrPostId=[message.id]," + + " commentOrPostId=[message.id], messageContent=[message.content]," + + " messageImageFile=[message.imageFile]," + " commentOrPostCreationDate=[message.creationDate], isAppend=[false])\n" - + " LogicalFilter(condition=[<>(friend, person)])\n" + + " GraphLogicalProject(friend=[friend], message=[message]," + + " isAppend=[false])\n" + " GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST, COMMENT]}]," - + " alias=[message], fusedFilter=[[<(_.creationDate, 20130301000000000)]]," - + " opt=[START], physicalOpt=[ITSELF])\n" - + " GraphPhysicalExpand(tableConfig=[{isAll=false," - + " tables=[HASCREATOR]}], alias=[_], startAlias=[friend], opt=[IN]," - + " physicalOpt=[VERTEX])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," + + " alias=[message], fusedFilter=[[<(_.creationDate, ?1)]], opt=[START]," + + " physicalOpt=[ITSELF])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST," + + " PERSON), EdgeLabel(HASCREATOR, COMMENT, PERSON)]], alias=[_]," + + " startAlias=[friend], opt=[IN], physicalOpt=[VERTEX])\n" + + " LogicalFilter(condition=[<>(friend.id, ?0)])\n" + + " GraphLogicalAggregate(keys=[{variables=[friend]," + + " aliases=[friend]}], values=[[]])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," + " alias=[friend], opt=[END])\n" - + " " + + " " + " GraphLogicalPathExpand(fused=[GraphPhysicalExpand(tableConfig=[{isAll=false," + " tables=[KNOWS]}], alias=[_], opt=[BOTH], physicalOpt=[VERTEX])\n" + "], offset=[1], fetch=[2], path_opt=[ARBITRARY], result_opt=[END_V]," - + " alias=[_], start_alias=[person])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false," - + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," - + " 2199023382370)])", + + " alias=[_], start_alias=[p])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false," + + " tables=[PERSON]}], alias=[p], opt=[VERTEX], uniqueKeyFilters=[=(_.id," + + " ?0)])", after.explain().trim()); } @@ -638,99 +684,101 @@ public void ldbc10_test() { RelNode before = com.alibaba.graphscope.cypher.antlr4.Utils.eval( "MATCH (person:PERSON {id: $personId})-[:KNOWS*2..3]-(friend:" - + " PERSON)-[:ISLOCATEDIN]->(city:PLACE)\n" - + "OPTIONAL MATCH (friend :" - + " PERSON)<-[:HASCREATOR]-(post:POST)\n" - + "OPTIONAL MATCH" - + " (friend)<-[:HASCREATOR]-(post1:POST)-[:HASTAG]->(tag:TAG)<-[:HASINTEREST]-(person:" - + " PERSON)\n" - + "WHERE \n" - + " NOT friend=person \n" - + " AND NOT (friend:PERSON)-[:KNOWS]-(person :PERSON {id:" - + " $personId})\n" - + "WITH \n" - + " person, city, friend, post, post1, friend.birthday as" - + " birthday\n" - + "\n" - + "WHERE birthday > 2012 AND birthday < 2013\n" - + "WITH \n" - + " friend, \n" - + " city, \n" - + " count(distinct post) as postCount,\n" - + " count(distinct post1) as commonPostCount\n" - + "\n" - + "RETURN \n" - + " friend.id AS personId,\n" - + " friend.firstName AS personFirstName,\n" - + " friend.lastName AS personLastName,\n" - + " commonPostCount - (postCount - commonPostCount) AS" - + " commonInterestScore,\n" - + " friend.gender AS personGender,\n" - + " city.name AS personCityName\n" - + "ORDER BY commonInterestScore DESC, personId ASC\n" - + "LIMIT 10;", + + " PERSON)-[:ISLOCATEDIN]->(city:PLACE)\n" + + "WHERE \n" + + " NOT friend=person \n" + + " AND NOT (friend:PERSON)-[:KNOWS]-(person :PERSON" + + " {id: $personId})\n" + + "WITH \n" + + " person, \n" + + " city, \n" + + " friend, \n" + + " friend.birthday as birthday\n" + + "WITH DISTINCT friend, city, person\n" + + "\n" + + "OPTIONAL MATCH (friend :" + + " PERSON)<-[:HASCREATOR]-(post:POST)\n" + + "WITH friend, city, person, count(post) as postCount\n" + + "\n" + + "OPTIONAL MATCH" + + " (friend)<-[:HASCREATOR]-(post1:POST)-[:HASTAG]->(tag:TAG)<-[:HASINTEREST]-(person:" + + " PERSON {id: $personId})\n" + + "WITH friend, city, postCount, count(distinct post1) as" + + " commonPostCount\n" + + "\n" + + "RETURN friend.id AS personId,\n" + + " friend.firstName AS personFirstName,\n" + + " friend.lastName AS personLastName,\n" + + " commonPostCount - (postCount - commonPostCount) AS" + + " commonInterestScore,\n" + + " friend.gender AS personGender,\n" + + " city.name AS personCityName\n" + + "ORDER BY commonInterestScore DESC, personId ASC\n" + + "LIMIT 10;", builder) .build(); RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); Assert.assertEquals( - "root:\n" - + "GraphLogicalSort(sort0=[commonInterestScore], sort1=[personId], dir0=[DESC]," - + " dir1=[ASC], fetch=[10])\n" - + " GraphLogicalProject(personId=[friend.id]," - + " personFirstName=[friend.firstName], personLastName=[friend.lastName]," - + " commonInterestScore=[-(commonPostCount, -(postCount, commonPostCount))]," - + " personGender=[friend.gender], personCityName=[city.name]," - + " isAppend=[false])\n" - + " GraphLogicalAggregate(keys=[{variables=[friend, city], aliases=[friend," - + " city]}], values=[[{operands=[post], aggFunction=COUNT, alias='postCount'," - + " distinct=true}, {operands=[post1], aggFunction=COUNT," - + " alias='commonPostCount', distinct=true}]])\n" - + " LogicalFilter(condition=[AND(>(birthday, 2012), <(birthday, 2013))])\n" - + " GraphLogicalProject(person=[person], city=[city], friend=[friend]," - + " post=[post], post1=[post1], birthday=[friend.birthday], isAppend=[false])\n" - + " LogicalJoin(condition=[AND(=(person, person), =(friend, friend))]," - + " joinType=[anti])\n" - + " LogicalFilter(condition=[<>(friend, person)])\n" - + " GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}]," - + " alias=[post], opt=[START], physicalOpt=[ITSELF])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR," - + " POST, PERSON)]], alias=[_], startAlias=[friend], opt=[IN]," - + " physicalOpt=[VERTEX], optional=[true])\n" - + " MultiJoin(joinFilter=[=(tag, tag)]," - + " isFullOuterJoin=[false], joinTypes=[[INNER, INNER]]," - + " outerJoinConditions=[[NULL, NULL]], projFields=[[ALL, ALL]])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASTAG," - + " POST, TAG)]], alias=[tag], startAlias=[post1], opt=[OUT]," - + " physicalOpt=[VERTEX], optional=[true])\n" - + " CommonTableScan(table=[[common#-2135802270]])\n" - + " GraphPhysicalExpand(tableConfig=[{isAll=false," - + " tables=[HASINTEREST]}], alias=[tag], startAlias=[person], opt=[OUT]," - + " physicalOpt=[VERTEX], optional=[true])\n" - + " CommonTableScan(table=[[common#-2135802270]])\n" - + " GraphPhysicalExpand(tableConfig=[{isAll=false, tables=[KNOWS]}]," - + " alias=[friend], startAlias=[person], opt=[BOTH], physicalOpt=[VERTEX])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false," - + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," - + " ?0)])\n" - + "common#-2135802270:\n" - + "GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}], alias=[post1]," - + " opt=[START], physicalOpt=[ITSELF])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST, PERSON)]]," - + " alias=[_], startAlias=[friend], opt=[IN], physicalOpt=[VERTEX]," - + " optional=[true])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN, PERSON," - + " PLACE)]], alias=[city], startAlias=[friend], opt=[OUT]," - + " physicalOpt=[VERTEX])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," - + " alias=[friend], opt=[END])\n" - + " " - + " GraphLogicalPathExpand(fused=[GraphPhysicalExpand(tableConfig=[{isAll=false," - + " tables=[KNOWS]}], alias=[_], opt=[BOTH], physicalOpt=[VERTEX])\n" - + "], offset=[2], fetch=[1], path_opt=[ARBITRARY], result_opt=[END_V]," - + " alias=[_], start_alias=[person])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}]," - + " alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])", - com.alibaba.graphscope.common.ir.tools.Utils.toString(after).trim()); + "GraphLogicalSort(sort0=[commonInterestScore], sort1=[personId], dir0=[DESC]," + + " dir1=[ASC], fetch=[10])\n" + + " GraphLogicalProject(personId=[friend.id]," + + " personFirstName=[friend.firstName], personLastName=[friend.lastName]," + + " commonInterestScore=[-(commonPostCount, -(postCount, commonPostCount))]," + + " personGender=[friend.gender], personCityName=[city.name]," + + " isAppend=[false])\n" + + " GraphLogicalAggregate(keys=[{variables=[friend, city, postCount]," + + " aliases=[friend, city, postCount]}], values=[[{operands=[post1]," + + " aggFunction=COUNT, alias='commonPostCount', distinct=true}]])\n" + + " LogicalJoin(condition=[AND(=(friend, friend), =(person, person))]," + + " joinType=[left])\n" + + " GraphLogicalAggregate(keys=[{variables=[friend, city, person]," + + " aliases=[friend, city, person]}], values=[[{operands=[post]," + + " aggFunction=COUNT, alias='postCount', distinct=false}]])\n" + + " GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}]," + + " alias=[post], opt=[START], physicalOpt=[ITSELF])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST," + + " PERSON)]], alias=[_], startAlias=[friend], opt=[IN], physicalOpt=[VERTEX]," + + " optional=[true])\n" + + " GraphLogicalAggregate(keys=[{variables=[friend, city, person]," + + " aliases=[friend, city, person]}], values=[[]])\n" + + " GraphLogicalProject(person=[person], city=[city]," + + " friend=[friend], birthday=[friend.birthday], isAppend=[false])\n" + + " LogicalJoin(condition=[AND(=(person, person), =(friend," + + " friend))], joinType=[anti])\n" + + " LogicalFilter(condition=[<>(friend, person)])\n" + + " " + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN, PERSON, PLACE)]]," + + " alias=[city], startAlias=[friend], opt=[OUT], physicalOpt=[VERTEX])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=false," + + " tables=[PERSON]}], alias=[friend], opt=[END])\n" + + " " + + " GraphLogicalPathExpand(fused=[GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[KNOWS]}], alias=[_], opt=[BOTH], physicalOpt=[VERTEX])\n" + + "], offset=[2], fetch=[1], path_opt=[ARBITRARY], result_opt=[END_V]," + + " alias=[_], start_alias=[person])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false," + + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," + + " ?0)])\n" + + " GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[KNOWS]}], alias=[friend], startAlias=[person], opt=[BOTH]," + + " physicalOpt=[VERTEX])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false," + + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," + + " ?0)])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST," + + " PERSON)]], alias=[friend], startAlias=[post1], opt=[OUT]," + + " physicalOpt=[VERTEX])\n" + + " GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}]," + + " alias=[post1], opt=[START], physicalOpt=[ITSELF])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASTAG, POST," + + " TAG)]], alias=[_], startAlias=[tag], opt=[IN], physicalOpt=[VERTEX])\n" + + " GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[HASINTEREST]}], alias=[tag], startAlias=[person], opt=[OUT]," + + " physicalOpt=[VERTEX])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false," + + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," + + " ?0)])", + after.explain().trim()); } @Test @@ -855,4 +903,91 @@ public void ldbc12_test() { + " uniqueKeyFilters=[=(_.id, 2199023382370)])", after.explain().trim()); } + + @Test + public void ldbc14_test() { + GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); + RelNode before = + com.alibaba.graphscope.cypher.antlr4.Utils.eval( + "MATCH all ShortestPath((person1:PERSON { id: $person1Id" + + " })-[path:KNOWS*0..10]-(person2:PERSON { id: $person2Id" + + " }))\n" + + "WITH path, gs.function.relationships(path) as rels_in_path," + + " gs.function.nodes(path) as nodes_in_path\n" + + "UNWIND rels_in_path as rel\n" + + "WITH path, rels_in_path, nodes_in_path," + + " gs.function.startNode(rel) as rel0," + + " gs.function.endNode(rel) as rel1\n" + + "OPTIONAL MATCH" + + " (rel0:PERSON)<-[:HASCREATOR]-(n)-[:REPLYOF]-(m)-[:HASCREATOR]->(rel1:PERSON)\n" + + "With path, nodes_in_path, rels_in_path,\n" + + " CASE WHEN labels(m) <> labels(n) THEN 1 ELSE 0 END as" + + " ra,\n" + + " CASE WHEN labels(m) = labels(n) THEN 1 ELSE 0 END as" + + " rb\n" + + "With path, nodes_in_path, rels_in_path, SUM(ra) AS" + + " weight1Count, SUM(rb) as weight2Count\n" + + "UNWIND nodes_in_path as node\n" + + "WITH path, COLLECT(node.id) as personIdsInPath," + + " weight1Count, weight2Count\n" + + "RETURN personIdsInPath, (weight1Count +" + + " gs.function.toFloat(weight2Count) / 2) AS pathWeight\n" + + "ORDER BY pathWeight DESC;", + builder) + .build(); + RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); + Assert.assertEquals( + "root:\n" + + "GraphLogicalSort(sort0=[pathWeight], dir0=[DESC])\n" + + " GraphLogicalProject(personIdsInPath=[personIdsInPath]," + + " pathWeight=[+(weight1Count, /(gs.function.toFloat(weight2Count), 2))]," + + " isAppend=[false])\n" + + " GraphLogicalAggregate(keys=[{variables=[path, weight1Count," + + " weight2Count], aliases=[path, weight1Count, weight2Count]}]," + + " values=[[{operands=[node.id], aggFunction=COLLECT, alias='personIdsInPath'," + + " distinct=false}]])\n" + + " GraphLogicalUnfold(key=[nodes_in_path], alias=[node])\n" + + " GraphLogicalAggregate(keys=[{variables=[path, nodes_in_path," + + " rels_in_path], aliases=[path, nodes_in_path, rels_in_path]}]," + + " values=[[{operands=[ra], aggFunction=SUM, alias='weight1Count'," + + " distinct=false}, {operands=[rb], aggFunction=SUM, alias='weight2Count'," + + " distinct=false}]])\n" + + " GraphLogicalProject(path=[path], nodes_in_path=[nodes_in_path]," + + " rels_in_path=[rels_in_path], ra=[CASE(<>(m.~label, n.~label), 1, 0)]," + + " rb=[CASE(=(m.~label, n.~label), 1, 0)], isAppend=[false])\n" + + " MultiJoin(joinFilter=[=(m, m)], isFullOuterJoin=[false]," + + " joinTypes=[[INNER, INNER]], outerJoinConditions=[[NULL, NULL]]," + + " projFields=[[ALL, ALL]])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(REPLYOF, COMMENT," + + " POST), EdgeLabel(REPLYOF, COMMENT, COMMENT)]], alias=[m], opt=[BOTH]," + + " physicalOpt=[VERTEX], optional=[true])\n" + + " GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[HASCREATOR]}], alias=[n], startAlias=[rel0], opt=[IN]," + + " physicalOpt=[VERTEX], optional=[true])\n" + + " CommonTableScan(table=[[common#-1230129050]])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST," + + " PERSON), EdgeLabel(HASCREATOR, COMMENT, PERSON)]], alias=[m]," + + " startAlias=[rel1], opt=[IN], physicalOpt=[VERTEX], optional=[true])\n" + + " CommonTableScan(table=[[common#-1230129050]])\n" + + "common#-1230129050:\n" + + "GraphLogicalProject(path=[path], rels_in_path=[rels_in_path]," + + " nodes_in_path=[nodes_in_path], rel0=[gs.function.startNode(rel)]," + + " rel1=[gs.function.endNode(rel)], isAppend=[false])\n" + + " GraphLogicalUnfold(key=[rels_in_path], alias=[rel])\n" + + " GraphLogicalProject(path=[path]," + + " rels_in_path=[gs.function.relationships(path)]," + + " nodes_in_path=[gs.function.nodes(path)], isAppend=[false])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," + + " alias=[person2], fusedFilter=[[=(_.id, ?1)]], opt=[END])\n" + + " " + + " GraphLogicalPathExpand(expand=[GraphLogicalExpand(tableConfig=[{isAll=false," + + " tables=[KNOWS]}], alias=[_], opt=[BOTH])\n" + + "], getV=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," + + " alias=[_], opt=[OTHER])\n" + + "], fetch=[10], path_opt=[ALL_SHORTEST], result_opt=[ALL_V_E], alias=[path]," + + " start_alias=[person1])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}]," + + " alias=[person1], opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])", + com.alibaba.graphscope.common.ir.tools.Utils.toString(after).trim()); + } } From b05b29f697bd2859644ac7c068dc02e2aeab9f59 Mon Sep 17 00:00:00 2001 From: shirly121 Date: Thu, 17 Oct 2024 20:18:37 +0800 Subject: [PATCH 3/3] support features for ic benchmark --- .../compiler/src/main/antlr4/CypherGS.g4 | 10 +- .../common/ir/planner/GraphRelOptimizer.java | 20 + .../planner/rules/FlatJoinToExpandRule.java | 1 + .../ir/rel/graph/GraphLogicalExpand.java | 1 + .../common/ir/rel/graph/GraphLogicalGetV.java | 1 + .../ir/rel/graph/GraphLogicalSource.java | 1 + .../antlr4/visitor/CallSubQueryVisitor.java | 1 + .../antlr4/visitor/GraphBuilderVisitor.java | 3 +- .../common/ir/planner/cbo/LdbcTest.java | 548 ++++++++---------- .../src/test/resources/logback-test.xml | 29 + 10 files changed, 303 insertions(+), 312 deletions(-) create mode 100644 interactive_engine/compiler/src/test/resources/logback-test.xml diff --git a/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 b/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 index f041655c01b0..0f527ba58d93 100644 --- a/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 +++ b/interactive_engine/compiler/src/main/antlr4/CypherGS.g4 @@ -52,7 +52,15 @@ CALL : ( 'C' | 'c' ) ( 'A' | 'a' ) ( 'L' | 'l' ) ( 'L' | 'l' ) ; YIELD : ( 'Y' | 'y' ) ( 'I' | 'i' ) ( 'E' | 'e' ) ( 'L' | 'l' ) ( 'D' | 'd' ) ; oC_RegularQuery - : oC_Match ( SP? ( oC_Match | oC_With | oC_Unwind | oC_UnionCallSubQuery ) )* ( SP oC_Return ) ; + : ( oC_ReadingClause SP? )* SP? oC_ReadingClause ( SP oC_Return ) + ; + +oC_ReadingClause + : oC_Match + | oC_Unwind + | oC_With + | oC_UnionCallSubQuery + ; oC_SubQuery : ( ( oC_Match | oC_With | oC_Unwind ) SP? )* ( SP? oC_Return ) ; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphRelOptimizer.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphRelOptimizer.java index ae3039bd132f..739eef95538a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphRelOptimizer.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/GraphRelOptimizer.java @@ -21,6 +21,8 @@ import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.glogue.calcite.GraphRelMetadataQuery; import com.alibaba.graphscope.common.ir.meta.glogue.calcite.handler.GraphMetadataHandlerProvider; +import com.alibaba.graphscope.common.ir.meta.schema.CommonOptTable; +import com.alibaba.graphscope.common.ir.rel.CommonTableScan; import com.alibaba.graphscope.common.ir.rel.GraphShuttle; import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalSource; import com.alibaba.graphscope.common.ir.rel.graph.match.AbstractLogicalMatch; @@ -32,6 +34,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.plan.RelOptPlanner; @@ -45,6 +48,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import java.util.List; +import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -100,10 +104,26 @@ public RelNode optimize(RelNode before, GraphIOProcessor ioProcessor) { public static class MatchOptimizer extends GraphShuttle { private final GraphIOProcessor ioProcessor; private final RelOptPlanner matchPlanner; + // record the common rel(s) which has been optimized + private final Map commonTableToOpt; public MatchOptimizer(GraphIOProcessor ioProcessor, RelOptPlanner matchPlanner) { this.ioProcessor = ioProcessor; this.matchPlanner = matchPlanner; + this.commonTableToOpt = Maps.newHashMap(); + } + + @Override + public RelNode visit(CommonTableScan tableScan) { + CommonOptTable optTable = (CommonOptTable) tableScan.getTable(); + String tableName = optTable.getQualifiedName().get(0); + RelNode commonOpt = commonTableToOpt.get(tableName); + if (commonOpt == null) { + commonOpt = optTable.getCommon().accept(this); + commonTableToOpt.put(tableName, commonOpt); + } + return new CommonTableScan( + tableScan.getCluster(), tableScan.getTraitSet(), new CommonOptTable(commonOpt)); } @Override diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinToExpandRule.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinToExpandRule.java index 193d3d86cb0f..7c3527cbeddf 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinToExpandRule.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/planner/rules/FlatJoinToExpandRule.java @@ -24,6 +24,7 @@ import com.alibaba.graphscope.common.ir.rel.type.AliasNameWithId; import com.alibaba.graphscope.common.ir.rex.RexGraphVariable; import com.google.common.collect.Lists; + import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.core.JoinRelType; import org.apache.calcite.rel.logical.LogicalFilter; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java index 4b10f028779b..d3ced787374a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalExpand.java @@ -23,6 +23,7 @@ import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; import com.google.common.collect.ImmutableList; + import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java index cbebf7388fb0..52d6035a9d9d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalGetV.java @@ -22,6 +22,7 @@ import com.alibaba.graphscope.common.ir.rel.type.TableConfig; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.google.common.collect.ImmutableList; + import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.plan.RelTraitSet; import org.apache.calcite.rel.RelNode; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java index 70c6a3d9f8bc..3c1d41bc5f79 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/graph/GraphLogicalSource.java @@ -20,6 +20,7 @@ import com.alibaba.graphscope.common.ir.rel.type.TableConfig; import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.google.common.collect.ImmutableList; + import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.RelShuttle; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/CallSubQueryVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/CallSubQueryVisitor.java index b627a9858e2b..83f9dc89a2d7 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/CallSubQueryVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/CallSubQueryVisitor.java @@ -23,6 +23,7 @@ import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.grammar.CypherGSBaseVisitor; import com.alibaba.graphscope.grammar.CypherGSParser; + import org.apache.calcite.plan.GraphOptCluster; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.RelNode; diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java index cc22c7f8de53..59f82c4ea4ae 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/antlr4/visitor/GraphBuilderVisitor.java @@ -64,8 +64,7 @@ public GraphBuilder visitOC_UnionCallSubQuery(CypherGSParser.OC_UnionCallSubQuer for (int i = 0; i < ctx.oC_CallSubQuery().size(); ++i) { CypherGSParser.OC_CallSubQueryContext callSubQuery = ctx.oC_CallSubQuery(i); if (callSubQuery != null) { - branches.add( - new CallSubQueryVisitor(this.builder).visit(callSubQuery)); + branches.add(new CallSubQueryVisitor(this.builder).visit(callSubQuery)); } } Preconditions.checkArgument( diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java index 8f4c380f8ca7..d94969f4c6e6 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/planner/cbo/LdbcTest.java @@ -28,8 +28,9 @@ public static void beforeClass() { "graph.planner.opt", "CBO", "graph.planner.rules", - "NotMatchToAntiJoinRule, FilterIntoJoinRule, FilterMatchRule, FlatJoinToExpandRule," - + " ExtendIntersectRule, ExpandGetVFusionRule")); + "NotMatchToAntiJoinRule, FilterIntoJoinRule, FilterMatchRule," + + " FlatJoinToExpandRule, ExtendIntersectRule," + + " ExpandGetVFusionRule")); optimizer = new GraphRelOptimizer(configs); irMeta = Utils.mockIrMeta( @@ -43,92 +44,108 @@ public void ldbc1_test() { GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); RelNode before = com.alibaba.graphscope.cypher.antlr4.Utils.eval( - "MATCH shortestPath(p: PERSON {id : $personId})" - + " -[k:KNOWS*1..4]-(f:PERSON {firstName : $firstName})\n" - + "where f <> p\n" - + "WITH f, length(k) as distance\n" + "MATCH (p: PERSON{id: $personId}) -[k:KNOWS*1..4]-(f: PERSON" + + " {firstName: $firstName})\n" + + "OPTIONAL MATCH (f:" + + " PERSON)-[workAt:WORKAT]->(company:ORGANISATION)-[:ISLOCATEDIN]->(country:PLACE)\n" + + "OPTIONAL MATCH (f:" + + " PERSON)-[studyAt:STUDYAT]->(university)-[:ISLOCATEDIN]->(universityCity:PLACE)\n" + + "MATCH (f:PERSON)-[:ISLOCATEDIN]->(locationCity:PLACE)\n" + + "WHERE p <> f\n" + + "with f AS f, company, university, workAt, country, studyAt," + + " universityCity, locationCity, length(k) as len\n" + + "with f AS f, company, university, workAt, country, studyAt," + + " universityCity, locationCity, min(len) as distance\n" + "ORDER BY distance ASC, f.lastName ASC, f.id ASC\n" + "LIMIT 20\n" + "\n" - + "\n" - + "OPTIONAL MATCH (f:" - + " PERSON)-[workAt:WORKAT]->(company:ORGANISATION)-[:ISLOCATEDIN]->(country:PLACE)\n" + "WITH \n" - + " f, distance,\n" + + " f, distance, locationCity,\n" + " CASE\n" + " WHEN company is null Then null\n" + " ELSE [company.name, workAt.workFrom, country.name]\n" - + " END as companies\n" - + "WITH f, collect(companies) as company_info, distance\n" - + "\n" - + "OPTIONAL MATCH (f:" - + " PERSON)-[studyAt:STUDYAT]->(university)-[:ISLOCATEDIN]->(universityCity:PLACE)\n" - + "WITH\n" - + " f, company_info, distance,\n" - + "\tCASE \n" - + "\t\tWHEN university is null Then null\n" - + "\t\tELSE [university.name, studyAt.classYear," + + " END as companies,\n" + + " CASE \n" + + "\t\t WHEN university is null Then null\n" + + "\t\t ELSE [university.name, studyAt.classYear," + " universityCity.name]\n" - + "\tEND as universities\n" - + "WITH f, collect(universities) as university_info ," - + " company_info, distance\n" + + "\t END as universities\n" + + "WITH f, distance, locationCity, collect(companies) as" + + " company_info, collect(universities) as university_info\n" + "\n" - + "MATCH (f:PERSON)-[:ISLOCATEDIN]->(locationCity:PLACE)\n" - + "\n" - + "return f.id AS friendId;", + + "return f.id AS friendId,\n" + + " f.lastName AS friendLastName,\n" + + " distance AS distanceFromPerson,\n" + + " f.birthday AS friendBirthday,\n" + + " f.creationDate AS friendCreationDate,\n" + + " f.gender AS friendGender,\n" + + " f.browserUsed AS friendBrowserUsed,\n" + + " f.locationIP AS friendLocationIp,\n" + + " locationCity.name AS friendCityName,\n" + + " university_info AS friendUniversities,\n" + + " company_info AS friendCompanies;", builder) .build(); RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); Assert.assertEquals( - "GraphLogicalProject(friendId=[f.id], isAppend=[false])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN, PERSON, PLACE)]]," - + " alias=[locationCity], startAlias=[f], opt=[OUT], physicalOpt=[VERTEX])\n" - + " GraphLogicalAggregate(keys=[{variables=[f, company_info, distance]," - + " aliases=[f, company_info, distance]}], values=[[{operands=[universities]," - + " aggFunction=COLLECT, alias='university_info', distinct=false}]])\n" - + " GraphLogicalProject(f=[f], company_info=[company_info]," - + " distance=[distance], universities=[CASE(IS NULL(university), null:NULL," - + " ARRAY(university.name, studyAt.classYear, universityCity.name))]," - + " isAppend=[false])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN," + "GraphLogicalProject(friendId=[f.id], friendLastName=[f.lastName]," + + " distanceFromPerson=[distance], friendBirthday=[f.birthday]," + + " friendCreationDate=[f.creationDate], friendGender=[f.gender]," + + " friendBrowserUsed=[f.browserUsed], friendLocationIp=[f.locationIP]," + + " friendCityName=[locationCity.name], friendUniversities=[university_info]," + + " friendCompanies=[company_info], isAppend=[false])\n" + + " GraphLogicalAggregate(keys=[{variables=[f, distance, locationCity]," + + " aliases=[f, distance, locationCity]}], values=[[{operands=[companies]," + + " aggFunction=COLLECT, alias='company_info', distinct=false}," + + " {operands=[universities], aggFunction=COLLECT, alias='university_info'," + + " distinct=false}]])\n" + + " GraphLogicalProject(f=[f], distance=[distance]," + + " locationCity=[locationCity], companies=[CASE(IS NULL(company), null:NULL," + + " ARRAY(company.name, workAt.workFrom, country.name))], universities=[CASE(IS" + + " NULL(university), null:NULL, ARRAY(university.name, studyAt.classYear," + + " universityCity.name))], isAppend=[false])\n" + + " GraphLogicalSort(sort0=[distance], sort1=[f.lastName], sort2=[f.id]," + + " dir0=[ASC], dir1=[ASC], dir2=[ASC], fetch=[20])\n" + + " GraphLogicalAggregate(keys=[{variables=[f, company, university," + + " workAt, country, studyAt, universityCity, locationCity], aliases=[f," + + " company, university, workAt, country, studyAt, universityCity," + + " locationCity]}], values=[[{operands=[len], aggFunction=MIN," + + " alias='distance', distinct=false}]])\n" + + " GraphLogicalProject(f=[f], company=[company]," + + " university=[university], workAt=[workAt], country=[country]," + + " studyAt=[studyAt], universityCity=[universityCity]," + + " locationCity=[locationCity], len=[k.~len], isAppend=[false])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN, PERSON," + + " PLACE)]], alias=[locationCity], startAlias=[f], opt=[OUT]," + + " physicalOpt=[VERTEX])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN," + " ORGANISATION, PLACE)]], alias=[universityCity], opt=[OUT]," + " physicalOpt=[VERTEX], optional=[true])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false," + + " GraphLogicalGetV(tableConfig=[{isAll=false," + " tables=[ORGANISATION]}], alias=[university], opt=[END])\n" - + " GraphLogicalExpand(tableConfig=[{isAll=false," + + " GraphLogicalExpand(tableConfig=[{isAll=false," + " tables=[STUDYAT]}], alias=[studyAt], startAlias=[f], opt=[OUT]," + " optional=[true])\n" - + " GraphLogicalAggregate(keys=[{variables=[f, distance]," - + " aliases=[f, distance]}], values=[[{operands=[companies]," - + " aggFunction=COLLECT, alias='company_info', distinct=false}]])\n" - + " GraphLogicalProject(f=[f], distance=[distance]," - + " companies=[CASE(IS NULL(company), null:NULL, ARRAY(company.name," - + " workAt.workFrom, country.name))], isAppend=[false])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN," + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN," + " ORGANISATION, PLACE)]], alias=[country], opt=[OUT], physicalOpt=[VERTEX]," + " optional=[true])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false," + + " GraphLogicalGetV(tableConfig=[{isAll=false," + " tables=[ORGANISATION]}], alias=[company], opt=[END])\n" - + " GraphLogicalExpand(tableConfig=[{isAll=false," + + " GraphLogicalExpand(tableConfig=[{isAll=false," + " tables=[WORKAT]}], alias=[workAt], startAlias=[f], opt=[OUT]," + " optional=[true])\n" - + " GraphLogicalSort(sort0=[distance]," - + " sort1=[f.lastName], sort2=[f.id], dir0=[ASC], dir1=[ASC], dir2=[ASC]," - + " fetch=[20])\n" - + " GraphLogicalProject(f=[f], distance=[k.~len]," - + " isAppend=[false])\n" - + " LogicalFilter(condition=[<>(f, p)])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false," + + " LogicalFilter(condition=[<>(p, f)])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=false," + " tables=[PERSON]}], alias=[f], fusedFilter=[[=(_.firstName, ?1)]]," + " opt=[END])\n" - + " " + + " " + " GraphLogicalPathExpand(expand=[GraphLogicalExpand(tableConfig=[{isAll=false," + " tables=[KNOWS]}], alias=[_], opt=[BOTH])\n" + "], getV=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," + " alias=[_], opt=[OTHER])\n" - + "], offset=[1], fetch=[3], path_opt=[ANY_SHORTEST], result_opt=[ALL_V_E]," + + "], offset=[1], fetch=[3], path_opt=[ARBITRARY], result_opt=[ALL_V_E]," + " alias=[k], start_alias=[p])\n" - + " " + + " " + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}], alias=[p]," + " opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])", after.explain().trim()); @@ -340,86 +357,86 @@ public void ldbc5_test() { RelNode before = com.alibaba.graphscope.cypher.antlr4.Utils.eval( "MATCH (person:PERSON { id: $personId })-[:KNOWS*1..3]-(friend)\n" - + "WITH DISTINCT friend\n" - + "WHERE friend.id <> $personId\n" - + "MATCH (friend)<-[membership:HASMEMBER]-(forum)\n" - + "WHERE membership.joinDate > $minDate\n" - + "CALL {\n" - + " WITH forum\n" - + " RETURN forum, 0 AS postCount\n" - + " ORDER BY forum.id ASC\n" - + " LIMIT 20\n" - + "}\n" - + "UNION\n" - + "CALL {\n" - + " WITH friend, collect(forum) AS forums\n" - + " MATCH" - + " (friend)<-[:HASCREATOR]-(post)<-[:CONTAINEROF]-(forum)\n" - + " WHERE forum IN forums\n" - + " WITH forum, count(post) AS postCount\n" - + " RETURN forum, postCount\n" - + " ORDER BY postCount DESC, forum.id ASC\n" - + " LIMIT 20\n" - + "}\n" - + "WITH forum, max(postCount) AS postCount\n" - + "RETURN forum, postCount\n" - + "ORDER BY postCount DESC, forum.id ASC\n" - + "LIMIT 20;", + + "WITH DISTINCT friend\n" + + "WHERE friend.id <> $personId\n" + + "MATCH (friend)<-[membership:HASMEMBER]-(forum)\n" + + "WHERE membership.joinDate > $minDate\n" + + "CALL {\n" + + " WITH forum\n" + + " RETURN forum, 0 AS postCount\n" + + " ORDER BY forum.id ASC\n" + + " LIMIT 20\n" + + "}\n" + + "UNION\n" + + "CALL {\n" + + " WITH friend, collect(forum) AS forums\n" + + " MATCH" + + " (friend)<-[:HASCREATOR]-(post)<-[:CONTAINEROF]-(forum)\n" + + " WHERE forum IN forums\n" + + " WITH forum, count(post) AS postCount\n" + + " RETURN forum, postCount\n" + + " ORDER BY postCount DESC, forum.id ASC\n" + + " LIMIT 20\n" + + "}\n" + + "WITH forum, max(postCount) AS postCount\n" + + "RETURN forum, postCount\n" + + "ORDER BY postCount DESC, forum.id ASC\n" + + "LIMIT 20;", builder) .build(); RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); Assert.assertEquals( "root:\n" - + "GraphLogicalSort(sort0=[postCount], sort1=[forum.id], dir0=[DESC]," - + " dir1=[ASC], fetch=[20])\n" - + " GraphLogicalProject(forum=[forum], postCount=[postCount]," - + " isAppend=[false])\n" - + " GraphLogicalAggregate(keys=[{variables=[forum], aliases=[forum]}]," - + " values=[[{operands=[postCount], aggFunction=MAX, alias='postCount'," - + " distinct=false}]])\n" - + " LogicalUnion(all=[true])\n" - + " GraphLogicalSort(sort0=[forum.id], dir0=[ASC], fetch=[20])\n" - + " GraphLogicalProject(forum=[forum], postCount=[0]," - + " isAppend=[false])\n" - + " GraphLogicalProject(forum=[forum], isAppend=[false])\n" - + " CommonTableScan(table=[[common#1874145243]])\n" - + " GraphLogicalSort(sort0=[postCount], sort1=[forum.id], dir0=[DESC]," - + " dir1=[ASC], fetch=[20])\n" - + " GraphLogicalProject(forum=[forum], postCount=[postCount]," - + " isAppend=[false])\n" - + " GraphLogicalAggregate(keys=[{variables=[forum]," - + " aliases=[forum]}], values=[[{operands=[post], aggFunction=COUNT," - + " alias='postCount', distinct=false}]])\n" - + " LogicalFilter(condition=[IN(forum, forums)])\n" - + " GraphPhysicalExpand(tableConfig=[{isAll=false," - + " tables=[CONTAINEROF]}], alias=[forum], opt=[IN], physicalOpt=[VERTEX])\n" - + " GraphPhysicalGetV(tableConfig=[{isAll=false," - + " tables=[POST]}], alias=[post], opt=[START], physicalOpt=[ITSELF])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR," - + " POST, PERSON)]], alias=[_], startAlias=[friend], opt=[IN]," - + " physicalOpt=[VERTEX])\n" - + " GraphLogicalAggregate(keys=[{variables=[friend]," - + " aliases=[friend]}], values=[[{operands=[forum], aggFunction=COLLECT," - + " alias='forums', distinct=false}]])\n" - + " CommonTableScan(table=[[common#1874145243]])\n" - + "common#1874145243:\n" - + "GraphLogicalGetV(tableConfig=[{isAll=false, tables=[FORUM]}], alias=[forum]," - + " opt=[START])\n" - + " GraphLogicalExpand(tableConfig=[{isAll=false, tables=[HASMEMBER]}]," - + " alias=[membership], startAlias=[friend], fusedFilter=[[>(_.joinDate, ?1)]]," - + " opt=[IN])\n" - + " LogicalFilter(condition=[<>(friend.id, ?0)])\n" - + " GraphLogicalAggregate(keys=[{variables=[friend], aliases=[friend]}]," - + " values=[[]])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," - + " alias=[friend], opt=[END])\n" - + " " - + " GraphLogicalPathExpand(fused=[GraphPhysicalExpand(tableConfig=[{isAll=false," - + " tables=[KNOWS]}], alias=[_], opt=[BOTH], physicalOpt=[VERTEX])\n" - + "], offset=[1], fetch=[2], path_opt=[ARBITRARY], result_opt=[END_V]," - + " alias=[_], start_alias=[person])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}]," - + " alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])", + + "GraphLogicalSort(sort0=[postCount], sort1=[forum.id], dir0=[DESC]," + + " dir1=[ASC], fetch=[20])\n" + + " GraphLogicalProject(forum=[forum], postCount=[postCount]," + + " isAppend=[false])\n" + + " GraphLogicalAggregate(keys=[{variables=[forum], aliases=[forum]}]," + + " values=[[{operands=[postCount], aggFunction=MAX, alias='postCount'," + + " distinct=false}]])\n" + + " LogicalUnion(all=[true])\n" + + " GraphLogicalSort(sort0=[forum.id], dir0=[ASC], fetch=[20])\n" + + " GraphLogicalProject(forum=[forum], postCount=[0]," + + " isAppend=[false])\n" + + " GraphLogicalProject(forum=[forum], isAppend=[false])\n" + + " CommonTableScan(table=[[common#1874145243]])\n" + + " GraphLogicalSort(sort0=[postCount], sort1=[forum.id], dir0=[DESC]," + + " dir1=[ASC], fetch=[20])\n" + + " GraphLogicalProject(forum=[forum], postCount=[postCount]," + + " isAppend=[false])\n" + + " GraphLogicalAggregate(keys=[{variables=[forum]," + + " aliases=[forum]}], values=[[{operands=[post], aggFunction=COUNT," + + " alias='postCount', distinct=false}]])\n" + + " LogicalFilter(condition=[IN(forum, forums)])\n" + + " GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[CONTAINEROF]}], alias=[forum], opt=[IN], physicalOpt=[VERTEX])\n" + + " GraphPhysicalGetV(tableConfig=[{isAll=false," + + " tables=[POST]}], alias=[post], opt=[START], physicalOpt=[ITSELF])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR," + + " POST, PERSON)]], alias=[_], startAlias=[friend], opt=[IN]," + + " physicalOpt=[VERTEX])\n" + + " GraphLogicalAggregate(keys=[{variables=[friend]," + + " aliases=[friend]}], values=[[{operands=[forum], aggFunction=COLLECT," + + " alias='forums', distinct=false}]])\n" + + " CommonTableScan(table=[[common#1874145243]])\n" + + "common#1874145243:\n" + + "GraphLogicalGetV(tableConfig=[{isAll=false, tables=[FORUM]}], alias=[forum]," + + " opt=[START])\n" + + " GraphLogicalExpand(tableConfig=[{isAll=false, tables=[HASMEMBER]}]," + + " alias=[membership], startAlias=[friend], fusedFilter=[[>(_.joinDate, ?1)]]," + + " opt=[IN])\n" + + " LogicalFilter(condition=[<>(friend.id, ?0)])\n" + + " GraphLogicalAggregate(keys=[{variables=[friend], aliases=[friend]}]," + + " values=[[]])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," + + " alias=[friend], opt=[END])\n" + + " " + + " GraphLogicalPathExpand(fused=[GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[KNOWS]}], alias=[_], opt=[BOTH], physicalOpt=[VERTEX])\n" + + "], offset=[1], fetch=[2], path_opt=[ARBITRARY], result_opt=[END_V]," + + " alias=[_], start_alias=[person])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}]," + + " alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])", com.alibaba.graphscope.common.ir.tools.Utils.toString(after).trim()); } @@ -684,100 +701,100 @@ public void ldbc10_test() { RelNode before = com.alibaba.graphscope.cypher.antlr4.Utils.eval( "MATCH (person:PERSON {id: $personId})-[:KNOWS*2..3]-(friend:" - + " PERSON)-[:ISLOCATEDIN]->(city:PLACE)\n" - + "WHERE \n" - + " NOT friend=person \n" - + " AND NOT (friend:PERSON)-[:KNOWS]-(person :PERSON" - + " {id: $personId})\n" - + "WITH \n" - + " person, \n" - + " city, \n" - + " friend, \n" - + " friend.birthday as birthday\n" - + "WITH DISTINCT friend, city, person\n" - + "\n" - + "OPTIONAL MATCH (friend :" - + " PERSON)<-[:HASCREATOR]-(post:POST)\n" - + "WITH friend, city, person, count(post) as postCount\n" - + "\n" - + "OPTIONAL MATCH" - + " (friend)<-[:HASCREATOR]-(post1:POST)-[:HASTAG]->(tag:TAG)<-[:HASINTEREST]-(person:" - + " PERSON {id: $personId})\n" - + "WITH friend, city, postCount, count(distinct post1) as" - + " commonPostCount\n" - + "\n" - + "RETURN friend.id AS personId,\n" - + " friend.firstName AS personFirstName,\n" - + " friend.lastName AS personLastName,\n" - + " commonPostCount - (postCount - commonPostCount) AS" - + " commonInterestScore,\n" - + " friend.gender AS personGender,\n" - + " city.name AS personCityName\n" - + "ORDER BY commonInterestScore DESC, personId ASC\n" - + "LIMIT 10;", + + " PERSON)-[:ISLOCATEDIN]->(city:PLACE)\n" + + "WHERE \n" + + " NOT friend=person \n" + + " AND NOT (friend:PERSON)-[:KNOWS]-(person :PERSON" + + " {id: $personId})\n" + + "WITH \n" + + " person, \n" + + " city, \n" + + " friend, \n" + + " friend.birthday as birthday\n" + + "WITH DISTINCT friend, city, person\n" + + "\n" + + "OPTIONAL MATCH (friend :" + + " PERSON)<-[:HASCREATOR]-(post:POST)\n" + + "WITH friend, city, person, count(post) as postCount\n" + + "\n" + + "OPTIONAL MATCH" + + " (friend)<-[:HASCREATOR]-(post1:POST)-[:HASTAG]->(tag:TAG)<-[:HASINTEREST]-(person:" + + " PERSON {id: $personId})\n" + + "WITH friend, city, postCount, count(distinct post1) as" + + " commonPostCount\n" + + "\n" + + "RETURN friend.id AS personId,\n" + + " friend.firstName AS personFirstName,\n" + + " friend.lastName AS personLastName,\n" + + " commonPostCount - (postCount - commonPostCount) AS" + + " commonInterestScore,\n" + + " friend.gender AS personGender,\n" + + " city.name AS personCityName\n" + + "ORDER BY commonInterestScore DESC, personId ASC\n" + + "LIMIT 10;", builder) .build(); RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); Assert.assertEquals( "GraphLogicalSort(sort0=[commonInterestScore], sort1=[personId], dir0=[DESC]," - + " dir1=[ASC], fetch=[10])\n" - + " GraphLogicalProject(personId=[friend.id]," - + " personFirstName=[friend.firstName], personLastName=[friend.lastName]," - + " commonInterestScore=[-(commonPostCount, -(postCount, commonPostCount))]," - + " personGender=[friend.gender], personCityName=[city.name]," - + " isAppend=[false])\n" - + " GraphLogicalAggregate(keys=[{variables=[friend, city, postCount]," - + " aliases=[friend, city, postCount]}], values=[[{operands=[post1]," - + " aggFunction=COUNT, alias='commonPostCount', distinct=true}]])\n" - + " LogicalJoin(condition=[AND(=(friend, friend), =(person, person))]," - + " joinType=[left])\n" - + " GraphLogicalAggregate(keys=[{variables=[friend, city, person]," - + " aliases=[friend, city, person]}], values=[[{operands=[post]," - + " aggFunction=COUNT, alias='postCount', distinct=false}]])\n" - + " GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}]," - + " alias=[post], opt=[START], physicalOpt=[ITSELF])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST," - + " PERSON)]], alias=[_], startAlias=[friend], opt=[IN], physicalOpt=[VERTEX]," - + " optional=[true])\n" - + " GraphLogicalAggregate(keys=[{variables=[friend, city, person]," - + " aliases=[friend, city, person]}], values=[[]])\n" - + " GraphLogicalProject(person=[person], city=[city]," - + " friend=[friend], birthday=[friend.birthday], isAppend=[false])\n" - + " LogicalJoin(condition=[AND(=(person, person), =(friend," - + " friend))], joinType=[anti])\n" - + " LogicalFilter(condition=[<>(friend, person)])\n" - + " " - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN, PERSON, PLACE)]]," - + " alias=[city], startAlias=[friend], opt=[OUT], physicalOpt=[VERTEX])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false," - + " tables=[PERSON]}], alias=[friend], opt=[END])\n" - + " " - + " GraphLogicalPathExpand(fused=[GraphPhysicalExpand(tableConfig=[{isAll=false," - + " tables=[KNOWS]}], alias=[_], opt=[BOTH], physicalOpt=[VERTEX])\n" - + "], offset=[2], fetch=[1], path_opt=[ARBITRARY], result_opt=[END_V]," - + " alias=[_], start_alias=[person])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false," - + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," - + " ?0)])\n" - + " GraphPhysicalExpand(tableConfig=[{isAll=false," - + " tables=[KNOWS]}], alias=[friend], startAlias=[person], opt=[BOTH]," - + " physicalOpt=[VERTEX])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false," - + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," - + " ?0)])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST," - + " PERSON)]], alias=[friend], startAlias=[post1], opt=[OUT]," - + " physicalOpt=[VERTEX])\n" - + " GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}]," - + " alias=[post1], opt=[START], physicalOpt=[ITSELF])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASTAG, POST," - + " TAG)]], alias=[_], startAlias=[tag], opt=[IN], physicalOpt=[VERTEX])\n" - + " GraphPhysicalExpand(tableConfig=[{isAll=false," - + " tables=[HASINTEREST]}], alias=[tag], startAlias=[person], opt=[OUT]," - + " physicalOpt=[VERTEX])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false," - + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," - + " ?0)])", + + " dir1=[ASC], fetch=[10])\n" + + " GraphLogicalProject(personId=[friend.id]," + + " personFirstName=[friend.firstName], personLastName=[friend.lastName]," + + " commonInterestScore=[-(commonPostCount, -(postCount, commonPostCount))]," + + " personGender=[friend.gender], personCityName=[city.name]," + + " isAppend=[false])\n" + + " GraphLogicalAggregate(keys=[{variables=[friend, city, postCount]," + + " aliases=[friend, city, postCount]}], values=[[{operands=[post1]," + + " aggFunction=COUNT, alias='commonPostCount', distinct=true}]])\n" + + " LogicalJoin(condition=[AND(=(friend, friend), =(person, person))]," + + " joinType=[left])\n" + + " GraphLogicalAggregate(keys=[{variables=[friend, city, person]," + + " aliases=[friend, city, person]}], values=[[{operands=[post]," + + " aggFunction=COUNT, alias='postCount', distinct=false}]])\n" + + " GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}]," + + " alias=[post], opt=[START], physicalOpt=[ITSELF])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST," + + " PERSON)]], alias=[_], startAlias=[friend], opt=[IN], physicalOpt=[VERTEX]," + + " optional=[true])\n" + + " GraphLogicalAggregate(keys=[{variables=[friend, city, person]," + + " aliases=[friend, city, person]}], values=[[]])\n" + + " GraphLogicalProject(person=[person], city=[city]," + + " friend=[friend], birthday=[friend.birthday], isAppend=[false])\n" + + " LogicalJoin(condition=[AND(=(person, person), =(friend," + + " friend))], joinType=[anti])\n" + + " LogicalFilter(condition=[<>(friend, person)])\n" + + " " + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(ISLOCATEDIN, PERSON, PLACE)]]," + + " alias=[city], startAlias=[friend], opt=[OUT], physicalOpt=[VERTEX])\n" + + " GraphLogicalGetV(tableConfig=[{isAll=false," + + " tables=[PERSON]}], alias=[friend], opt=[END])\n" + + " " + + " GraphLogicalPathExpand(fused=[GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[KNOWS]}], alias=[_], opt=[BOTH], physicalOpt=[VERTEX])\n" + + "], offset=[2], fetch=[1], path_opt=[ARBITRARY], result_opt=[END_V]," + + " alias=[_], start_alias=[person])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false," + + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," + + " ?0)])\n" + + " GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[KNOWS]}], alias=[friend], startAlias=[person], opt=[BOTH]," + + " physicalOpt=[VERTEX])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false," + + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," + + " ?0)])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST," + + " PERSON)]], alias=[friend], startAlias=[post1], opt=[OUT]," + + " physicalOpt=[VERTEX])\n" + + " GraphPhysicalGetV(tableConfig=[{isAll=false, tables=[POST]}]," + + " alias=[post1], opt=[START], physicalOpt=[ITSELF])\n" + + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASTAG, POST," + + " TAG)]], alias=[_], startAlias=[tag], opt=[IN], physicalOpt=[VERTEX])\n" + + " GraphPhysicalExpand(tableConfig=[{isAll=false," + + " tables=[HASINTEREST]}], alias=[tag], startAlias=[person], opt=[OUT]," + + " physicalOpt=[VERTEX])\n" + + " GraphLogicalSource(tableConfig=[{isAll=false," + + " tables=[PERSON]}], alias=[person], opt=[VERTEX], uniqueKeyFilters=[=(_.id," + + " ?0)])", after.explain().trim()); } @@ -903,91 +920,4 @@ public void ldbc12_test() { + " uniqueKeyFilters=[=(_.id, 2199023382370)])", after.explain().trim()); } - - @Test - public void ldbc14_test() { - GraphBuilder builder = Utils.mockGraphBuilder(optimizer, irMeta); - RelNode before = - com.alibaba.graphscope.cypher.antlr4.Utils.eval( - "MATCH all ShortestPath((person1:PERSON { id: $person1Id" - + " })-[path:KNOWS*0..10]-(person2:PERSON { id: $person2Id" - + " }))\n" - + "WITH path, gs.function.relationships(path) as rels_in_path," - + " gs.function.nodes(path) as nodes_in_path\n" - + "UNWIND rels_in_path as rel\n" - + "WITH path, rels_in_path, nodes_in_path," - + " gs.function.startNode(rel) as rel0," - + " gs.function.endNode(rel) as rel1\n" - + "OPTIONAL MATCH" - + " (rel0:PERSON)<-[:HASCREATOR]-(n)-[:REPLYOF]-(m)-[:HASCREATOR]->(rel1:PERSON)\n" - + "With path, nodes_in_path, rels_in_path,\n" - + " CASE WHEN labels(m) <> labels(n) THEN 1 ELSE 0 END as" - + " ra,\n" - + " CASE WHEN labels(m) = labels(n) THEN 1 ELSE 0 END as" - + " rb\n" - + "With path, nodes_in_path, rels_in_path, SUM(ra) AS" - + " weight1Count, SUM(rb) as weight2Count\n" - + "UNWIND nodes_in_path as node\n" - + "WITH path, COLLECT(node.id) as personIdsInPath," - + " weight1Count, weight2Count\n" - + "RETURN personIdsInPath, (weight1Count +" - + " gs.function.toFloat(weight2Count) / 2) AS pathWeight\n" - + "ORDER BY pathWeight DESC;", - builder) - .build(); - RelNode after = optimizer.optimize(before, new GraphIOProcessor(builder, irMeta)); - Assert.assertEquals( - "root:\n" - + "GraphLogicalSort(sort0=[pathWeight], dir0=[DESC])\n" - + " GraphLogicalProject(personIdsInPath=[personIdsInPath]," - + " pathWeight=[+(weight1Count, /(gs.function.toFloat(weight2Count), 2))]," - + " isAppend=[false])\n" - + " GraphLogicalAggregate(keys=[{variables=[path, weight1Count," - + " weight2Count], aliases=[path, weight1Count, weight2Count]}]," - + " values=[[{operands=[node.id], aggFunction=COLLECT, alias='personIdsInPath'," - + " distinct=false}]])\n" - + " GraphLogicalUnfold(key=[nodes_in_path], alias=[node])\n" - + " GraphLogicalAggregate(keys=[{variables=[path, nodes_in_path," - + " rels_in_path], aliases=[path, nodes_in_path, rels_in_path]}]," - + " values=[[{operands=[ra], aggFunction=SUM, alias='weight1Count'," - + " distinct=false}, {operands=[rb], aggFunction=SUM, alias='weight2Count'," - + " distinct=false}]])\n" - + " GraphLogicalProject(path=[path], nodes_in_path=[nodes_in_path]," - + " rels_in_path=[rels_in_path], ra=[CASE(<>(m.~label, n.~label), 1, 0)]," - + " rb=[CASE(=(m.~label, n.~label), 1, 0)], isAppend=[false])\n" - + " MultiJoin(joinFilter=[=(m, m)], isFullOuterJoin=[false]," - + " joinTypes=[[INNER, INNER]], outerJoinConditions=[[NULL, NULL]]," - + " projFields=[[ALL, ALL]])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(REPLYOF, COMMENT," - + " POST), EdgeLabel(REPLYOF, COMMENT, COMMENT)]], alias=[m], opt=[BOTH]," - + " physicalOpt=[VERTEX], optional=[true])\n" - + " GraphPhysicalExpand(tableConfig=[{isAll=false," - + " tables=[HASCREATOR]}], alias=[n], startAlias=[rel0], opt=[IN]," - + " physicalOpt=[VERTEX], optional=[true])\n" - + " CommonTableScan(table=[[common#-1230129050]])\n" - + " GraphPhysicalExpand(tableConfig=[[EdgeLabel(HASCREATOR, POST," - + " PERSON), EdgeLabel(HASCREATOR, COMMENT, PERSON)]], alias=[m]," - + " startAlias=[rel1], opt=[IN], physicalOpt=[VERTEX], optional=[true])\n" - + " CommonTableScan(table=[[common#-1230129050]])\n" - + "common#-1230129050:\n" - + "GraphLogicalProject(path=[path], rels_in_path=[rels_in_path]," - + " nodes_in_path=[nodes_in_path], rel0=[gs.function.startNode(rel)]," - + " rel1=[gs.function.endNode(rel)], isAppend=[false])\n" - + " GraphLogicalUnfold(key=[rels_in_path], alias=[rel])\n" - + " GraphLogicalProject(path=[path]," - + " rels_in_path=[gs.function.relationships(path)]," - + " nodes_in_path=[gs.function.nodes(path)], isAppend=[false])\n" - + " GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," - + " alias=[person2], fusedFilter=[[=(_.id, ?1)]], opt=[END])\n" - + " " - + " GraphLogicalPathExpand(expand=[GraphLogicalExpand(tableConfig=[{isAll=false," - + " tables=[KNOWS]}], alias=[_], opt=[BOTH])\n" - + "], getV=[GraphLogicalGetV(tableConfig=[{isAll=false, tables=[PERSON]}]," - + " alias=[_], opt=[OTHER])\n" - + "], fetch=[10], path_opt=[ALL_SHORTEST], result_opt=[ALL_V_E], alias=[path]," - + " start_alias=[person1])\n" - + " GraphLogicalSource(tableConfig=[{isAll=false, tables=[PERSON]}]," - + " alias=[person1], opt=[VERTEX], uniqueKeyFilters=[=(_.id, ?0)])", - com.alibaba.graphscope.common.ir.tools.Utils.toString(after).trim()); - } } diff --git a/interactive_engine/compiler/src/test/resources/logback-test.xml b/interactive_engine/compiler/src/test/resources/logback-test.xml new file mode 100644 index 000000000000..6aa06705038f --- /dev/null +++ b/interactive_engine/compiler/src/test/resources/logback-test.xml @@ -0,0 +1,29 @@ + + + + + 　　　　　 + 　　　　　　　　　[%d{ISO8601}][%p][%t][%c:%L] %m%n + 　　　　　 + 　　　 + + + + + \ No newline at end of file