use of com.alibaba.maxgraph.compiler.logical.chain.LogicalChainSourceVertex in project GraphScope by alibaba.
the class LogicalQueryPlan method checkVertexChain.
private boolean checkVertexChain(LogicalVertex sourceVertex, LogicalVertex targetVertex) {
ProcessorFunction targetFunction = targetVertex.getProcessorFunction();
QueryFlowOuterClass.OperatorType targetOperatorType = targetFunction.getOperatorType();
if (sourceVertex instanceof LogicalSourceVertex) {
if ((targetOperatorType == QueryFlowOuterClass.OperatorType.OUT || targetOperatorType == QueryFlowOuterClass.OperatorType.OUT_E) && targetVertex.getAfterRequirementList().isEmpty() && targetVertex.getBeforeRequirementList().isEmpty() && targetVertex.getProcessorFunction().getRangeLimit() == null && sourceVertex.getAfterRequirementList().isEmpty() && sourceVertex.getBeforeRequirementList().isEmpty() && sourceVertex.getProcessorFunction().getRangeLimit() == null && ((ProcessorSourceFunction) sourceVertex.getProcessorFunction()).getOdpsQueryInput() == null) {
return true;
}
} else if (sourceVertex instanceof LogicalChainSourceVertex) {
if ((targetOperatorType == QueryFlowOuterClass.OperatorType.IN_V || targetOperatorType == QueryFlowOuterClass.OperatorType.OUT_V || targetOperatorType == QueryFlowOuterClass.OperatorType.BOTH_V || targetOperatorType == QueryFlowOuterClass.OperatorType.OTHER_V) && targetVertex.getBeforeRequirementList().isEmpty() && targetVertex.getAfterRequirementList().isEmpty() && targetVertex.getProcessorFunction().getRangeLimit() == null) {
return true;
}
}
return false;
}
use of com.alibaba.maxgraph.compiler.logical.chain.LogicalChainSourceVertex in project GraphScope by alibaba.
the class LogicalQueryPlan method chainOptimize.
/**
* Chain the operator, here we chain the g.V().outE().inV() source operator first
*
* @return The chained logical query plan
*/
public LogicalQueryPlan chainOptimize() {
if (this.isPullGraphEnable()) {
List<LogicalVertex> orderVertexList = PlanUtils.getOrderVertexList(plan);
boolean outVertexFlag = false;
for (LogicalVertex logicalVertex : orderVertexList) {
if (logicalVertex.getProcessorFunction() != null) {
QueryFlowOuterClass.OperatorType operatorType = logicalVertex.getProcessorFunction().getOperatorType();
if (QueryFlowOuterClass.OperatorType.OUT == operatorType || QueryFlowOuterClass.OperatorType.OUT_E == operatorType || QueryFlowOuterClass.OperatorType.IN == operatorType || QueryFlowOuterClass.OperatorType.IN_E == operatorType || QueryFlowOuterClass.OperatorType.BOTH == operatorType || QueryFlowOuterClass.OperatorType.BOTH_E == operatorType) {
if (!outVertexFlag && (QueryFlowOuterClass.OperatorType.OUT == operatorType || QueryFlowOuterClass.OperatorType.OUT_E == operatorType)) {
outVertexFlag = true;
continue;
}
logicalVertex.getProcessorFunction().getArgumentBuilder().setExecLocalDisable(true);
break;
}
}
}
}
boolean chainFlag = true;
while (chainFlag) {
chainFlag = false;
List<LogicalVertex> orderVertexList = PlanUtils.getOrderVertexList(plan);
for (LogicalVertex logicalVertex : orderVertexList) {
List<LogicalVertex> targetVertexList = PlanUtils.getTargetVertexList(plan, logicalVertex);
if (targetVertexList.size() == 1) {
LogicalVertex targetVertex = targetVertexList.get(0);
if (checkVertexChain(logicalVertex, targetVertex)) {
LogicalChainSourceVertex logicalChainSourceVertex;
if (logicalVertex instanceof LogicalSourceVertex) {
logicalChainSourceVertex = new LogicalChainSourceVertex((LogicalSourceVertex) logicalVertex);
removeVertex(logicalVertex);
addLogicalVertex(logicalChainSourceVertex);
logicalChainSourceVertex.addLogicalVertex(targetVertex);
List<Pair<LogicalEdge, LogicalVertex>> targetEdgeVertexList = PlanUtils.getTargetEdgeVertexList(plan, targetVertex);
removeVertex(targetVertex);
for (Pair<LogicalEdge, LogicalVertex> targetPair : targetEdgeVertexList) {
addLogicalEdge(logicalChainSourceVertex, targetPair.getRight(), targetPair.getLeft());
targetPair.getRight().resetInputVertex(targetVertex, logicalChainSourceVertex);
}
} else if (logicalVertex instanceof LogicalChainSourceVertex) {
logicalChainSourceVertex = LogicalChainSourceVertex.class.cast(logicalVertex);
logicalChainSourceVertex.addLogicalVertex(targetVertex);
removeVertex(targetVertex);
} else {
break;
}
chainFlag = true;
break;
}
} else {
break;
}
}
}
return this;
}
use of com.alibaba.maxgraph.compiler.logical.chain.LogicalChainSourceVertex in project GraphScope by alibaba.
the class QueryFlowBuilder method buildQueryFlow.
private void buildQueryFlow(QueryPlan.Builder queryPlanBuilder, LogicalQueryPlan queryPlan, long snapshot) {
List<LogicalVertex> logicalVertexList = queryPlan.getLogicalVertexList();
Map<Integer, Pair<Integer, Integer>> dfsGraphVertexMapping = Maps.newHashMap();
for (LogicalVertex logicalVertex : logicalVertexList) {
ProcessorFunction processorFunction = logicalVertex.getProcessorFunction();
if (processorFunction instanceof ProcessorRepeatFunction) {
ProcessorRepeatFunction processorRepeatFunction = ProcessorRepeatFunction.class.cast(processorFunction);
QueryFlowBuilder repeatBuilder = new QueryFlowBuilder();
QueryFlow queryFlow = repeatBuilder.buildQueryFlow(processorRepeatFunction.getRepeatPlan(), snapshot).build();
RepeatArgumentProto.Builder repeatArgument = RepeatArgumentProto.newBuilder().setPlan(queryFlow.getQueryPlan()).setLoopLimit((int) processorRepeatFunction.getMaxLoopTimes()).setFeedbackId(processorRepeatFunction.getFeedbackVertex().getId()).setEmitFlag(processorRepeatFunction.isHasEmitFlag());
if (processorRepeatFunction.getLeaveVertex() != null) {
repeatArgument.setLeaveId(processorRepeatFunction.getLeaveVertex().getId());
} else {
repeatArgument.setLeaveId(-1);
}
logger.info("repeat argument => ");
logger.info(TextFormat.printToString(repeatArgument));
OperatorBase.Builder repeatBase = OperatorBase.newBuilder().setId(logicalVertex.getId()).setOperatorType(OperatorType.REPEAT).setArgument(Value.newBuilder().setPayload(repeatArgument.build().toByteString()));
queryPlanBuilder.addUnaryOp(UnaryOperator.newBuilder().setBase(repeatBase).setInputOperatorId(queryPlan.getSourceVertex(logicalVertex).getId()));
queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
} else if (processorFunction instanceof ProcessorChainFunction) {
OperatorBase.Builder baseBuilder = OperatorBase.newBuilder();
baseBuilder.setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId());
if (null != logicalVertex.getEarlyStopArgument()) {
baseBuilder.setEarlyStopArgument(logicalVertex.getEarlyStopArgument().build());
}
ProcessorChainFunction processorChainFunction = ProcessorChainFunction.class.cast(processorFunction);
processorChainFunction.getChainVertexList().forEach(v -> baseBuilder.addChainedFunction(createChainedFunction(v)));
if (logicalVertex instanceof LogicalChainSourceVertex) {
LogicalSourceVertex logicalSourceVertex = ((LogicalChainSourceVertex) logicalVertex).getLogicalSourceVertex();
baseBuilder.setArgument(logicalSourceVertex.getProcessorFunction().getArgumentBuilder()).addAllLogicalCompare(logicalSourceVertex.getProcessorFunction().getLogicalCompareList());
SourceOperator.Builder sourceBuilder = SourceOperator.newBuilder().setBase(baseBuilder);
queryPlanBuilder.setSourceOp(sourceBuilder);
} else {
LogicalChainUnaryVertex logicalUnaryVertex = (LogicalChainUnaryVertex) logicalVertex;
UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(queryPlan.getSourceVertex(logicalUnaryVertex).getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
LogicalEdge logicalEdge = queryPlan.getLogicalEdge(queryPlan.getSourceVertex(logicalUnaryVertex), logicalUnaryVertex);
unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
if (logicalEdge.getShuffleType() == EdgeShuffleType.SHUFFLE_BY_KEY) {
InputEdgeShuffle.Builder shuffleBuilder = InputEdgeShuffle.newBuilder().setShuffleType(InputShuffleType.SHUFFLE_BY_ID_TYPE);
if (logicalEdge.getShufflePropId() != 0) {
shuffleBuilder.setShuffleValue(logicalEdge.getShufflePropId());
}
unaryOperator.setInputShuffle(shuffleBuilder);
}
queryPlanBuilder.addUnaryOp(unaryOperator);
}
queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
} else if (processorFunction instanceof ProcessorLabelValueFunction) {
ProcessorLabelValueFunction processorLabelValueFunction = (ProcessorLabelValueFunction) processorFunction;
LogicalVertex labelValueVertex = processorLabelValueFunction.getLabelValueVertex();
OperatorBase.Builder labelOperatorBuilder = createChainedFunction(labelValueVertex);
int labelId = processorLabelValueFunction.getLabelId();
OperatorBase.Builder baseBuilder = OperatorBase.newBuilder().setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId()).setArgument(Value.newBuilder().setIntValue(labelId).setBoolValue(processorLabelValueFunction.getRequireLabelFlag()).setPayload(labelOperatorBuilder.build().toByteString()));
LogicalVertex inputVertex = queryPlan.getSourceVertex(logicalVertex);
UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(inputVertex.getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
LogicalEdge logicalEdge = queryPlan.getLogicalEdge(inputVertex, logicalVertex);
unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
queryPlanBuilder.addUnaryOp(unaryOperator).addOperatorIdList(logicalVertex.getId());
} else {
if (logicalVertex instanceof LogicalSourceDelegateVertex) {
continue;
}
OperatorBase.Builder baseBuilder = OperatorBase.newBuilder();
baseBuilder.setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId());
if (processorFunction.getArgumentBuilder() != null) {
baseBuilder.setArgument(processorFunction.getArgumentBuilder());
}
processorFunction.getLogicalCompareList().forEach(baseBuilder::addLogicalCompare);
if (processorFunction.getRangeLimit() != null) {
baseBuilder.setRangeLimit(processorFunction.getRangeLimit());
}
baseBuilder.addAllAfterRequirement(logicalVertex.getAfterRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
baseBuilder.addAllBeforeRequirement(logicalVertex.getBeforeRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
if (logicalVertex instanceof LogicalSourceVertex) {
ProcessorSourceFunction processorSourceFunction = (ProcessorSourceFunction) logicalVertex.getProcessorFunction();
SourceOperator.Builder sourceBuilder = SourceOperator.newBuilder().setBase(baseBuilder);
if (null != processorSourceFunction.getOdpsQueryInput()) {
sourceBuilder.setOdpsInput(processorSourceFunction.getOdpsQueryInput()).setSourceType(SourceType.ODPS);
} else {
sourceBuilder.setSourceType(SourceType.GRAPH);
}
queryPlanBuilder.setSourceOp(sourceBuilder);
} else if (logicalVertex instanceof LogicalUnaryVertex) {
EarlyStopArgument.Builder earlyStopArgument = logicalVertex.getEarlyStopArgument();
if (null != earlyStopArgument && (earlyStopArgument.getGlobalFilterFlag() || earlyStopArgument.getGlobalStopFlag())) {
baseBuilder.setEarlyStopArgument(logicalVertex.getEarlyStopArgument().build());
}
LogicalUnaryVertex logicalUnaryVertex = LogicalUnaryVertex.class.cast(logicalVertex);
UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(queryPlan.getSourceVertex(logicalUnaryVertex).getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
LogicalEdge logicalEdge = queryPlan.getLogicalEdge(queryPlan.getSourceVertex(logicalUnaryVertex), logicalUnaryVertex);
unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
queryPlanBuilder.addUnaryOp(unaryOperator);
if (processorFunction.getOperatorType() == OperatorType.DFS_REPEAT_GRAPH) {
VertexIdManager vertexIdManager = queryPlan.getPlanVertexIdManager();
int cmdLeftId = vertexIdManager.getId();
int dataRightId = vertexIdManager.getId();
dfsGraphVertexMapping.put(baseBuilder.getId(), Pair.of(cmdLeftId, dataRightId));
UnaryOperator.Builder cmdLeftOp = UnaryOperator.newBuilder().setBase(OperatorBase.newBuilder().setOperatorType(OperatorType.DFS_REPEAT_CMD).setId(cmdLeftId)).setInputOperatorId(baseBuilder.getId()).setShuffleType(InputShuffleType.FORWARD_TYPE);
queryPlanBuilder.addUnaryOp(cmdLeftOp);
UnaryOperator.Builder dataRightOp = UnaryOperator.newBuilder().setBase(OperatorBase.newBuilder().setOperatorType(OperatorType.DFS_REPEAT_DATA).setId(dataRightId)).setInputOperatorId(baseBuilder.getId()).setShuffleType(InputShuffleType.FORWARD_TYPE);
queryPlanBuilder.addUnaryOp(dataRightOp);
}
} else if (logicalVertex instanceof LogicalBinaryVertex) {
LogicalBinaryVertex logicalBinaryVertex = LogicalBinaryVertex.class.cast(logicalVertex);
BinaryOperator.Builder binaryOperator = BinaryOperator.newBuilder().setBase(baseBuilder).setLeftInputOperatorId(parseInputOperatorId(logicalBinaryVertex.getLeftInput().getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping)).setRightInputOperatorId(parseInputOperatorId(logicalBinaryVertex.getRightInput().getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
LogicalEdge leftEdge = queryPlan.getLogicalEdge(logicalBinaryVertex.getLeftInput(), logicalBinaryVertex);
LogicalEdge rightEdge = queryPlan.getLogicalEdge(logicalBinaryVertex.getRightInput(), logicalBinaryVertex);
binaryOperator.setLeftShuffleType(CompilerUtils.parseShuffleTypeFromEdge(leftEdge)).setLeftStreamIndex(leftEdge.getStreamIndex()).setRightShuffleType(CompilerUtils.parseShuffleTypeFromEdge(rightEdge)).setRightStreamIndex(rightEdge.getStreamIndex());
queryPlanBuilder.addBinaryOp(binaryOperator);
} else {
throw new IllegalArgumentException(logicalVertex.toString());
}
queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
Pair<Integer, Integer> dfsCmdPair = dfsGraphVertexMapping.get(logicalVertex.getId());
if (null != dfsCmdPair) {
queryPlanBuilder.addOperatorIdList(dfsCmdPair.getLeft());
queryPlanBuilder.addOperatorIdList(dfsCmdPair.getRight());
}
}
}
}
Aggregations