use of com.alibaba.maxgraph.compiler.logical.chain.ProcessorChainFunction in project GraphScope by alibaba.
the class QueryFlowBuilder method buildQueryFlow.
private void buildQueryFlow(QueryPlan.Builder queryPlanBuilder, LogicalQueryPlan queryPlan, long snapshot) {
List<LogicalVertex> logicalVertexList = queryPlan.getLogicalVertexList();
Map<Integer, Pair<Integer, Integer>> dfsGraphVertexMapping = Maps.newHashMap();
for (LogicalVertex logicalVertex : logicalVertexList) {
ProcessorFunction processorFunction = logicalVertex.getProcessorFunction();
if (processorFunction instanceof ProcessorRepeatFunction) {
ProcessorRepeatFunction processorRepeatFunction = ProcessorRepeatFunction.class.cast(processorFunction);
QueryFlowBuilder repeatBuilder = new QueryFlowBuilder();
QueryFlow queryFlow = repeatBuilder.buildQueryFlow(processorRepeatFunction.getRepeatPlan(), snapshot).build();
RepeatArgumentProto.Builder repeatArgument = RepeatArgumentProto.newBuilder().setPlan(queryFlow.getQueryPlan()).setLoopLimit((int) processorRepeatFunction.getMaxLoopTimes()).setFeedbackId(processorRepeatFunction.getFeedbackVertex().getId()).setEmitFlag(processorRepeatFunction.isHasEmitFlag());
if (processorRepeatFunction.getLeaveVertex() != null) {
repeatArgument.setLeaveId(processorRepeatFunction.getLeaveVertex().getId());
} else {
repeatArgument.setLeaveId(-1);
}
logger.info("repeat argument => ");
logger.info(TextFormat.printToString(repeatArgument));
OperatorBase.Builder repeatBase = OperatorBase.newBuilder().setId(logicalVertex.getId()).setOperatorType(OperatorType.REPEAT).setArgument(Value.newBuilder().setPayload(repeatArgument.build().toByteString()));
queryPlanBuilder.addUnaryOp(UnaryOperator.newBuilder().setBase(repeatBase).setInputOperatorId(queryPlan.getSourceVertex(logicalVertex).getId()));
queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
} else if (processorFunction instanceof ProcessorChainFunction) {
OperatorBase.Builder baseBuilder = OperatorBase.newBuilder();
baseBuilder.setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId());
if (null != logicalVertex.getEarlyStopArgument()) {
baseBuilder.setEarlyStopArgument(logicalVertex.getEarlyStopArgument().build());
}
ProcessorChainFunction processorChainFunction = ProcessorChainFunction.class.cast(processorFunction);
processorChainFunction.getChainVertexList().forEach(v -> baseBuilder.addChainedFunction(createChainedFunction(v)));
if (logicalVertex instanceof LogicalChainSourceVertex) {
LogicalSourceVertex logicalSourceVertex = ((LogicalChainSourceVertex) logicalVertex).getLogicalSourceVertex();
baseBuilder.setArgument(logicalSourceVertex.getProcessorFunction().getArgumentBuilder()).addAllLogicalCompare(logicalSourceVertex.getProcessorFunction().getLogicalCompareList());
SourceOperator.Builder sourceBuilder = SourceOperator.newBuilder().setBase(baseBuilder);
queryPlanBuilder.setSourceOp(sourceBuilder);
} else {
LogicalChainUnaryVertex logicalUnaryVertex = (LogicalChainUnaryVertex) logicalVertex;
UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(queryPlan.getSourceVertex(logicalUnaryVertex).getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
LogicalEdge logicalEdge = queryPlan.getLogicalEdge(queryPlan.getSourceVertex(logicalUnaryVertex), logicalUnaryVertex);
unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
if (logicalEdge.getShuffleType() == EdgeShuffleType.SHUFFLE_BY_KEY) {
InputEdgeShuffle.Builder shuffleBuilder = InputEdgeShuffle.newBuilder().setShuffleType(InputShuffleType.SHUFFLE_BY_ID_TYPE);
if (logicalEdge.getShufflePropId() != 0) {
shuffleBuilder.setShuffleValue(logicalEdge.getShufflePropId());
}
unaryOperator.setInputShuffle(shuffleBuilder);
}
queryPlanBuilder.addUnaryOp(unaryOperator);
}
queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
} else if (processorFunction instanceof ProcessorLabelValueFunction) {
ProcessorLabelValueFunction processorLabelValueFunction = (ProcessorLabelValueFunction) processorFunction;
LogicalVertex labelValueVertex = processorLabelValueFunction.getLabelValueVertex();
OperatorBase.Builder labelOperatorBuilder = createChainedFunction(labelValueVertex);
int labelId = processorLabelValueFunction.getLabelId();
OperatorBase.Builder baseBuilder = OperatorBase.newBuilder().setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId()).setArgument(Value.newBuilder().setIntValue(labelId).setBoolValue(processorLabelValueFunction.getRequireLabelFlag()).setPayload(labelOperatorBuilder.build().toByteString()));
LogicalVertex inputVertex = queryPlan.getSourceVertex(logicalVertex);
UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(inputVertex.getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
LogicalEdge logicalEdge = queryPlan.getLogicalEdge(inputVertex, logicalVertex);
unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
queryPlanBuilder.addUnaryOp(unaryOperator).addOperatorIdList(logicalVertex.getId());
} else {
if (logicalVertex instanceof LogicalSourceDelegateVertex) {
continue;
}
OperatorBase.Builder baseBuilder = OperatorBase.newBuilder();
baseBuilder.setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId());
if (processorFunction.getArgumentBuilder() != null) {
baseBuilder.setArgument(processorFunction.getArgumentBuilder());
}
processorFunction.getLogicalCompareList().forEach(baseBuilder::addLogicalCompare);
if (processorFunction.getRangeLimit() != null) {
baseBuilder.setRangeLimit(processorFunction.getRangeLimit());
}
baseBuilder.addAllAfterRequirement(logicalVertex.getAfterRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
baseBuilder.addAllBeforeRequirement(logicalVertex.getBeforeRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
if (logicalVertex instanceof LogicalSourceVertex) {
ProcessorSourceFunction processorSourceFunction = (ProcessorSourceFunction) logicalVertex.getProcessorFunction();
SourceOperator.Builder sourceBuilder = SourceOperator.newBuilder().setBase(baseBuilder);
if (null != processorSourceFunction.getOdpsQueryInput()) {
sourceBuilder.setOdpsInput(processorSourceFunction.getOdpsQueryInput()).setSourceType(SourceType.ODPS);
} else {
sourceBuilder.setSourceType(SourceType.GRAPH);
}
queryPlanBuilder.setSourceOp(sourceBuilder);
} else if (logicalVertex instanceof LogicalUnaryVertex) {
EarlyStopArgument.Builder earlyStopArgument = logicalVertex.getEarlyStopArgument();
if (null != earlyStopArgument && (earlyStopArgument.getGlobalFilterFlag() || earlyStopArgument.getGlobalStopFlag())) {
baseBuilder.setEarlyStopArgument(logicalVertex.getEarlyStopArgument().build());
}
LogicalUnaryVertex logicalUnaryVertex = LogicalUnaryVertex.class.cast(logicalVertex);
UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(queryPlan.getSourceVertex(logicalUnaryVertex).getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
LogicalEdge logicalEdge = queryPlan.getLogicalEdge(queryPlan.getSourceVertex(logicalUnaryVertex), logicalUnaryVertex);
unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
queryPlanBuilder.addUnaryOp(unaryOperator);
if (processorFunction.getOperatorType() == OperatorType.DFS_REPEAT_GRAPH) {
VertexIdManager vertexIdManager = queryPlan.getPlanVertexIdManager();
int cmdLeftId = vertexIdManager.getId();
int dataRightId = vertexIdManager.getId();
dfsGraphVertexMapping.put(baseBuilder.getId(), Pair.of(cmdLeftId, dataRightId));
UnaryOperator.Builder cmdLeftOp = UnaryOperator.newBuilder().setBase(OperatorBase.newBuilder().setOperatorType(OperatorType.DFS_REPEAT_CMD).setId(cmdLeftId)).setInputOperatorId(baseBuilder.getId()).setShuffleType(InputShuffleType.FORWARD_TYPE);
queryPlanBuilder.addUnaryOp(cmdLeftOp);
UnaryOperator.Builder dataRightOp = UnaryOperator.newBuilder().setBase(OperatorBase.newBuilder().setOperatorType(OperatorType.DFS_REPEAT_DATA).setId(dataRightId)).setInputOperatorId(baseBuilder.getId()).setShuffleType(InputShuffleType.FORWARD_TYPE);
queryPlanBuilder.addUnaryOp(dataRightOp);
}
} else if (logicalVertex instanceof LogicalBinaryVertex) {
LogicalBinaryVertex logicalBinaryVertex = LogicalBinaryVertex.class.cast(logicalVertex);
BinaryOperator.Builder binaryOperator = BinaryOperator.newBuilder().setBase(baseBuilder).setLeftInputOperatorId(parseInputOperatorId(logicalBinaryVertex.getLeftInput().getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping)).setRightInputOperatorId(parseInputOperatorId(logicalBinaryVertex.getRightInput().getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
LogicalEdge leftEdge = queryPlan.getLogicalEdge(logicalBinaryVertex.getLeftInput(), logicalBinaryVertex);
LogicalEdge rightEdge = queryPlan.getLogicalEdge(logicalBinaryVertex.getRightInput(), logicalBinaryVertex);
binaryOperator.setLeftShuffleType(CompilerUtils.parseShuffleTypeFromEdge(leftEdge)).setLeftStreamIndex(leftEdge.getStreamIndex()).setRightShuffleType(CompilerUtils.parseShuffleTypeFromEdge(rightEdge)).setRightStreamIndex(rightEdge.getStreamIndex());
queryPlanBuilder.addBinaryOp(binaryOperator);
} else {
throw new IllegalArgumentException(logicalVertex.toString());
}
queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
Pair<Integer, Integer> dfsCmdPair = dfsGraphVertexMapping.get(logicalVertex.getId());
if (null != dfsCmdPair) {
queryPlanBuilder.addOperatorIdList(dfsCmdPair.getLeft());
queryPlanBuilder.addOperatorIdList(dfsCmdPair.getRight());
}
}
}
}
Aggregations