Examples with LogicalSourceDelegateVertex com.alibaba.maxgraph.compiler.logical.LogicalSourceDelegateVertex used on opensource projects

Example 1 with LogicalSourceDelegateVertex

use of com.alibaba.maxgraph.compiler.logical.LogicalSourceDelegateVertex in project GraphScope by alibaba.

the class LambdaMapTreeNode method buildLogicalQueryPlan.

@Override
public LogicalSubQueryPlan buildLogicalQueryPlan(ContextManager contextManager) {
    if (this.mapFunction instanceof RangeSumFunction) {
        RangeSumFunction rangeSumFunction = RangeSumFunction.class.cast(this.mapFunction);
        int propId = SchemaUtils.getPropId(rangeSumFunction.getPropName(), schema);
        ProcessorFunction processorFunction = new ProcessorFunction(QueryFlowOuterClass.OperatorType.RANGE_SUM, Message.Value.newBuilder().setIntValue(propId).addIntValueList(rangeSumFunction.getStart()).addIntValueList(rangeSumFunction.getCount()));
        return parseSingleUnaryVertex(contextManager.getVertexIdManager(), contextManager.getTreeNodeLabelManager(), processorFunction, contextManager);
    } else if (this.mapFunction instanceof MapPropFillFunction) {
        MapPropFillFunction mapPropFillFunction = MapPropFillFunction.class.cast(this.mapFunction);
        setPropLocalFlag(true);
        List<String> propNameList = mapPropFillFunction.getPropNameList();
        List<Integer> propIdList = propNameList.stream().map(v -> CompilerUtils.getPropertyId(schema, v)).collect(Collectors.toList());
        LogicalVertex logicalVertex = getInputNode().getOutputVertex();
        if (logicalVertex.getProcessorFunction().getOperatorType() == QueryFlowOuterClass.OperatorType.PROP_FILL) {
            logicalVertex.getProcessorFunction().getArgumentBuilder().addAllIntValueList(propIdList);
            LogicalSubQueryPlan logicalSubQueryPlan = new LogicalSubQueryPlan(contextManager);
            logicalSubQueryPlan.addLogicalVertex(new LogicalSourceDelegateVertex(logicalVertex));
            return logicalSubQueryPlan;
        } else {
            ProcessorFunction processorFunction = new ProcessorFunction(QueryFlowOuterClass.OperatorType.PROP_FILL, Message.Value.newBuilder().addAllIntValueList(propIdList));
            return parseSingleUnaryVertex(contextManager.getVertexIdManager(), contextManager.getTreeNodeLabelManager(), processorFunction, contextManager);
        }
    } else {
        Message.Value.Builder argumentBuilder = Message.Value.newBuilder().setStrValue(this.lambdaIndex);
        ProcessorFunction processorFunction = new ProcessorFunction(QueryFlowOuterClass.OperatorType.LAMBDA_MAP, argumentBuilder);
        return parseSingleUnaryVertex(contextManager.getVertexIdManager(), contextManager.getTreeNodeLabelManager(), processorFunction, contextManager);
    }
}

Example 2 with LogicalSourceDelegateVertex

use of com.alibaba.maxgraph.compiler.logical.LogicalSourceDelegateVertex in project GraphScope by alibaba.

the class QueryFlowBuilder method buildQueryFlow.

private void buildQueryFlow(QueryPlan.Builder queryPlanBuilder, LogicalQueryPlan queryPlan, long snapshot) {
    List<LogicalVertex> logicalVertexList = queryPlan.getLogicalVertexList();
    Map<Integer, Pair<Integer, Integer>> dfsGraphVertexMapping = Maps.newHashMap();
    for (LogicalVertex logicalVertex : logicalVertexList) {
        ProcessorFunction processorFunction = logicalVertex.getProcessorFunction();
        if (processorFunction instanceof ProcessorRepeatFunction) {
            ProcessorRepeatFunction processorRepeatFunction = ProcessorRepeatFunction.class.cast(processorFunction);
            QueryFlowBuilder repeatBuilder = new QueryFlowBuilder();
            QueryFlow queryFlow = repeatBuilder.buildQueryFlow(processorRepeatFunction.getRepeatPlan(), snapshot).build();
            RepeatArgumentProto.Builder repeatArgument = RepeatArgumentProto.newBuilder().setPlan(queryFlow.getQueryPlan()).setLoopLimit((int) processorRepeatFunction.getMaxLoopTimes()).setFeedbackId(processorRepeatFunction.getFeedbackVertex().getId()).setEmitFlag(processorRepeatFunction.isHasEmitFlag());
            if (processorRepeatFunction.getLeaveVertex() != null) {
                repeatArgument.setLeaveId(processorRepeatFunction.getLeaveVertex().getId());
            } else {
                repeatArgument.setLeaveId(-1);
            }
            logger.info("repeat argument => ");
            logger.info(TextFormat.printToString(repeatArgument));
            OperatorBase.Builder repeatBase = OperatorBase.newBuilder().setId(logicalVertex.getId()).setOperatorType(OperatorType.REPEAT).setArgument(Value.newBuilder().setPayload(repeatArgument.build().toByteString()));
            queryPlanBuilder.addUnaryOp(UnaryOperator.newBuilder().setBase(repeatBase).setInputOperatorId(queryPlan.getSourceVertex(logicalVertex).getId()));
            queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
        } else if (processorFunction instanceof ProcessorChainFunction) {
            OperatorBase.Builder baseBuilder = OperatorBase.newBuilder();
            baseBuilder.setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId());
            if (null != logicalVertex.getEarlyStopArgument()) {
                baseBuilder.setEarlyStopArgument(logicalVertex.getEarlyStopArgument().build());
            }
            ProcessorChainFunction processorChainFunction = ProcessorChainFunction.class.cast(processorFunction);
            processorChainFunction.getChainVertexList().forEach(v -> baseBuilder.addChainedFunction(createChainedFunction(v)));
            if (logicalVertex instanceof LogicalChainSourceVertex) {
                LogicalSourceVertex logicalSourceVertex = ((LogicalChainSourceVertex) logicalVertex).getLogicalSourceVertex();
                baseBuilder.setArgument(logicalSourceVertex.getProcessorFunction().getArgumentBuilder()).addAllLogicalCompare(logicalSourceVertex.getProcessorFunction().getLogicalCompareList());
                SourceOperator.Builder sourceBuilder = SourceOperator.newBuilder().setBase(baseBuilder);
                queryPlanBuilder.setSourceOp(sourceBuilder);
            } else {
                LogicalChainUnaryVertex logicalUnaryVertex = (LogicalChainUnaryVertex) logicalVertex;
                UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(queryPlan.getSourceVertex(logicalUnaryVertex).getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
                LogicalEdge logicalEdge = queryPlan.getLogicalEdge(queryPlan.getSourceVertex(logicalUnaryVertex), logicalUnaryVertex);
                unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
                if (logicalEdge.getShuffleType() == EdgeShuffleType.SHUFFLE_BY_KEY) {
                    InputEdgeShuffle.Builder shuffleBuilder = InputEdgeShuffle.newBuilder().setShuffleType(InputShuffleType.SHUFFLE_BY_ID_TYPE);
                    if (logicalEdge.getShufflePropId() != 0) {
                        shuffleBuilder.setShuffleValue(logicalEdge.getShufflePropId());
                    }
                    unaryOperator.setInputShuffle(shuffleBuilder);
                }
                queryPlanBuilder.addUnaryOp(unaryOperator);
            }
            queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
        } else if (processorFunction instanceof ProcessorLabelValueFunction) {
            ProcessorLabelValueFunction processorLabelValueFunction = (ProcessorLabelValueFunction) processorFunction;
            LogicalVertex labelValueVertex = processorLabelValueFunction.getLabelValueVertex();
            OperatorBase.Builder labelOperatorBuilder = createChainedFunction(labelValueVertex);
            int labelId = processorLabelValueFunction.getLabelId();
            OperatorBase.Builder baseBuilder = OperatorBase.newBuilder().setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId()).setArgument(Value.newBuilder().setIntValue(labelId).setBoolValue(processorLabelValueFunction.getRequireLabelFlag()).setPayload(labelOperatorBuilder.build().toByteString()));
            LogicalVertex inputVertex = queryPlan.getSourceVertex(logicalVertex);
            UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(inputVertex.getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
            LogicalEdge logicalEdge = queryPlan.getLogicalEdge(inputVertex, logicalVertex);
            unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
            queryPlanBuilder.addUnaryOp(unaryOperator).addOperatorIdList(logicalVertex.getId());
        } else {
            if (logicalVertex instanceof LogicalSourceDelegateVertex) {
                continue;
            }
            OperatorBase.Builder baseBuilder = OperatorBase.newBuilder();
            baseBuilder.setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId());
            if (processorFunction.getArgumentBuilder() != null) {
                baseBuilder.setArgument(processorFunction.getArgumentBuilder());
            }
            processorFunction.getLogicalCompareList().forEach(baseBuilder::addLogicalCompare);
            if (processorFunction.getRangeLimit() != null) {
                baseBuilder.setRangeLimit(processorFunction.getRangeLimit());
            }
            baseBuilder.addAllAfterRequirement(logicalVertex.getAfterRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
            baseBuilder.addAllBeforeRequirement(logicalVertex.getBeforeRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
            if (logicalVertex instanceof LogicalSourceVertex) {
                ProcessorSourceFunction processorSourceFunction = (ProcessorSourceFunction) logicalVertex.getProcessorFunction();
                SourceOperator.Builder sourceBuilder = SourceOperator.newBuilder().setBase(baseBuilder);
                if (null != processorSourceFunction.getOdpsQueryInput()) {
                    sourceBuilder.setOdpsInput(processorSourceFunction.getOdpsQueryInput()).setSourceType(SourceType.ODPS);
                } else {
                    sourceBuilder.setSourceType(SourceType.GRAPH);
                }
                queryPlanBuilder.setSourceOp(sourceBuilder);
            } else if (logicalVertex instanceof LogicalUnaryVertex) {
                EarlyStopArgument.Builder earlyStopArgument = logicalVertex.getEarlyStopArgument();
                if (null != earlyStopArgument && (earlyStopArgument.getGlobalFilterFlag() || earlyStopArgument.getGlobalStopFlag())) {
                    baseBuilder.setEarlyStopArgument(logicalVertex.getEarlyStopArgument().build());
                }
                LogicalUnaryVertex logicalUnaryVertex = LogicalUnaryVertex.class.cast(logicalVertex);
                UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(queryPlan.getSourceVertex(logicalUnaryVertex).getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
                LogicalEdge logicalEdge = queryPlan.getLogicalEdge(queryPlan.getSourceVertex(logicalUnaryVertex), logicalUnaryVertex);
                unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
                queryPlanBuilder.addUnaryOp(unaryOperator);
                if (processorFunction.getOperatorType() == OperatorType.DFS_REPEAT_GRAPH) {
                    VertexIdManager vertexIdManager = queryPlan.getPlanVertexIdManager();
                    int cmdLeftId = vertexIdManager.getId();
                    int dataRightId = vertexIdManager.getId();
                    dfsGraphVertexMapping.put(baseBuilder.getId(), Pair.of(cmdLeftId, dataRightId));
                    UnaryOperator.Builder cmdLeftOp = UnaryOperator.newBuilder().setBase(OperatorBase.newBuilder().setOperatorType(OperatorType.DFS_REPEAT_CMD).setId(cmdLeftId)).setInputOperatorId(baseBuilder.getId()).setShuffleType(InputShuffleType.FORWARD_TYPE);
                    queryPlanBuilder.addUnaryOp(cmdLeftOp);
                    UnaryOperator.Builder dataRightOp = UnaryOperator.newBuilder().setBase(OperatorBase.newBuilder().setOperatorType(OperatorType.DFS_REPEAT_DATA).setId(dataRightId)).setInputOperatorId(baseBuilder.getId()).setShuffleType(InputShuffleType.FORWARD_TYPE);
                    queryPlanBuilder.addUnaryOp(dataRightOp);
                }
            } else if (logicalVertex instanceof LogicalBinaryVertex) {
                LogicalBinaryVertex logicalBinaryVertex = LogicalBinaryVertex.class.cast(logicalVertex);
                BinaryOperator.Builder binaryOperator = BinaryOperator.newBuilder().setBase(baseBuilder).setLeftInputOperatorId(parseInputOperatorId(logicalBinaryVertex.getLeftInput().getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping)).setRightInputOperatorId(parseInputOperatorId(logicalBinaryVertex.getRightInput().getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
                LogicalEdge leftEdge = queryPlan.getLogicalEdge(logicalBinaryVertex.getLeftInput(), logicalBinaryVertex);
                LogicalEdge rightEdge = queryPlan.getLogicalEdge(logicalBinaryVertex.getRightInput(), logicalBinaryVertex);
                binaryOperator.setLeftShuffleType(CompilerUtils.parseShuffleTypeFromEdge(leftEdge)).setLeftStreamIndex(leftEdge.getStreamIndex()).setRightShuffleType(CompilerUtils.parseShuffleTypeFromEdge(rightEdge)).setRightStreamIndex(rightEdge.getStreamIndex());
                queryPlanBuilder.addBinaryOp(binaryOperator);
            } else {
                throw new IllegalArgumentException(logicalVertex.toString());
            }
            queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
            Pair<Integer, Integer> dfsCmdPair = dfsGraphVertexMapping.get(logicalVertex.getId());
            if (null != dfsCmdPair) {
                queryPlanBuilder.addOperatorIdList(dfsCmdPair.getLeft());
                queryPlanBuilder.addOperatorIdList(dfsCmdPair.getRight());
            }
        }
    }
}

Example 3 with LogicalSourceDelegateVertex

use of com.alibaba.maxgraph.compiler.logical.LogicalSourceDelegateVertex in project GraphScope by alibaba.

the class CostEstimator method costOperator.

/**
 * This method computes the cost of an operator. The cost is composed of cost for input shipping,
 * locally processing an input, and running the operator.
 * <p>
 * It requires at least that all inputs are set and have a proper ship strategy set,
 * which is not equal to <tt>NONE</tt>.
 *
 * @param queryPlan The given query plan
 * @param vertex    The node to compute the costs for.
 */
private Costs costOperator(LogicalQueryPlan queryPlan, LogicalVertex vertex) {
    Costs costs = new Costs();
    if (vertex instanceof LogicalSourceVertex) {
        if (vertex instanceof LogicalSourceDelegateVertex) {
            LogicalSourceDelegateVertex logicalSourceDelegateVertex = LogicalSourceDelegateVertex.class.cast(vertex);
            LogicalVertex delegateVertex = logicalSourceDelegateVertex.getDelegateVertex();
            costs.setCpuCost(delegateVertex.getEstimatedNumRecords());
            costs.setNetworkCost(delegateVertex.getEstimatedOutputSize());
        } else {
            costs.setCpuCost(vertex.getEstimatedNumRecords());
            costs.setNetworkCost(0);
        }
    } else if (vertex instanceof LogicalUnaryVertex) {
        LogicalUnaryVertex unaryVertex = LogicalUnaryVertex.class.cast(vertex);
        LogicalVertex inputVertex = unaryVertex.getInputVertex();
        LogicalEdge logicalEdge = queryPlan.getLogicalEdge(inputVertex, unaryVertex);
        if (null != logicalEdge && logicalEdge.getShuffleType() != EdgeShuffleType.FORWARD) {
            costs.setNetworkCost(inputVertex.getEstimatedOutputSize());
        } else {
            costs.setNetworkCost(0);
        }
        costs.setCpuCost(inputVertex.getEstimatedNumRecords() * getOperatorFactor(vertex.getProcessorFunction()));
    } else {
        LogicalBinaryVertex logicalBinaryVertex = LogicalBinaryVertex.class.cast(vertex);
        LogicalVertex leftVertex = logicalBinaryVertex.getLeftInput();
        double networkCost = 0;
        if (queryPlan.getLogicalEdge(leftVertex, logicalBinaryVertex).getShuffleType() != EdgeShuffleType.FORWARD) {
            networkCost += leftVertex.getEstimatedOutputSize();
        }
        LogicalVertex rightVertex = logicalBinaryVertex.getRightInput();
        if (queryPlan.getLogicalEdge(rightVertex, logicalBinaryVertex).getShuffleType() != EdgeShuffleType.FORWARD) {
            networkCost += rightVertex.getEstimatedOutputSize();
        }
        costs.setNetworkCost(networkCost);
        costs.setCpuCost(leftVertex.getEstimatedNumRecords() * 2 + rightVertex.getEstimatedNumRecords());
    }
    return costs;
}

Example 4 with LogicalSourceDelegateVertex

use of com.alibaba.maxgraph.compiler.logical.LogicalSourceDelegateVertex in project GraphScope by alibaba.

the class RepeatTreeNode method buildLogicalQueryPlan.

@Override
public LogicalSubQueryPlan buildLogicalQueryPlan(ContextManager contextManager) {
    TreeNodeLabelManager labelManager = contextManager.getTreeNodeLabelManager();
    VertexIdManager vertexIdManager = contextManager.getVertexIdManager();
    LogicalSubQueryPlan logicalSubQueryPlan = new LogicalSubQueryPlan(contextManager);
    LogicalVertex delegateSourceVertex = getInputNode().getOutputVertex();
    logicalSubQueryPlan.addLogicalVertex(delegateSourceVertex);
    List<LogicalVertex> outputVertexList = Lists.newArrayList();
    LogicalVertex inputVertex = delegateSourceVertex;
    ProcessorRepeatFunction processorRepeatFunction = new ProcessorRepeatFunction();
    if (null != untilFirstTreeNode) {
        Pair<LogicalVertex, Pair<LogicalVertex, LogicalSubQueryPlan>> untilTable = buildUntilQueryPlan(untilTreeNode, delegateSourceVertex, labelManager, contextManager, vertexIdManager);
        outputVertexList.add(untilTable.getLeft());
        inputVertex = untilTable.getRight().getLeft();
        logicalSubQueryPlan.mergeLogicalQueryPlan(untilTable.getRight().getRight());
    }
    if (null != emitFirstTreeNode) {
        LogicalVertex emitVertex;
        if (emitFirstTreeNode instanceof SourceDelegateNode) {
            emitVertex = inputVertex;
        } else {
            emitVertex = buildFilterResultPlan(contextManager, vertexIdManager, labelManager, logicalSubQueryPlan, inputVertex, emitFirstTreeNode);
        }
        outputVertexList.add(emitVertex);
        processorRepeatFunction.enableEmitFlag();
    }
    checkArgument(null != repeatBodyTreeNode, "repeat body tree node can't be null");
    processorRepeatFunction.setMaxLoopTimes(maxLoopTimes);
    LogicalVertex repeatVertex = new LogicalUnaryVertex(vertexIdManager.getId(), processorRepeatFunction, false, inputVertex);
    logicalSubQueryPlan.addLogicalVertex(repeatVertex);
    logicalSubQueryPlan.addLogicalEdge(inputVertex, repeatVertex, new LogicalEdge());
    List<LogicalVertex> leaveVertexList = Lists.newArrayList();
    LogicalSubQueryPlan repeatBodyPlan = new LogicalSubQueryPlan(contextManager);
    LogicalSourceDelegateVertex repeatSourceVertex = new LogicalSourceDelegateVertex(inputVertex);
    repeatBodyPlan.addLogicalVertex(repeatSourceVertex);
    repeatBodyPlan.mergeLogicalQueryPlan(TreeNodeUtils.buildQueryPlanWithSource(repeatBodyTreeNode, labelManager, contextManager, vertexIdManager, repeatSourceVertex));
    LogicalVertex feedbackVertex = repeatBodyPlan.getOutputVertex();
    if (null != untilTreeNode) {
        Pair<LogicalVertex, Pair<LogicalVertex, LogicalSubQueryPlan>> untilTable = buildUntilQueryPlan(untilTreeNode, feedbackVertex, labelManager, contextManager, vertexIdManager);
        feedbackVertex = untilTable.getRight().getLeft();
        LogicalVertex untilVertex = untilTable.getLeft();
        untilVertex.getAfterRequirementList().add(QueryFlowOuterClass.RequirementValue.newBuilder().setReqType(QueryFlowOuterClass.RequirementType.KEY_DEL));
        leaveVertexList.add(untilVertex);
        repeatBodyPlan.mergeLogicalQueryPlan(untilTable.getRight().getRight());
    }
    if (null != emitTreeNode) {
        LogicalVertex emitVertex;
        if (emitTreeNode instanceof SourceDelegateNode) {
            emitVertex = feedbackVertex;
        } else {
            emitVertex = buildFilterResultPlan(contextManager, vertexIdManager, labelManager, repeatBodyPlan, feedbackVertex, emitTreeNode);
        }
        if (null != dfsEmitTreeNode) {
            repeatBodyPlan.mergeLogicalQueryPlan(TreeNodeUtils.buildQueryPlanWithSource(dfsEmitTreeNode, labelManager, contextManager, vertexIdManager, emitVertex));
            emitVertex = repeatBodyPlan.getOutputVertex();
        }
        leaveVertexList.add(emitVertex);
        processorRepeatFunction.enableEmitFlag();
    }
    if (null != dfsFeedTreeNode) {
        dfsFeedTreeNode.setRepeatSourceVertex(repeatBodyPlan.getTargetVertex(repeatSourceVertex));
        LogicalSubQueryPlan dfsQueryPlan = TreeNodeUtils.buildQueryPlanWithSource(dfsFeedTreeNode, labelManager, contextManager, vertexIdManager, feedbackVertex);
        feedbackVertex = dfsQueryPlan.getOutputVertex();
        repeatBodyPlan.mergeLogicalQueryPlan(dfsQueryPlan);
    }
    processorRepeatFunction.setEnterVertex(repeatBodyPlan.getTargetVertex(repeatSourceVertex));
    processorRepeatFunction.setFeedbackVertex(feedbackVertex);
    processorRepeatFunction.setRepeatPlan(repeatBodyPlan);
    if (!leaveVertexList.isEmpty()) {
        if (leaveVertexList.size() == 1) {
            processorRepeatFunction.setLeaveVertex(leaveVertexList.get(0));
        } else {
            LogicalVertex unionLeaveVertex = unionVertexList(vertexIdManager, repeatBodyPlan, leaveVertexList);
            processorRepeatFunction.setLeaveVertex(unionLeaveVertex);
        }
    }
    LogicalVertex outputVertex = repeatVertex;
    if (!outputVertexList.isEmpty()) {
        List<LogicalVertex> outputUnionVertexList = Lists.newArrayList(outputVertexList);
        outputUnionVertexList.add(repeatVertex);
        outputVertex = unionVertexList(vertexIdManager, logicalSubQueryPlan, outputUnionVertexList);
    }
    addUsedLabelAndRequirement(outputVertex, labelManager);
    setFinishVertex(outputVertex, labelManager);
    return logicalSubQueryPlan;
}