Examples with ProcessorFunction com.alibaba.maxgraph.compiler.logical.function.ProcessorFunction used on opensource projects

Example 31 with ProcessorFunction

use of com.alibaba.maxgraph.compiler.logical.function.ProcessorFunction in project GraphScope by alibaba.

the class FoldTreeNode method buildLogicalQueryPlan.

@Override
public LogicalSubQueryPlan buildLogicalQueryPlan(ContextManager contextManager) {
    LogicalSubQueryPlan logicalSubQueryPlan = new LogicalSubQueryPlan(contextManager);
    LogicalVertex sourceVertex = getInputNode().getOutputVertex();
    logicalSubQueryPlan.addLogicalVertex(sourceVertex);
    QueryFlowOuterClass.OperatorType foldType = getUseKeyOperator(QueryFlowOuterClass.OperatorType.FOLD);
    LogicalVertex foldVertex = new LogicalUnaryVertex(contextManager.getVertexIdManager().getId(), new ProcessorFunction(foldType, Message.Value.newBuilder()), false, sourceVertex);
    logicalSubQueryPlan.addLogicalVertex(foldVertex);
    logicalSubQueryPlan.addLogicalEdge(sourceVertex, foldVertex, new LogicalEdge(EdgeShuffleType.SHUFFLE_BY_CONST));
    setFinishVertex(foldVertex, contextManager.getTreeNodeLabelManager());
    addUsedLabelAndRequirement(foldVertex, contextManager.getTreeNodeLabelManager());
    return logicalSubQueryPlan;
}

Example 32 with ProcessorFunction

use of com.alibaba.maxgraph.compiler.logical.function.ProcessorFunction in project GraphScope by alibaba.

the class GroupTreeNode method buildLogicalQueryPlan.

@Override
public LogicalSubQueryPlan buildLogicalQueryPlan(ContextManager contextManager) {
    LogicalSubQueryPlan logicalSubQueryPlan = new LogicalSubQueryPlan(contextManager);
    LogicalVertex sourceVertex = getInputNode().getOutputVertex();
    logicalSubQueryPlan.addLogicalVertex(sourceVertex);
    QueryFlowOuterClass.EnterKeyArgumentProto enterKeyArgumentProto = null;
    if (keyTreeNode == null || keyTreeNode instanceof SourceDelegateNode) {
        enterKeyArgumentProto = QueryFlowOuterClass.EnterKeyArgumentProto.newBuilder().setEnterKeyType(QueryFlowOuterClass.EnterKeyTypeProto.KEY_SELF).build();
    } else if (UnaryTreeNode.class.cast(keyTreeNode).getInputNode() instanceof SourceDelegateNode) {
        Optional<QueryFlowOuterClass.EnterKeyArgumentProto> propLabelId = TreeNodeUtils.parseEnterKeyArgument(keyTreeNode, schema, contextManager.getTreeNodeLabelManager());
        if (propLabelId.isPresent()) {
            enterKeyArgumentProto = propLabelId.get();
        }
    }
    if (null != enterKeyArgumentProto) {
        // use enter key operator to generate key data
        ProcessorFunction enterKeyFunction = new ProcessorFunction(QueryFlowOuterClass.OperatorType.ENTER_KEY, Message.Value.newBuilder().setPayload(enterKeyArgumentProto.toByteString()));
        LogicalVertex enterKeyVertex = new LogicalUnaryVertex(contextManager.getVertexIdManager().getId(), enterKeyFunction, false, sourceVertex);
        enterKeyArgumentProto.getPropIdListList().forEach(v -> {
            if (v < 0) {
                enterKeyFunction.getUsedLabelList().add(v);
            }
        });
        if (enterKeyArgumentProto.getPropLabelId() < 0) {
            enterKeyFunction.getUsedLabelList().add(enterKeyArgumentProto.getPropLabelId());
        }
        logicalSubQueryPlan.addLogicalVertex(enterKeyVertex);
        logicalSubQueryPlan.addLogicalEdge(sourceVertex, enterKeyVertex, new LogicalEdge());
    } else {
        TreeNode currentKeyNode = TreeNodeUtils.buildSingleOutputNode(keyTreeNode, schema);
        LogicalSubQueryPlan keyQueryPlan = TreeNodeUtils.buildSubQueryPlan(currentKeyNode, sourceVertex, contextManager, true);
        LogicalVertex keyValueVertex = keyQueryPlan.getOutputVertex();
        logicalSubQueryPlan.mergeLogicalQueryPlan(keyQueryPlan);
        sourceVertex = TreeNodeUtils.getSourceTreeNode(keyTreeNode).getOutputVertex();
        if (sourceVertex.getProcessorFunction().getOperatorType() == QueryFlowOuterClass.OperatorType.ENTER_KEY) {
            ProcessorFunction joinKeyFunction = new ProcessorFunction(QueryFlowOuterClass.OperatorType.JOIN_RIGHT_VALUE_KEY);
            LogicalBinaryVertex joinKeyVertex = new LogicalBinaryVertex(contextManager.getVertexIdManager().getId(), joinKeyFunction, false, sourceVertex, keyValueVertex);
            logicalSubQueryPlan.addLogicalVertex(joinKeyVertex);
            logicalSubQueryPlan.addLogicalEdge(sourceVertex, joinKeyVertex, new LogicalEdge());
            logicalSubQueryPlan.addLogicalEdge(keyValueVertex, joinKeyVertex, new LogicalEdge());
        } else {
            String keyValueLabel = contextManager.getTreeNodeLabelManager().createSysLabelStart(keyValueVertex, "val");
            String sourceLabel = contextManager.getTreeNodeLabelManager().createSysLabelStart(sourceVertex, "source");
            ProcessorFunction selectSourceFunction = TreeNodeUtils.createSelectOneFunction(sourceLabel, Pop.first, contextManager.getTreeNodeLabelManager().getLabelIndexList());
            LogicalVertex selectVertex = new LogicalUnaryVertex(contextManager.getVertexIdManager().getId(), selectSourceFunction, false, keyValueVertex);
            logicalSubQueryPlan.addLogicalVertex(selectVertex);
            logicalSubQueryPlan.addLogicalEdge(keyValueVertex, selectVertex, new LogicalEdge(EdgeShuffleType.FORWARD));
            int keyValueLabelId = contextManager.getTreeNodeLabelManager().getLabelIndex(keyValueLabel);
            ProcessorFunction enterKeyFunction = new ProcessorFunction(QueryFlowOuterClass.OperatorType.ENTER_KEY, Message.Value.newBuilder().setPayload(QueryFlowOuterClass.EnterKeyArgumentProto.newBuilder().setEnterKeyType(QueryFlowOuterClass.EnterKeyTypeProto.KEY_PROP_LABEL).setUniqFlag(false).setPropLabelId(keyValueLabelId).build().toByteString()));
            enterKeyFunction.getUsedLabelList().add(keyValueLabelId);
            LogicalVertex enterKeyVertex = new LogicalUnaryVertex(contextManager.getVertexIdManager().getId(), enterKeyFunction, false, selectVertex);
            logicalSubQueryPlan.addLogicalVertex(enterKeyVertex);
            logicalSubQueryPlan.addLogicalEdge(selectVertex, enterKeyVertex, new LogicalEdge(EdgeShuffleType.FORWARD));
        }
    }
    LogicalVertex outputVertex = logicalSubQueryPlan.getOutputVertex();
    LogicalSubQueryPlan aggregateQueryPlan = TreeNodeUtils.buildSubQueryPlan(valueTreeNode, outputVertex, contextManager, false);
    LogicalVertex aggregateVertex = aggregateQueryPlan.getOutputVertex();
    logicalSubQueryPlan.mergeLogicalQueryPlan(aggregateQueryPlan);
    // convert by key value to entry
    ProcessorFunction processorFunction = new ProcessorFunction(QueryFlowOuterClass.OperatorType.BYKEY_ENTRY);
    LogicalVertex bykeyEntryVertex = new LogicalUnaryVertex(contextManager.getVertexIdManager().getId(), processorFunction, false, aggregateVertex);
    logicalSubQueryPlan.addLogicalVertex(bykeyEntryVertex);
    logicalSubQueryPlan.addLogicalEdge(aggregateVertex, bykeyEntryVertex, new LogicalEdge());
    outputVertex = logicalSubQueryPlan.getOutputVertex();
    ProcessorFunction foldMapFunction = new ProcessorFunction(QueryFlowOuterClass.OperatorType.FOLDMAP);
    LogicalVertex foldMapVertex = new LogicalUnaryVertex(contextManager.getVertexIdManager().getId(), foldMapFunction, false, outputVertex);
    logicalSubQueryPlan.addLogicalVertex(foldMapVertex);
    logicalSubQueryPlan.addLogicalEdge(outputVertex, foldMapVertex, new LogicalEdge());
    addUsedLabelAndRequirement(foldMapVertex, contextManager.getTreeNodeLabelManager());
    setFinishVertex(foldMapVertex, contextManager.getTreeNodeLabelManager());
    return logicalSubQueryPlan;
}

Example 33 with ProcessorFunction

use of com.alibaba.maxgraph.compiler.logical.function.ProcessorFunction in project GraphScope by alibaba.

the class QueryFlowBuilder method buildQueryFlow.

private void buildQueryFlow(QueryPlan.Builder queryPlanBuilder, LogicalQueryPlan queryPlan, long snapshot) {
    List<LogicalVertex> logicalVertexList = queryPlan.getLogicalVertexList();
    Map<Integer, Pair<Integer, Integer>> dfsGraphVertexMapping = Maps.newHashMap();
    for (LogicalVertex logicalVertex : logicalVertexList) {
        ProcessorFunction processorFunction = logicalVertex.getProcessorFunction();
        if (processorFunction instanceof ProcessorRepeatFunction) {
            ProcessorRepeatFunction processorRepeatFunction = ProcessorRepeatFunction.class.cast(processorFunction);
            QueryFlowBuilder repeatBuilder = new QueryFlowBuilder();
            QueryFlow queryFlow = repeatBuilder.buildQueryFlow(processorRepeatFunction.getRepeatPlan(), snapshot).build();
            RepeatArgumentProto.Builder repeatArgument = RepeatArgumentProto.newBuilder().setPlan(queryFlow.getQueryPlan()).setLoopLimit((int) processorRepeatFunction.getMaxLoopTimes()).setFeedbackId(processorRepeatFunction.getFeedbackVertex().getId()).setEmitFlag(processorRepeatFunction.isHasEmitFlag());
            if (processorRepeatFunction.getLeaveVertex() != null) {
                repeatArgument.setLeaveId(processorRepeatFunction.getLeaveVertex().getId());
            } else {
                repeatArgument.setLeaveId(-1);
            }
            logger.info("repeat argument => ");
            logger.info(TextFormat.printToString(repeatArgument));
            OperatorBase.Builder repeatBase = OperatorBase.newBuilder().setId(logicalVertex.getId()).setOperatorType(OperatorType.REPEAT).setArgument(Value.newBuilder().setPayload(repeatArgument.build().toByteString()));
            queryPlanBuilder.addUnaryOp(UnaryOperator.newBuilder().setBase(repeatBase).setInputOperatorId(queryPlan.getSourceVertex(logicalVertex).getId()));
            queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
        } else if (processorFunction instanceof ProcessorChainFunction) {
            OperatorBase.Builder baseBuilder = OperatorBase.newBuilder();
            baseBuilder.setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId());
            if (null != logicalVertex.getEarlyStopArgument()) {
                baseBuilder.setEarlyStopArgument(logicalVertex.getEarlyStopArgument().build());
            }
            ProcessorChainFunction processorChainFunction = ProcessorChainFunction.class.cast(processorFunction);
            processorChainFunction.getChainVertexList().forEach(v -> baseBuilder.addChainedFunction(createChainedFunction(v)));
            if (logicalVertex instanceof LogicalChainSourceVertex) {
                LogicalSourceVertex logicalSourceVertex = ((LogicalChainSourceVertex) logicalVertex).getLogicalSourceVertex();
                baseBuilder.setArgument(logicalSourceVertex.getProcessorFunction().getArgumentBuilder()).addAllLogicalCompare(logicalSourceVertex.getProcessorFunction().getLogicalCompareList());
                SourceOperator.Builder sourceBuilder = SourceOperator.newBuilder().setBase(baseBuilder);
                queryPlanBuilder.setSourceOp(sourceBuilder);
            } else {
                LogicalChainUnaryVertex logicalUnaryVertex = (LogicalChainUnaryVertex) logicalVertex;
                UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(queryPlan.getSourceVertex(logicalUnaryVertex).getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
                LogicalEdge logicalEdge = queryPlan.getLogicalEdge(queryPlan.getSourceVertex(logicalUnaryVertex), logicalUnaryVertex);
                unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
                if (logicalEdge.getShuffleType() == EdgeShuffleType.SHUFFLE_BY_KEY) {
                    InputEdgeShuffle.Builder shuffleBuilder = InputEdgeShuffle.newBuilder().setShuffleType(InputShuffleType.SHUFFLE_BY_ID_TYPE);
                    if (logicalEdge.getShufflePropId() != 0) {
                        shuffleBuilder.setShuffleValue(logicalEdge.getShufflePropId());
                    }
                    unaryOperator.setInputShuffle(shuffleBuilder);
                }
                queryPlanBuilder.addUnaryOp(unaryOperator);
            }
            queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
        } else if (processorFunction instanceof ProcessorLabelValueFunction) {
            ProcessorLabelValueFunction processorLabelValueFunction = (ProcessorLabelValueFunction) processorFunction;
            LogicalVertex labelValueVertex = processorLabelValueFunction.getLabelValueVertex();
            OperatorBase.Builder labelOperatorBuilder = createChainedFunction(labelValueVertex);
            int labelId = processorLabelValueFunction.getLabelId();
            OperatorBase.Builder baseBuilder = OperatorBase.newBuilder().setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId()).setArgument(Value.newBuilder().setIntValue(labelId).setBoolValue(processorLabelValueFunction.getRequireLabelFlag()).setPayload(labelOperatorBuilder.build().toByteString()));
            LogicalVertex inputVertex = queryPlan.getSourceVertex(logicalVertex);
            UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(inputVertex.getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
            LogicalEdge logicalEdge = queryPlan.getLogicalEdge(inputVertex, logicalVertex);
            unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
            queryPlanBuilder.addUnaryOp(unaryOperator).addOperatorIdList(logicalVertex.getId());
        } else {
            if (logicalVertex instanceof LogicalSourceDelegateVertex) {
                continue;
            }
            OperatorBase.Builder baseBuilder = OperatorBase.newBuilder();
            baseBuilder.setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId());
            if (processorFunction.getArgumentBuilder() != null) {
                baseBuilder.setArgument(processorFunction.getArgumentBuilder());
            }
            processorFunction.getLogicalCompareList().forEach(baseBuilder::addLogicalCompare);
            if (processorFunction.getRangeLimit() != null) {
                baseBuilder.setRangeLimit(processorFunction.getRangeLimit());
            }
            baseBuilder.addAllAfterRequirement(logicalVertex.getAfterRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
            baseBuilder.addAllBeforeRequirement(logicalVertex.getBeforeRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
            if (logicalVertex instanceof LogicalSourceVertex) {
                ProcessorSourceFunction processorSourceFunction = (ProcessorSourceFunction) logicalVertex.getProcessorFunction();
                SourceOperator.Builder sourceBuilder = SourceOperator.newBuilder().setBase(baseBuilder);
                if (null != processorSourceFunction.getOdpsQueryInput()) {
                    sourceBuilder.setOdpsInput(processorSourceFunction.getOdpsQueryInput()).setSourceType(SourceType.ODPS);
                } else {
                    sourceBuilder.setSourceType(SourceType.GRAPH);
                }
                queryPlanBuilder.setSourceOp(sourceBuilder);
            } else if (logicalVertex instanceof LogicalUnaryVertex) {
                EarlyStopArgument.Builder earlyStopArgument = logicalVertex.getEarlyStopArgument();
                if (null != earlyStopArgument && (earlyStopArgument.getGlobalFilterFlag() || earlyStopArgument.getGlobalStopFlag())) {
                    baseBuilder.setEarlyStopArgument(logicalVertex.getEarlyStopArgument().build());
                }
                LogicalUnaryVertex logicalUnaryVertex = LogicalUnaryVertex.class.cast(logicalVertex);
                UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(queryPlan.getSourceVertex(logicalUnaryVertex).getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
                LogicalEdge logicalEdge = queryPlan.getLogicalEdge(queryPlan.getSourceVertex(logicalUnaryVertex), logicalUnaryVertex);
                unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
                queryPlanBuilder.addUnaryOp(unaryOperator);
                if (processorFunction.getOperatorType() == OperatorType.DFS_REPEAT_GRAPH) {
                    VertexIdManager vertexIdManager = queryPlan.getPlanVertexIdManager();
                    int cmdLeftId = vertexIdManager.getId();
                    int dataRightId = vertexIdManager.getId();
                    dfsGraphVertexMapping.put(baseBuilder.getId(), Pair.of(cmdLeftId, dataRightId));
                    UnaryOperator.Builder cmdLeftOp = UnaryOperator.newBuilder().setBase(OperatorBase.newBuilder().setOperatorType(OperatorType.DFS_REPEAT_CMD).setId(cmdLeftId)).setInputOperatorId(baseBuilder.getId()).setShuffleType(InputShuffleType.FORWARD_TYPE);
                    queryPlanBuilder.addUnaryOp(cmdLeftOp);
                    UnaryOperator.Builder dataRightOp = UnaryOperator.newBuilder().setBase(OperatorBase.newBuilder().setOperatorType(OperatorType.DFS_REPEAT_DATA).setId(dataRightId)).setInputOperatorId(baseBuilder.getId()).setShuffleType(InputShuffleType.FORWARD_TYPE);
                    queryPlanBuilder.addUnaryOp(dataRightOp);
                }
            } else if (logicalVertex instanceof LogicalBinaryVertex) {
                LogicalBinaryVertex logicalBinaryVertex = LogicalBinaryVertex.class.cast(logicalVertex);
                BinaryOperator.Builder binaryOperator = BinaryOperator.newBuilder().setBase(baseBuilder).setLeftInputOperatorId(parseInputOperatorId(logicalBinaryVertex.getLeftInput().getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping)).setRightInputOperatorId(parseInputOperatorId(logicalBinaryVertex.getRightInput().getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
                LogicalEdge leftEdge = queryPlan.getLogicalEdge(logicalBinaryVertex.getLeftInput(), logicalBinaryVertex);
                LogicalEdge rightEdge = queryPlan.getLogicalEdge(logicalBinaryVertex.getRightInput(), logicalBinaryVertex);
                binaryOperator.setLeftShuffleType(CompilerUtils.parseShuffleTypeFromEdge(leftEdge)).setLeftStreamIndex(leftEdge.getStreamIndex()).setRightShuffleType(CompilerUtils.parseShuffleTypeFromEdge(rightEdge)).setRightStreamIndex(rightEdge.getStreamIndex());
                queryPlanBuilder.addBinaryOp(binaryOperator);
            } else {
                throw new IllegalArgumentException(logicalVertex.toString());
            }
            queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
            Pair<Integer, Integer> dfsCmdPair = dfsGraphVertexMapping.get(logicalVertex.getId());
            if (null != dfsCmdPair) {
                queryPlanBuilder.addOperatorIdList(dfsCmdPair.getLeft());
                queryPlanBuilder.addOperatorIdList(dfsCmdPair.getRight());
            }
        }
    }
}

Example 34 with ProcessorFunction

use of com.alibaba.maxgraph.compiler.logical.function.ProcessorFunction in project GraphScope by alibaba.

the class PropertiesTreeNode method buildLogicalQueryPlan.

@Override
public LogicalSubQueryPlan buildLogicalQueryPlan(ContextManager contextManager) {
    TreeNodeLabelManager labelManager = contextManager.getTreeNodeLabelManager();
    VertexIdManager vertexIdManager = contextManager.getVertexIdManager();
    Message.Value.Builder argumentBuilder = Message.Value.newBuilder().setBoolFlag(edgePropFlag());
    if (null != propertyKeys) {
        Arrays.stream(propertyKeys).forEach(v -> argumentBuilder.addIntValueList(CompilerUtils.getPropertyId(schema, v)));
    }
    QueryFlowOuterClass.OperatorType operatorType = returnType == PropertyType.VALUE ? QueryFlowOuterClass.OperatorType.PROP_VALUE : QueryFlowOuterClass.OperatorType.PROPERTIES;
    ProcessorFunction processorFunction = new ProcessorFunction(operatorType, argumentBuilder);
    return parseSingleUnaryVertex(vertexIdManager, labelManager, processorFunction, contextManager);
}

Example 35 with ProcessorFunction

use of com.alibaba.maxgraph.compiler.logical.function.ProcessorFunction in project GraphScope by alibaba.

the class RangeGlobalTreeNode method buildLogicalQueryPlan.

@Override
public LogicalSubQueryPlan buildLogicalQueryPlan(ContextManager contextManager) {
    TreeNodeLabelManager labelManager = contextManager.getTreeNodeLabelManager();
    VertexIdManager vertexIdManager = contextManager.getVertexIdManager();
    LogicalSubQueryPlan logicalSubQueryPlan = new LogicalSubQueryPlan(contextManager);
    LogicalVertex sourceVertex = getInputNode().getOutputVertex();
    logicalSubQueryPlan.addLogicalVertex(sourceVertex);
    ProcessorFunction combinerFunction = new ProcessorFunction(QueryFlowOuterClass.OperatorType.COMBINER_RANGE, createArgumentBuilder().addLongValueList(0).addLongValueList(high));
    LogicalUnaryVertex combinerVertex = new LogicalUnaryVertex(vertexIdManager.getId(), combinerFunction, false, sourceVertex);
    combinerVertex.setEarlyStopFlag(super.earlyStopArgument);
    logicalSubQueryPlan.addLogicalVertex(combinerVertex);
    logicalSubQueryPlan.addLogicalEdge(sourceVertex, combinerVertex, LogicalEdge.forwardEdge());
    QueryFlowOuterClass.OperatorType operatorType = getUseKeyOperator(QueryFlowOuterClass.OperatorType.RANGE);
    Message.Value.Builder argumentBuilder = Message.Value.newBuilder().addLongValueList(low).addLongValueList(high);
    ProcessorFunction processorFunction = new ProcessorFunction(operatorType, argumentBuilder);
    LogicalUnaryVertex rangeVertex = new LogicalUnaryVertex(vertexIdManager.getId(), processorFunction, false, combinerVertex);
    logicalSubQueryPlan.addLogicalVertex(rangeVertex);
    logicalSubQueryPlan.addLogicalEdge(combinerVertex, rangeVertex, new LogicalEdge());
    addUsedLabelAndRequirement(rangeVertex, labelManager);
    setFinishVertex(rangeVertex, labelManager);
    return logicalSubQueryPlan;
}