Examples with ProcessorLabelValueFunction com.alibaba.maxgraph.compiler.logical.function.ProcessorLabelValueFunction used on opensource projects

Example 1 with ProcessorLabelValueFunction

use of com.alibaba.maxgraph.compiler.logical.function.ProcessorLabelValueFunction in project GraphScope by alibaba.

the class QueryFlowBuilder method buildQueryFlow.

private void buildQueryFlow(QueryPlan.Builder queryPlanBuilder, LogicalQueryPlan queryPlan, long snapshot) {
    List<LogicalVertex> logicalVertexList = queryPlan.getLogicalVertexList();
    Map<Integer, Pair<Integer, Integer>> dfsGraphVertexMapping = Maps.newHashMap();
    for (LogicalVertex logicalVertex : logicalVertexList) {
        ProcessorFunction processorFunction = logicalVertex.getProcessorFunction();
        if (processorFunction instanceof ProcessorRepeatFunction) {
            ProcessorRepeatFunction processorRepeatFunction = ProcessorRepeatFunction.class.cast(processorFunction);
            QueryFlowBuilder repeatBuilder = new QueryFlowBuilder();
            QueryFlow queryFlow = repeatBuilder.buildQueryFlow(processorRepeatFunction.getRepeatPlan(), snapshot).build();
            RepeatArgumentProto.Builder repeatArgument = RepeatArgumentProto.newBuilder().setPlan(queryFlow.getQueryPlan()).setLoopLimit((int) processorRepeatFunction.getMaxLoopTimes()).setFeedbackId(processorRepeatFunction.getFeedbackVertex().getId()).setEmitFlag(processorRepeatFunction.isHasEmitFlag());
            if (processorRepeatFunction.getLeaveVertex() != null) {
                repeatArgument.setLeaveId(processorRepeatFunction.getLeaveVertex().getId());
            } else {
                repeatArgument.setLeaveId(-1);
            }
            logger.info("repeat argument => ");
            logger.info(TextFormat.printToString(repeatArgument));
            OperatorBase.Builder repeatBase = OperatorBase.newBuilder().setId(logicalVertex.getId()).setOperatorType(OperatorType.REPEAT).setArgument(Value.newBuilder().setPayload(repeatArgument.build().toByteString()));
            queryPlanBuilder.addUnaryOp(UnaryOperator.newBuilder().setBase(repeatBase).setInputOperatorId(queryPlan.getSourceVertex(logicalVertex).getId()));
            queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
        } else if (processorFunction instanceof ProcessorChainFunction) {
            OperatorBase.Builder baseBuilder = OperatorBase.newBuilder();
            baseBuilder.setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId());
            if (null != logicalVertex.getEarlyStopArgument()) {
                baseBuilder.setEarlyStopArgument(logicalVertex.getEarlyStopArgument().build());
            }
            ProcessorChainFunction processorChainFunction = ProcessorChainFunction.class.cast(processorFunction);
            processorChainFunction.getChainVertexList().forEach(v -> baseBuilder.addChainedFunction(createChainedFunction(v)));
            if (logicalVertex instanceof LogicalChainSourceVertex) {
                LogicalSourceVertex logicalSourceVertex = ((LogicalChainSourceVertex) logicalVertex).getLogicalSourceVertex();
                baseBuilder.setArgument(logicalSourceVertex.getProcessorFunction().getArgumentBuilder()).addAllLogicalCompare(logicalSourceVertex.getProcessorFunction().getLogicalCompareList());
                SourceOperator.Builder sourceBuilder = SourceOperator.newBuilder().setBase(baseBuilder);
                queryPlanBuilder.setSourceOp(sourceBuilder);
            } else {
                LogicalChainUnaryVertex logicalUnaryVertex = (LogicalChainUnaryVertex) logicalVertex;
                UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(queryPlan.getSourceVertex(logicalUnaryVertex).getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
                LogicalEdge logicalEdge = queryPlan.getLogicalEdge(queryPlan.getSourceVertex(logicalUnaryVertex), logicalUnaryVertex);
                unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
                if (logicalEdge.getShuffleType() == EdgeShuffleType.SHUFFLE_BY_KEY) {
                    InputEdgeShuffle.Builder shuffleBuilder = InputEdgeShuffle.newBuilder().setShuffleType(InputShuffleType.SHUFFLE_BY_ID_TYPE);
                    if (logicalEdge.getShufflePropId() != 0) {
                        shuffleBuilder.setShuffleValue(logicalEdge.getShufflePropId());
                    }
                    unaryOperator.setInputShuffle(shuffleBuilder);
                }
                queryPlanBuilder.addUnaryOp(unaryOperator);
            }
            queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
        } else if (processorFunction instanceof ProcessorLabelValueFunction) {
            ProcessorLabelValueFunction processorLabelValueFunction = (ProcessorLabelValueFunction) processorFunction;
            LogicalVertex labelValueVertex = processorLabelValueFunction.getLabelValueVertex();
            OperatorBase.Builder labelOperatorBuilder = createChainedFunction(labelValueVertex);
            int labelId = processorLabelValueFunction.getLabelId();
            OperatorBase.Builder baseBuilder = OperatorBase.newBuilder().setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId()).setArgument(Value.newBuilder().setIntValue(labelId).setBoolValue(processorLabelValueFunction.getRequireLabelFlag()).setPayload(labelOperatorBuilder.build().toByteString()));
            LogicalVertex inputVertex = queryPlan.getSourceVertex(logicalVertex);
            UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(inputVertex.getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
            LogicalEdge logicalEdge = queryPlan.getLogicalEdge(inputVertex, logicalVertex);
            unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
            queryPlanBuilder.addUnaryOp(unaryOperator).addOperatorIdList(logicalVertex.getId());
        } else {
            if (logicalVertex instanceof LogicalSourceDelegateVertex) {
                continue;
            }
            OperatorBase.Builder baseBuilder = OperatorBase.newBuilder();
            baseBuilder.setOperatorType(processorFunction.getOperatorType()).setId(logicalVertex.getId());
            if (processorFunction.getArgumentBuilder() != null) {
                baseBuilder.setArgument(processorFunction.getArgumentBuilder());
            }
            processorFunction.getLogicalCompareList().forEach(baseBuilder::addLogicalCompare);
            if (processorFunction.getRangeLimit() != null) {
                baseBuilder.setRangeLimit(processorFunction.getRangeLimit());
            }
            baseBuilder.addAllAfterRequirement(logicalVertex.getAfterRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
            baseBuilder.addAllBeforeRequirement(logicalVertex.getBeforeRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
            if (logicalVertex instanceof LogicalSourceVertex) {
                ProcessorSourceFunction processorSourceFunction = (ProcessorSourceFunction) logicalVertex.getProcessorFunction();
                SourceOperator.Builder sourceBuilder = SourceOperator.newBuilder().setBase(baseBuilder);
                if (null != processorSourceFunction.getOdpsQueryInput()) {
                    sourceBuilder.setOdpsInput(processorSourceFunction.getOdpsQueryInput()).setSourceType(SourceType.ODPS);
                } else {
                    sourceBuilder.setSourceType(SourceType.GRAPH);
                }
                queryPlanBuilder.setSourceOp(sourceBuilder);
            } else if (logicalVertex instanceof LogicalUnaryVertex) {
                EarlyStopArgument.Builder earlyStopArgument = logicalVertex.getEarlyStopArgument();
                if (null != earlyStopArgument && (earlyStopArgument.getGlobalFilterFlag() || earlyStopArgument.getGlobalStopFlag())) {
                    baseBuilder.setEarlyStopArgument(logicalVertex.getEarlyStopArgument().build());
                }
                LogicalUnaryVertex logicalUnaryVertex = LogicalUnaryVertex.class.cast(logicalVertex);
                UnaryOperator.Builder unaryOperator = UnaryOperator.newBuilder().setBase(baseBuilder).setInputOperatorId(parseInputOperatorId(queryPlan.getSourceVertex(logicalUnaryVertex).getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
                LogicalEdge logicalEdge = queryPlan.getLogicalEdge(queryPlan.getSourceVertex(logicalUnaryVertex), logicalUnaryVertex);
                unaryOperator.setShuffleType(CompilerUtils.parseShuffleTypeFromEdge(logicalEdge)).setInputStreamIndex(logicalEdge.getStreamIndex());
                queryPlanBuilder.addUnaryOp(unaryOperator);
                if (processorFunction.getOperatorType() == OperatorType.DFS_REPEAT_GRAPH) {
                    VertexIdManager vertexIdManager = queryPlan.getPlanVertexIdManager();
                    int cmdLeftId = vertexIdManager.getId();
                    int dataRightId = vertexIdManager.getId();
                    dfsGraphVertexMapping.put(baseBuilder.getId(), Pair.of(cmdLeftId, dataRightId));
                    UnaryOperator.Builder cmdLeftOp = UnaryOperator.newBuilder().setBase(OperatorBase.newBuilder().setOperatorType(OperatorType.DFS_REPEAT_CMD).setId(cmdLeftId)).setInputOperatorId(baseBuilder.getId()).setShuffleType(InputShuffleType.FORWARD_TYPE);
                    queryPlanBuilder.addUnaryOp(cmdLeftOp);
                    UnaryOperator.Builder dataRightOp = UnaryOperator.newBuilder().setBase(OperatorBase.newBuilder().setOperatorType(OperatorType.DFS_REPEAT_DATA).setId(dataRightId)).setInputOperatorId(baseBuilder.getId()).setShuffleType(InputShuffleType.FORWARD_TYPE);
                    queryPlanBuilder.addUnaryOp(dataRightOp);
                }
            } else if (logicalVertex instanceof LogicalBinaryVertex) {
                LogicalBinaryVertex logicalBinaryVertex = LogicalBinaryVertex.class.cast(logicalVertex);
                BinaryOperator.Builder binaryOperator = BinaryOperator.newBuilder().setBase(baseBuilder).setLeftInputOperatorId(parseInputOperatorId(logicalBinaryVertex.getLeftInput().getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping)).setRightInputOperatorId(parseInputOperatorId(logicalBinaryVertex.getRightInput().getId(), processorFunction.getOperatorType(), dfsGraphVertexMapping));
                LogicalEdge leftEdge = queryPlan.getLogicalEdge(logicalBinaryVertex.getLeftInput(), logicalBinaryVertex);
                LogicalEdge rightEdge = queryPlan.getLogicalEdge(logicalBinaryVertex.getRightInput(), logicalBinaryVertex);
                binaryOperator.setLeftShuffleType(CompilerUtils.parseShuffleTypeFromEdge(leftEdge)).setLeftStreamIndex(leftEdge.getStreamIndex()).setRightShuffleType(CompilerUtils.parseShuffleTypeFromEdge(rightEdge)).setRightStreamIndex(rightEdge.getStreamIndex());
                queryPlanBuilder.addBinaryOp(binaryOperator);
            } else {
                throw new IllegalArgumentException(logicalVertex.toString());
            }
            queryPlanBuilder.addOperatorIdList(logicalVertex.getId());
            Pair<Integer, Integer> dfsCmdPair = dfsGraphVertexMapping.get(logicalVertex.getId());
            if (null != dfsCmdPair) {
                queryPlanBuilder.addOperatorIdList(dfsCmdPair.getLeft());
                queryPlanBuilder.addOperatorIdList(dfsCmdPair.getRight());
            }
        }
    }
}

Example 2 with ProcessorLabelValueFunction

use of com.alibaba.maxgraph.compiler.logical.function.ProcessorLabelValueFunction in project GraphScope by alibaba.

the class LogicalPlanBuilder method buildLabelValuePlan.

private void buildLabelValuePlan(LogicalQueryPlan queryPlan, ContextManager contextManager, GraphSchema schema, LogicalVertex outputVertex, TreeNodeLabelManager treeNodeLabelManager, VertexIdManager vertexIdManager, CostMappingManager costMappingManager, String field, TreeNode currNode) {
    // Create field value here
    String parentField = costMappingManager.getValueParent(field);
    if (!StringUtils.isEmpty(parentField)) {
        BaseTreeNode nextNode = (BaseTreeNode) currNode.getOutputNode();
        nextNode.removeBeforeLabel(parentField);
        // If the field is value field, process it and remove the label requirement in vertex
        int labelIndex = treeNodeLabelManager.getLabelIndex(parentField);
        TreeNode fieldValueTreeNode = costMappingManager.getComputeTreeByValue(parentField);
        if (!(fieldValueTreeNode instanceof SourceTreeNode)) {
            for (QueryFlowOuterClass.RequirementValue.Builder reqValue : outputVertex.getAfterRequirementList()) {
                if (reqValue.getReqType() == QueryFlowOuterClass.RequirementType.LABEL_START) {
                    List<Integer> labelIndexList = reqValue.getReqArgumentBuilder().getIntValueListList().stream().filter(v -> v != labelIndex).collect(Collectors.toList());
                    reqValue.getReqArgumentBuilder().clearIntValueList().addAllIntValueList(labelIndexList);
                }
            }
            TreeNode currentFilterTreeNode = TreeNodeUtils.buildSingleOutputNode(fieldValueTreeNode, schema);
            // build filter plan, and use join direct filter vertex to filter left stream
            LogicalSubQueryPlan fieldValuePlan = TreeNodeUtils.buildSubQueryPlan(currentFilterTreeNode, outputVertex, contextManager);
            List<LogicalVertex> fieldValueVertexList = fieldValuePlan.getLogicalVertexList();
            if (fieldValueVertexList.size() == 2) {
                LogicalVertex fieldValueVertex = fieldValueVertexList.get(1);
                ProcessorLabelValueFunction labelValueFunction = new ProcessorLabelValueFunction(labelIndex, fieldValueVertex);
                LogicalVertex labelValueVertex = new LogicalUnaryVertex(vertexIdManager.getId(), labelValueFunction, outputVertex);
                queryPlan.addLogicalVertex(labelValueVertex);
                queryPlan.addLogicalEdge(outputVertex, labelValueVertex, LogicalEdge.shuffleByKey(labelIndex));
            } else {
                LogicalVertex fieldValueVertex = fieldValuePlan.getOutputVertex();
                fieldValueVertex.getAfterRequirementList().add(QueryFlowOuterClass.RequirementValue.newBuilder().setReqType(QueryFlowOuterClass.RequirementType.LABEL_START).setReqArgument(Message.Value.newBuilder().addIntValueList(labelIndex)));
                queryPlan.mergeLogicalQueryPlan(fieldValuePlan);
                LogicalVertex outputKeyVertex = new LogicalUnaryVertex(vertexIdManager.getId(), new ProcessorFunction(QueryFlowOuterClass.OperatorType.KEY_MESSAGE), fieldValueVertex);
                queryPlan.addLogicalVertex(outputKeyVertex);
                queryPlan.addLogicalEdge(fieldValueVertex, outputKeyVertex, LogicalEdge.forwardEdge());
            }
        }
    }
}

Example 3 with ProcessorLabelValueFunction

use of com.alibaba.maxgraph.compiler.logical.function.ProcessorLabelValueFunction in project GraphScope by alibaba.

the class SelectTreeNode method parseJoinQueryPlan.

private LogicalSubQueryPlan parseJoinQueryPlan(ContextManager contextManager, VertexIdManager vertexIdManager, TreeNodeLabelManager labelManager, Map<String, Integer> labelIndexList) {
    LogicalSubQueryPlan logicalSubQueryPlan = new LogicalSubQueryPlan(contextManager);
    LogicalVertex delegateSourceVertex = getInputNode().getOutputVertex();
    logicalSubQueryPlan.addLogicalVertex(delegateSourceVertex);
    Set<String> selectKeySet = Sets.newHashSet();
    List<Integer> keyIdList = Lists.newArrayList();
    List<Integer> valueIdList = Lists.newArrayList();
    for (int i = 0; i < selectKeyList.size(); i++) {
        String selectKey = selectKeyList.get(i);
        if (selectKeySet.contains(selectKey)) {
            continue;
        }
        selectKeySet.add(selectKey);
        TreeNode traversalTreeNode = traversalTreeNodeList.get(i % traversalTreeNodeList.size());
        int labelIndex = labelIndexList.get(selectKey);
        keyIdList.add(labelIndex);
        valueIdList.add(labelIndexList.get(selectKey));
        if (!(traversalTreeNode instanceof SourceDelegateNode)) {
            TreeNode currentTraversalNode = TreeNodeUtils.buildSingleOutputNode(traversalTreeNode, schema);
            LogicalVertex outputVertex = logicalSubQueryPlan.getOutputVertex();
            LogicalSubQueryPlan traversalPlan = TreeNodeUtils.buildSubQueryPlan(currentTraversalNode, outputVertex, contextManager);
            List<LogicalVertex> resultVertexList = traversalPlan.getLogicalVertexList();
            if (resultVertexList.size() == 2) {
                LogicalVertex fieldValueVertex = resultVertexList.get(1);
                ProcessorLabelValueFunction labelValueFunction = new ProcessorLabelValueFunction(labelIndex, fieldValueVertex);
                labelValueFunction.setRequireLabelFlag(true);
                LogicalVertex labelValueVertex = new LogicalUnaryVertex(vertexIdManager.getId(), labelValueFunction, outputVertex);
                logicalSubQueryPlan.addLogicalVertex(labelValueVertex);
                logicalSubQueryPlan.addLogicalEdge(outputVertex, labelValueVertex, LogicalEdge.shuffleByKey(labelIndex));
            } else {
                LogicalVertex fieldValueVertex = traversalPlan.getOutputVertex();
                fieldValueVertex.getAfterRequirementList().add(QueryFlowOuterClass.RequirementValue.newBuilder().setReqType(QueryFlowOuterClass.RequirementType.LABEL_START).setReqArgument(Message.Value.newBuilder().addIntValueList(labelIndex)));
                logicalSubQueryPlan.mergeLogicalQueryPlan(traversalPlan);
                LogicalVertex outputKeyVertex = new LogicalUnaryVertex(vertexIdManager.getId(), new ProcessorFunction(QueryFlowOuterClass.OperatorType.KEY_MESSAGE), fieldValueVertex);
                logicalSubQueryPlan.addLogicalVertex(outputKeyVertex);
                logicalSubQueryPlan.addLogicalEdge(fieldValueVertex, outputKeyVertex, LogicalEdge.forwardEdge());
            }
        }
    }
    LogicalVertex selectInputVertex = logicalSubQueryPlan.getOutputVertex();
    Message.Value.Builder argumentBuilder = Message.Value.newBuilder().setBoolValue(true).setIntValue(pop == null ? Message.PopType.POP_EMPTY.getNumber() : Message.PopType.valueOf(StringUtils.upperCase(pop.name())).getNumber()).addAllIntValueList(valueIdList).addAllStrValueList(keyIdList.stream().map(labelManager::getLabelName).collect(Collectors.toList()));
    ProcessorFunction processorFunction = new ProcessorFunction(QueryFlowOuterClass.OperatorType.SELECT, argumentBuilder);
    processorFunction.getUsedLabelList().addAll(valueIdList);
    LogicalVertex selectVertex = new LogicalUnaryVertex(vertexIdManager.getId(), processorFunction, false, selectInputVertex);
    logicalSubQueryPlan.addLogicalVertex(selectVertex);
    logicalSubQueryPlan.addLogicalEdge(selectInputVertex, selectVertex, new LogicalEdge(EdgeShuffleType.FORWARD));
    return logicalSubQueryPlan;
}

Example 4 with ProcessorLabelValueFunction

use of com.alibaba.maxgraph.compiler.logical.function.ProcessorLabelValueFunction in project GraphScope by alibaba.

the class QueryFlowBuilder method createChainedFunction.

private OperatorBase.Builder createChainedFunction(LogicalVertex logicalVertex) {
    OperatorBase.Builder functionBuilder = OperatorBase.newBuilder();
    ProcessorFunction processorFunction = logicalVertex.getProcessorFunction();
    functionBuilder.setOperatorType(processorFunction.getOperatorType());
    Value.Builder argumentBuilder = processorFunction.getArgumentBuilder();
    if (processorFunction instanceof ProcessorLabelValueFunction) {
        if (null == argumentBuilder) {
            argumentBuilder = Value.newBuilder();
        }
        ProcessorLabelValueFunction processorLabelValueFunction = (ProcessorLabelValueFunction) processorFunction;
        argumentBuilder.setIntValue(processorLabelValueFunction.getLabelId());
        if (null != processorLabelValueFunction.getLabelValueVertex()) {
            OperatorBase.Builder labelValueBuilder = createChainedFunction(processorLabelValueFunction.getLabelValueVertex());
            argumentBuilder.setPayload(labelValueBuilder.build().toByteString());
        }
    }
    if (argumentBuilder != null) {
        functionBuilder.setArgument(argumentBuilder);
    }
    functionBuilder.addAllLogicalCompare(processorFunction.getLogicalCompareList());
    if (processorFunction.getRangeLimit() != null) {
        functionBuilder.setRangeLimit(processorFunction.getRangeLimit());
    }
    functionBuilder.addAllAfterRequirement(logicalVertex.getAfterRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
    functionBuilder.addAllBeforeRequirement(logicalVertex.getBeforeRequirementList().stream().map(v -> v.build()).collect(Collectors.toList()));
    return functionBuilder;
}