Examples with ExecEdge org.apache.flink.table.planner.plan.nodes.exec.ExecEdge used on opensource projects

Example 26 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class BatchExecHashJoin method translateToPlanInternal.

@Override
@SuppressWarnings("unchecked")
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    ExecEdge leftInputEdge = getInputEdges().get(0);
    ExecEdge rightInputEdge = getInputEdges().get(1);
    Transformation<RowData> leftInputTransform = (Transformation<RowData>) leftInputEdge.translateToPlan(planner);
    Transformation<RowData> rightInputTransform = (Transformation<RowData>) rightInputEdge.translateToPlan(planner);
    // get input types
    RowType leftType = (RowType) leftInputEdge.getOutputType();
    RowType rightType = (RowType) rightInputEdge.getOutputType();
    JoinUtil.validateJoinSpec(joinSpec, leftType, rightType, false);
    int[] leftKeys = joinSpec.getLeftKeys();
    int[] rightKeys = joinSpec.getRightKeys();
    LogicalType[] keyFieldTypes = IntStream.of(leftKeys).mapToObj(leftType::getTypeAt).toArray(LogicalType[]::new);
    RowType keyType = RowType.of(keyFieldTypes);
    GeneratedJoinCondition condFunc = JoinUtil.generateConditionFunction(config.getTableConfig(), joinSpec.getNonEquiCondition().orElse(null), leftType, rightType);
    // projection for equals
    GeneratedProjection leftProj = ProjectionCodeGenerator.generateProjection(new CodeGeneratorContext(config.getTableConfig()), "HashJoinLeftProjection", leftType, keyType, leftKeys);
    GeneratedProjection rightProj = ProjectionCodeGenerator.generateProjection(new CodeGeneratorContext(config.getTableConfig()), "HashJoinRightProjection", rightType, keyType, rightKeys);
    Transformation<RowData> buildTransform;
    Transformation<RowData> probeTransform;
    GeneratedProjection buildProj;
    GeneratedProjection probeProj;
    int[] buildKeys;
    int[] probeKeys;
    RowType buildType;
    RowType probeType;
    int buildRowSize;
    long buildRowCount;
    long probeRowCount;
    boolean reverseJoin = !leftIsBuild;
    if (leftIsBuild) {
        buildTransform = leftInputTransform;
        buildProj = leftProj;
        buildType = leftType;
        buildRowSize = estimatedLeftAvgRowSize;
        buildRowCount = estimatedLeftRowCount;
        buildKeys = leftKeys;
        probeTransform = rightInputTransform;
        probeProj = rightProj;
        probeType = rightType;
        probeRowCount = estimatedLeftRowCount;
        probeKeys = rightKeys;
    } else {
        buildTransform = rightInputTransform;
        buildProj = rightProj;
        buildType = rightType;
        buildRowSize = estimatedRightAvgRowSize;
        buildRowCount = estimatedRightRowCount;
        buildKeys = rightKeys;
        probeTransform = leftInputTransform;
        probeProj = leftProj;
        probeType = leftType;
        probeRowCount = estimatedLeftRowCount;
        probeKeys = leftKeys;
    }
    // operator
    StreamOperatorFactory<RowData> operator;
    FlinkJoinType joinType = joinSpec.getJoinType();
    HashJoinType hashJoinType = HashJoinType.of(leftIsBuild, joinType.isLeftOuter(), joinType.isRightOuter(), joinType == FlinkJoinType.SEMI, joinType == FlinkJoinType.ANTI);
    if (LongHashJoinGenerator.support(hashJoinType, keyType, joinSpec.getFilterNulls())) {
        operator = LongHashJoinGenerator.gen(config.getTableConfig(), hashJoinType, keyType, buildType, probeType, buildKeys, probeKeys, buildRowSize, buildRowCount, reverseJoin, condFunc);
    } else {
        operator = SimpleOperatorFactory.of(HashJoinOperator.newHashJoinOperator(hashJoinType, condFunc, reverseJoin, joinSpec.getFilterNulls(), buildProj, probeProj, tryDistinctBuildRow, buildRowSize, buildRowCount, probeRowCount, keyType));
    }
    long managedMemory = config.get(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_HASH_JOIN_MEMORY).getBytes();
    return ExecNodeUtil.createTwoInputTransformation(buildTransform, probeTransform, createTransformationName(config), createTransformationDescription(config), operator, InternalTypeInfo.of(getOutputType()), probeTransform.getParallelism(), managedMemory);
}

Example 27 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class CommonExecPythonCalc method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final Configuration pythonConfig = CommonPythonUtil.getMergedConfig(planner.getExecEnv(), config.getTableConfig());
    OneInputTransformation<RowData, RowData> ret = createPythonOneInputTransformation(inputTransform, config, pythonConfig);
    if (CommonPythonUtil.isPythonWorkerUsingManagedMemory(pythonConfig)) {
        ret.declareManagedMemoryUseCaseAtSlotScope(ManagedMemoryUseCase.PYTHON);
    }
    return ret;
}

Example 28 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class JsonPlanGraph method convertToExecNodeGraph.

ExecNodeGraph convertToExecNodeGraph() {
    Map<Integer, ExecNode<?>> idToExecNodes = new HashMap<>();
    for (ExecNode<?> execNode : nodes) {
        int id = execNode.getId();
        if (idToExecNodes.containsKey(id)) {
            throw new TableException(String.format("The id: %s is not unique for ExecNode: %s.\nplease check it.", id, execNode.getDescription()));
        }
        idToExecNodes.put(id, execNode);
    }
    Map<Integer, List<ExecEdge>> idToInputEdges = new HashMap<>();
    Map<Integer, List<ExecEdge>> idToOutputEdges = new HashMap<>();
    for (JsonPlanEdge edge : edges) {
        ExecNode<?> source = idToExecNodes.get(edge.getSourceId());
        if (source == null) {
            throw new TableException(String.format("Source node id: %s is not found in nodes.", edge.getSourceId()));
        }
        ExecNode<?> target = idToExecNodes.get(edge.getTargetId());
        if (target == null) {
            throw new TableException(String.format("Target node id: %s is not found in nodes.", edge.getTargetId()));
        }
        ExecEdge execEdge = ExecEdge.builder().source(source).target(target).shuffle(edge.getShuffle()).exchangeMode(edge.getExchangeMode()).build();
        idToInputEdges.computeIfAbsent(target.getId(), n -> new ArrayList<>()).add(execEdge);
        idToOutputEdges.computeIfAbsent(source.getId(), n -> new ArrayList<>()).add(execEdge);
    }
    List<ExecNode<?>> rootNodes = new ArrayList<>();
    for (Map.Entry<Integer, ExecNode<?>> entry : idToExecNodes.entrySet()) {
        int id = entry.getKey();
        ExecNode<?> node = entry.getValue();
        // connect input edges
        List<ExecEdge> inputEdges = idToInputEdges.getOrDefault(id, new ArrayList<>());
        node.setInputEdges(inputEdges);
        if (!idToOutputEdges.containsKey(id)) {
            // if the node has no output nodes, it's a root node
            rootNodes.add(node);
        }
    }
    return new ExecNodeGraph(flinkVersion, rootNodes);
}

Example 29 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class JsonPlanGraph method fromExecNodeGraph.

static JsonPlanGraph fromExecNodeGraph(ExecNodeGraph execGraph) {
    final List<ExecNode<?>> allNodes = new ArrayList<>();
    final List<JsonPlanEdge> allEdges = new ArrayList<>();
    final Set<Integer> nodesIds = new HashSet<>();
    // for quick search
    final Set<ExecNode<?>> visitedNodes = Sets.newIdentityHashSet();
    // visit the nodes as topological ordering
    final ExecNodeVisitor visitor = new ExecNodeVisitorImpl() {

        @Override
        public void visit(ExecNode<?> node) {
            if (visitedNodes.contains(node)) {
                return;
            }
            super.visitInputs(node);
            final int id = node.getId();
            if (nodesIds.contains(id)) {
                throw new TableException(String.format("The id: %s is not unique for ExecNode: %s.\nplease check it.", id, node.getDescription()));
            }
            allNodes.add(node);
            nodesIds.add(id);
            visitedNodes.add(node);
            for (ExecEdge execEdge : node.getInputEdges()) {
                allEdges.add(JsonPlanEdge.fromExecEdge(execEdge));
            }
        }
    };
    execGraph.getRootNodes().forEach(visitor::visit);
    checkArgument(allNodes.size() == nodesIds.size());
    return new JsonPlanGraph(execGraph.getFlinkVersion(), allNodes, allEdges);
}

Example 30 with ExecEdge

use of org.apache.flink.table.planner.plan.nodes.exec.ExecEdge in project flink by apache.

the class MultipleInputNodeCreationProcessor method createStreamMultipleInputNode.

private StreamExecMultipleInput createStreamMultipleInputNode(MultipleInputGroup group, List<Tuple3<ExecNode<?>, InputProperty, ExecEdge>> inputs) {
    ExecNode<?> rootNode = group.root.execNode;
    List<ExecNode<?>> inputNodes = new ArrayList<>();
    for (Tuple3<ExecNode<?>, InputProperty, ExecEdge> tuple3 : inputs) {
        inputNodes.add(tuple3.f0);
    }
    String description = ExecNodeUtil.getMultipleInputDescription(rootNode, inputNodes, new ArrayList<>());
    StreamExecMultipleInput multipleInput = new StreamExecMultipleInput(inputNodes.stream().map(i -> InputProperty.DEFAULT).collect(Collectors.toList()), rootNode, description);
    List<ExecEdge> inputEdges = new ArrayList<>(inputNodes.size());
    for (ExecNode<?> inputNode : inputNodes) {
        inputEdges.add(ExecEdge.builder().source(inputNode).target(multipleInput).build());
    }
    multipleInput.setInputEdges(inputEdges);
    return multipleInput;
}