Examples with InputProperty org.apache.flink.table.planner.plan.nodes.exec.InputProperty used on opensource projects

Example 6 with InputProperty

use of org.apache.flink.table.planner.plan.nodes.exec.InputProperty in project flink by apache.

the class BatchExecExchange method translateToPlanInternal.

@SuppressWarnings("unchecked")
@Override
protected Transformation<RowData> translateToPlanInternal(PlannerBase planner, ExecNodeConfig config) {
    final ExecEdge inputEdge = getInputEdges().get(0);
    final Transformation<RowData> inputTransform = (Transformation<RowData>) inputEdge.translateToPlan(planner);
    final RowType inputType = (RowType) inputEdge.getOutputType();
    boolean requireUndefinedExchangeMode = false;
    final StreamPartitioner<RowData> partitioner;
    final int parallelism;
    final InputProperty inputProperty = getInputProperties().get(0);
    final RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
    final InputProperty.DistributionType distributionType = requiredDistribution.getType();
    switch(distributionType) {
        case ANY:
            partitioner = null;
            parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
            break;
        case BROADCAST:
            partitioner = new BroadcastPartitioner<>();
            parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
            break;
        case SINGLETON:
            partitioner = new GlobalPartitioner<>();
            parallelism = 1;
            break;
        case HASH:
            partitioner = createHashPartitioner(((HashDistribution) requiredDistribution), inputType, config);
            parallelism = ExecutionConfig.PARALLELISM_DEFAULT;
            break;
        case KEEP_INPUT_AS_IS:
            KeepInputAsIsDistribution keepInputAsIsDistribution = (KeepInputAsIsDistribution) requiredDistribution;
            if (keepInputAsIsDistribution.isStrict()) {
                // explicitly use ForwardPartitioner to guarantee the data distribution is
                // exactly the same as input
                partitioner = new ForwardPartitioner<>();
                requireUndefinedExchangeMode = true;
            } else {
                RequiredDistribution inputDistribution = ((KeepInputAsIsDistribution) requiredDistribution).getInputDistribution();
                checkArgument(inputDistribution instanceof HashDistribution, "Only HashDistribution is supported now");
                partitioner = new ForwardForConsecutiveHashPartitioner<>(createHashPartitioner(((HashDistribution) inputDistribution), inputType, config));
            }
            parallelism = inputTransform.getParallelism();
            break;
        default:
            throw new TableException(distributionType + "is not supported now!");
    }
    final StreamExchangeMode exchangeMode = requireUndefinedExchangeMode ? StreamExchangeMode.UNDEFINED : getBatchStreamExchangeMode(config, requiredExchangeMode);
    final Transformation<RowData> transformation = new PartitionTransformation<>(inputTransform, partitioner, exchangeMode);
    transformation.setParallelism(parallelism);
    transformation.setOutputType(InternalTypeInfo.of(getOutputType()));
    return transformation;
}

Example 7 with InputProperty

use of org.apache.flink.table.planner.plan.nodes.exec.InputProperty in project flink by apache.

the class MultipleInputNodeCreationProcessor method createBatchMultipleInputNode.

private BatchExecMultipleInput createBatchMultipleInputNode(MultipleInputGroup group, List<Tuple3<ExecNode<?>, InputProperty, ExecEdge>> inputs) {
    // first calculate the input orders using InputPriorityConflictResolver
    Set<ExecNode<?>> inputSet = new HashSet<>();
    for (Tuple3<ExecNode<?>, InputProperty, ExecEdge> tuple3 : inputs) {
        inputSet.add(tuple3.f0);
    }
    InputOrderCalculator calculator = new InputOrderCalculator(group.root.execNode, inputSet, InputProperty.DamBehavior.BLOCKING);
    Map<ExecNode<?>, Integer> inputOrderMap = calculator.calculate();
    // then create input rels and edges with the input orders
    ExecNode<?> rootNode = group.root.execNode;
    List<ExecNode<?>> inputNodes = new ArrayList<>();
    List<InputProperty> inputProperties = new ArrayList<>();
    List<ExecEdge> originalEdges = new ArrayList<>();
    for (Tuple3<ExecNode<?>, InputProperty, ExecEdge> tuple3 : inputs) {
        ExecNode<?> inputNode = tuple3.f0;
        InputProperty originalInputEdge = tuple3.f1;
        ExecEdge edge = tuple3.f2;
        inputNodes.add(inputNode);
        inputProperties.add(InputProperty.builder().requiredDistribution(originalInputEdge.getRequiredDistribution()).damBehavior(originalInputEdge.getDamBehavior()).priority(inputOrderMap.get(inputNode)).build());
        originalEdges.add(edge);
    }
    String description = ExecNodeUtil.getMultipleInputDescription(rootNode, inputNodes, inputProperties);
    BatchExecMultipleInput multipleInput = new BatchExecMultipleInput(inputProperties, rootNode, originalEdges, description);
    List<ExecEdge> inputEdges = new ArrayList<>(inputNodes.size());
    for (ExecNode<?> inputNode : inputNodes) {
        inputEdges.add(ExecEdge.builder().source(inputNode).target(multipleInput).build());
    }
    multipleInput.setInputEdges(inputEdges);
    return multipleInput;
}

Example 8 with InputProperty

use of org.apache.flink.table.planner.plan.nodes.exec.InputProperty in project flink by apache.

the class ForwardHashExchangeProcessor method process.

@Override
public ExecNodeGraph process(ExecNodeGraph execGraph, ProcessorContext context) {
    if (execGraph.getRootNodes().get(0) instanceof StreamExecNode) {
        throw new TableException("StreamExecNode is not supported yet");
    }
    if (!context.getPlanner().getExecEnv().getConfig().isDynamicGraph()) {
        return execGraph;
    }
    ExecNodeVisitor visitor = new AbstractExecNodeExactlyOnceVisitor() {

        @Override
        protected void visitNode(ExecNode<?> node) {
            visitInputs(node);
            if (node instanceof CommonExecExchange) {
                return;
            }
            boolean changed = false;
            List<ExecEdge> newEdges = new ArrayList<>(node.getInputEdges());
            for (int i = 0; i < node.getInputProperties().size(); ++i) {
                InputProperty inputProperty = node.getInputProperties().get(i);
                RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
                ExecEdge edge = node.getInputEdges().get(i);
                if (requiredDistribution.getType() == DistributionType.SINGLETON) {
                    if (!hasExchangeInput(edge) && isInputSortedNode(node)) {
                        // if operation chaining is disabled, this could mark sure the
                        // sort node and its output can also be connected by
                        // ForwardPartitioner
                        ExecEdge newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                        newEdges.set(i, newEdge);
                        changed = true;
                    }
                    continue;
                }
                if (requiredDistribution.getType() != DistributionType.HASH) {
                    continue;
                }
                if (!hasExchangeInput(edge)) {
                    ExecEdge newEdge;
                    if (isInputSortedNode(node)) {
                        if (hasSortInputForInputSortedNode(node)) {
                            // add Exchange with keep_input_as_is distribution as the
                            // input of Sort
                            ExecNode<?> sort = edge.getSource();
                            ExecEdge newEdgeOfSort = addExchangeAndReconnectEdge(sort.getInputEdges().get(0), inputProperty, false);
                            sort.setInputEdges(Collections.singletonList(newEdgeOfSort));
                        }
                        // if operation chaining is disabled, this could mark sure the
                        // sort node and its output can also be connected by
                        // ForwardPartitioner
                        newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                    } else {
                        // add Exchange with keep_input_as_is distribution as the input
                        // of the node
                        newEdge = addExchangeAndReconnectEdge(edge, inputProperty, false);
                        updateOriginalEdgeInMultipleInput(node, i, (BatchExecExchange) newEdge.getSource());
                    }
                    // update the edge
                    newEdges.set(i, newEdge);
                    changed = true;
                } else if (hasSortInputForInputSortedNode(node)) {
                    // if operation chaining is disabled, this could mark sure the sort
                    // node and its output can also be connected by ForwardPartitioner
                    ExecEdge newEdge = addExchangeAndReconnectEdge(edge, inputProperty, true);
                    newEdges.set(i, newEdge);
                    changed = true;
                }
            }
            if (changed) {
                node.setInputEdges(newEdges);
            }
        }
    };
    execGraph.getRootNodes().forEach(s -> s.accept(visitor));
    return execGraph;
}

Example 9 with InputProperty

use of org.apache.flink.table.planner.plan.nodes.exec.InputProperty in project flink by apache.

the class InputPriorityConflictResolver method createExchange.

private BatchExecExchange createExchange(ExecNode<?> node, int idx) {
    ExecNode<?> inputNode = node.getInputEdges().get(idx).getSource();
    InputProperty inputProperty = node.getInputProperties().get(idx);
    InputProperty.RequiredDistribution requiredDistribution = inputProperty.getRequiredDistribution();
    if (requiredDistribution.getType() == InputProperty.DistributionType.BROADCAST) {
        // should not occur
        throw new IllegalStateException("Trying to resolve input priority conflict on broadcast side. This is not expected.");
    }
    InputProperty newInputProperty = InputProperty.builder().requiredDistribution(requiredDistribution).priority(inputProperty.getPriority()).damBehavior(getDamBehavior()).build();
    BatchExecExchange exchange = new BatchExecExchange(newInputProperty, (RowType) inputNode.getOutputType(), "Exchange");
    exchange.setRequiredExchangeMode(exchangeMode);
    ExecEdge execEdge = ExecEdge.builder().source(inputNode).target(exchange).build();
    exchange.setInputEdges(Collections.singletonList(execEdge));
    return exchange;
}

Example 10 with InputProperty

use of org.apache.flink.table.planner.plan.nodes.exec.InputProperty in project flink by apache.

the class InputPriorityGraphGenerator method calculatePipelinedAncestors.

/**
 * Find the ancestors by going through PIPELINED edges.
 */
@VisibleForTesting
List<ExecNode<?>> calculatePipelinedAncestors(ExecNode<?> node) {
    List<ExecNode<?>> ret = new ArrayList<>();
    AbstractExecNodeExactlyOnceVisitor ancestorVisitor = new AbstractExecNodeExactlyOnceVisitor() {

        @Override
        protected void visitNode(ExecNode<?> node) {
            boolean hasAncestor = false;
            if (!boundaries.contains(node)) {
                List<InputProperty> inputProperties = node.getInputProperties();
                for (int i = 0; i < inputProperties.size(); i++) {
                    // we only go through PIPELINED edges
                    if (inputProperties.get(i).getDamBehavior().stricterOrEqual(safeDamBehavior)) {
                        continue;
                    }
                    hasAncestor = true;
                    node.getInputEdges().get(i).getSource().accept(this);
                }
            }
            if (!hasAncestor) {
                ret.add(node);
            }
        }
    };
    node.accept(ancestorVisitor);
    return ret;
}