Examples with RequestedGlobalProperties org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties used on opensource projects

Example 16 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class FlatMapDescriptor method createPossibleGlobalProperties.

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
    RequestedGlobalProperties rgp = new RequestedGlobalProperties();
    rgp.setAnyDistribution();
    return Collections.singletonList(rgp);
}

Example 17 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class BinaryUnionNode method computeInterestingPropertiesForInputs.

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
    final InterestingProperties props = getInterestingProperties();
    // if no other properties exist, add the pruned trivials back
    if (props.getGlobalProperties().isEmpty()) {
        props.addGlobalProperties(new RequestedGlobalProperties());
    }
    props.addLocalProperties(new RequestedLocalProperties());
    this.input1.setInterestingProperties(props.clone());
    this.input2.setInterestingProperties(props.clone());
    this.channelProps = props.getGlobalProperties();
}

Example 18 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class BinaryUnionNode method getAlternativePlans.

@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
    // check that union has only a single successor
    if (this.getOutgoingConnections().size() > 1) {
        throw new CompilerException("BinaryUnionNode has more than one successor.");
    }
    boolean childrenSkippedDueToReplicatedInput = false;
    // check if we have a cached version
    if (this.cachedPlans != null) {
        return this.cachedPlans;
    }
    // step down to all producer nodes and calculate alternative plans
    final List<? extends PlanNode> subPlans1 = getFirstPredecessorNode().getAlternativePlans(estimator);
    final List<? extends PlanNode> subPlans2 = getSecondPredecessorNode().getAlternativePlans(estimator);
    List<DagConnection> broadcastConnections = getBroadcastConnections();
    if (broadcastConnections != null && broadcastConnections.size() > 0) {
        throw new CompilerException("Found BroadcastVariables on a Union operation");
    }
    final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>();
    final List<Set<? extends NamedChannel>> broadcastPlanChannels = Collections.emptyList();
    final BinaryUnionOpDescriptor operator = new BinaryUnionOpDescriptor();
    final RequestedLocalProperties noLocalProps = new RequestedLocalProperties();
    final ExecutionMode input1Mode = this.input1.getDataExchangeMode();
    final ExecutionMode input2Mode = this.input2.getDataExchangeMode();
    final int parallelism = getParallelism();
    final int inParallelism1 = getFirstPredecessorNode().getParallelism();
    final int inParallelism2 = getSecondPredecessorNode().getParallelism();
    final boolean dopChange1 = parallelism != inParallelism1;
    final boolean dopChange2 = parallelism != inParallelism2;
    final boolean input1breakPipeline = this.input1.isBreakingPipeline();
    final boolean input2breakPipeline = this.input2.isBreakingPipeline();
    // create all candidates
    for (PlanNode child1 : subPlans1) {
        if (child1.getGlobalProperties().isFullyReplicated()) {
            // fully replicated input is always locally forwarded if parallelism is not changed
            if (dopChange1) {
                // can not continue with this child
                childrenSkippedDueToReplicatedInput = true;
                continue;
            } else {
                this.input1.setShipStrategy(ShipStrategyType.FORWARD);
            }
        }
        for (PlanNode child2 : subPlans2) {
            if (child2.getGlobalProperties().isFullyReplicated()) {
                // fully replicated input is always locally forwarded if parallelism is not changed
                if (dopChange2) {
                    // can not continue with this child
                    childrenSkippedDueToReplicatedInput = true;
                    continue;
                } else {
                    this.input2.setShipStrategy(ShipStrategyType.FORWARD);
                }
            }
            // candidate at the joined branch plan. 
            if (!areBranchCompatible(child1, child2)) {
                continue;
            }
            for (RequestedGlobalProperties igps : this.channelProps) {
                // create a candidate channel for the first input. mark it cached, if the connection says so
                Channel c1 = new Channel(child1, this.input1.getMaterializationMode());
                if (this.input1.getShipStrategy() == null) {
                    // free to choose the ship strategy
                    igps.parameterizeChannel(c1, dopChange1, input1Mode, input1breakPipeline);
                    // ship strategy preserves/establishes them even under changing parallelisms
                    if (dopChange1 && !c1.getShipStrategy().isNetworkStrategy()) {
                        c1.getGlobalProperties().reset();
                    }
                } else {
                    // ship strategy fixed by compiler hint
                    ShipStrategyType shipStrategy = this.input1.getShipStrategy();
                    DataExchangeMode exMode = DataExchangeMode.select(input1Mode, shipStrategy, input1breakPipeline);
                    if (this.keys1 != null) {
                        c1.setShipStrategy(this.input1.getShipStrategy(), this.keys1.toFieldList(), exMode);
                    } else {
                        c1.setShipStrategy(this.input1.getShipStrategy(), exMode);
                    }
                    if (dopChange1) {
                        c1.adjustGlobalPropertiesForFullParallelismChange();
                    }
                }
                // create a candidate channel for the second input. mark it cached, if the connection says so
                Channel c2 = new Channel(child2, this.input2.getMaterializationMode());
                if (this.input2.getShipStrategy() == null) {
                    // free to choose the ship strategy
                    igps.parameterizeChannel(c2, dopChange2, input2Mode, input2breakPipeline);
                    // ship strategy preserves/establishes them even under changing parallelisms
                    if (dopChange2 && !c2.getShipStrategy().isNetworkStrategy()) {
                        c2.getGlobalProperties().reset();
                    }
                } else {
                    // ship strategy fixed by compiler hint
                    ShipStrategyType shipStrategy = this.input2.getShipStrategy();
                    DataExchangeMode exMode = DataExchangeMode.select(input2Mode, shipStrategy, input2breakPipeline);
                    if (this.keys2 != null) {
                        c2.setShipStrategy(this.input2.getShipStrategy(), this.keys2.toFieldList(), exMode);
                    } else {
                        c2.setShipStrategy(this.input2.getShipStrategy(), exMode);
                    }
                    if (dopChange2) {
                        c2.adjustGlobalPropertiesForFullParallelismChange();
                    }
                }
                // get the global properties and clear unique fields (not preserved anyways during the union)
                GlobalProperties p1 = c1.getGlobalProperties();
                GlobalProperties p2 = c2.getGlobalProperties();
                p1.clearUniqueFieldCombinations();
                p2.clearUniqueFieldCombinations();
                // partitioned on that field. 
                if (!igps.isTrivial() && !(p1.equals(p2))) {
                    if (c1.getShipStrategy() == ShipStrategyType.FORWARD && c2.getShipStrategy() != ShipStrategyType.FORWARD) {
                        // adjust c2 to c1
                        c2 = c2.clone();
                        p1.parameterizeChannel(c2, dopChange2, input2Mode, input2breakPipeline);
                    } else if (c2.getShipStrategy() == ShipStrategyType.FORWARD && c1.getShipStrategy() != ShipStrategyType.FORWARD) {
                        // adjust c1 to c2
                        c1 = c1.clone();
                        p2.parameterizeChannel(c1, dopChange1, input1Mode, input1breakPipeline);
                    } else if (c1.getShipStrategy() == ShipStrategyType.FORWARD && c2.getShipStrategy() == ShipStrategyType.FORWARD) {
                        boolean adjustC1 = c1.getEstimatedOutputSize() <= 0 || c2.getEstimatedOutputSize() <= 0 || c1.getEstimatedOutputSize() <= c2.getEstimatedOutputSize();
                        if (adjustC1) {
                            c2 = c2.clone();
                            p1.parameterizeChannel(c2, dopChange2, input2Mode, input2breakPipeline);
                        } else {
                            c1 = c1.clone();
                            p2.parameterizeChannel(c1, dopChange1, input1Mode, input1breakPipeline);
                        }
                    } else {
                        // excluded by the check that the required strategies must match
                        throw new CompilerException("Bug in Plan Enumeration for Union Node.");
                    }
                }
                instantiate(operator, c1, c2, broadcastPlanChannels, outputPlans, estimator, igps, igps, noLocalProps, noLocalProps);
            }
        }
    }
    if (outputPlans.isEmpty()) {
        if (childrenSkippedDueToReplicatedInput) {
            throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Invalid use of replicated input.");
        } else {
            throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Too restrictive plan hints.");
        }
    }
    // cost and prune the plans
    for (PlanNode node : outputPlans) {
        estimator.costOperator(node);
    }
    prunePlanAlternatives(outputPlans);
    outputPlans.trimToSize();
    this.cachedPlans = outputPlans;
    return outputPlans;
}

Example 19 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class BulkIterationNode method computeInterestingPropertiesForInputs.

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
    final InterestingProperties intProps = getInterestingProperties().clone();
    if (this.terminationCriterion != null) {
        // first propagate through termination Criterion. since it has no successors, it has no
        // interesting properties
        this.terminationCriterionRootConnection.setInterestingProperties(new InterestingProperties());
        this.terminationCriterion.accept(new InterestingPropertyVisitor(estimator));
    }
    // we need to make 2 interesting property passes, because the root of the step function needs also
    // the interesting properties as generated by the partial solution
    // give our own interesting properties (as generated by the iterations successors) to the step function and
    // make the first pass
    this.rootConnection.setInterestingProperties(intProps);
    this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator));
    // take the interesting properties of the partial solution and add them to the root interesting properties
    InterestingProperties partialSolutionIntProps = this.partialSolution.getInterestingProperties();
    intProps.getGlobalProperties().addAll(partialSolutionIntProps.getGlobalProperties());
    intProps.getLocalProperties().addAll(partialSolutionIntProps.getLocalProperties());
    // clear all interesting properties to prepare the second traversal
    // this clears only the path down from the next partial solution. The paths down
    // from the termination criterion (before they meet the paths down from the next partial solution)
    // remain unaffected by this step
    this.rootConnection.clearInterestingProperties();
    this.nextPartialSolution.accept(InterestingPropertiesClearer.INSTANCE);
    // 2nd pass
    this.rootConnection.setInterestingProperties(intProps);
    this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator));
    // now add the interesting properties of the partial solution to the input
    final InterestingProperties inProps = this.partialSolution.getInterestingProperties().clone();
    inProps.addGlobalProperties(new RequestedGlobalProperties());
    inProps.addLocalProperties(new RequestedLocalProperties());
    this.inConn.setInterestingProperties(inProps);
}

Example 20 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class DataSinkNode method getAlternativePlans.

// --------------------------------------------------------------------------------------------
//                                   Recursive Optimization
// --------------------------------------------------------------------------------------------
@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
    // check if we have a cached version
    if (this.cachedPlans != null) {
        return this.cachedPlans;
    }
    // calculate alternative sub-plans for predecessor
    List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
    List<PlanNode> outputPlans = new ArrayList<PlanNode>();
    final int parallelism = getParallelism();
    final int inDop = getPredecessorNode().getParallelism();
    final ExecutionMode executionMode = this.input.getDataExchangeMode();
    final boolean dopChange = parallelism != inDop;
    final boolean breakPipeline = this.input.isBreakingPipeline();
    InterestingProperties ips = this.input.getInterestingProperties();
    for (PlanNode p : subPlans) {
        for (RequestedGlobalProperties gp : ips.getGlobalProperties()) {
            for (RequestedLocalProperties lp : ips.getLocalProperties()) {
                Channel c = new Channel(p);
                gp.parameterizeChannel(c, dopChange, executionMode, breakPipeline);
                lp.parameterizeChannel(c);
                c.setRequiredLocalProps(lp);
                c.setRequiredGlobalProps(gp);
                // no need to check whether the created properties meet what we need in case
                // of ordering or global ordering, because the only interesting properties we have
                // are what we require
                outputPlans.add(new SinkPlanNode(this, "DataSink (" + this.getOperator().getName() + ")", c));
            }
        }
    }
    // cost and prune the plans
    for (PlanNode node : outputPlans) {
        estimator.costOperator(node);
    }
    prunePlanAlternatives(outputPlans);
    this.cachedPlans = outputPlans;
    return outputPlans;
}