Examples with OperatorDescriptorDual org.apache.flink.optimizer.operators.OperatorDescriptorDual used on opensource projects

Example 6 with OperatorDescriptorDual

use of org.apache.flink.optimizer.operators.OperatorDescriptorDual in project flink by apache.

the class CoGroupRawNode method initializeDataProperties.

private List<OperatorDescriptorDual> initializeDataProperties() {
    Ordering groupOrder1 = null;
    Ordering groupOrder2 = null;
    CoGroupRawOperatorBase<?, ?, ?, ?> cgc = getOperator();
    groupOrder1 = cgc.getGroupOrderForInputOne();
    groupOrder2 = cgc.getGroupOrderForInputTwo();
    if (groupOrder1 != null && groupOrder1.getNumberOfFields() == 0) {
        groupOrder1 = null;
    }
    if (groupOrder2 != null && groupOrder2.getNumberOfFields() == 0) {
        groupOrder2 = null;
    }
    return Collections.<OperatorDescriptorDual>singletonList(new CoGroupRawDescriptor(this.keys1, this.keys2, groupOrder1, groupOrder2));
}

Example 7 with OperatorDescriptorDual

use of org.apache.flink.optimizer.operators.OperatorDescriptorDual in project flink by apache.

the class JoinNode method getDataProperties.

private List<OperatorDescriptorDual> getDataProperties(InnerJoinOperatorBase<?, ?, ?, ?> joinOperatorBase, JoinHint joinHint, Partitioner<?> customPartitioner) {
    // see if an internal hint dictates the strategy to use
    Configuration conf = joinOperatorBase.getParameters();
    String localStrategy = conf.getString(Optimizer.HINT_LOCAL_STRATEGY, null);
    if (localStrategy != null) {
        final AbstractJoinDescriptor fixedDriverStrat;
        if (Optimizer.HINT_LOCAL_STRATEGY_SORT_BOTH_MERGE.equals(localStrategy) || Optimizer.HINT_LOCAL_STRATEGY_SORT_FIRST_MERGE.equals(localStrategy) || Optimizer.HINT_LOCAL_STRATEGY_SORT_SECOND_MERGE.equals(localStrategy) || Optimizer.HINT_LOCAL_STRATEGY_MERGE.equals(localStrategy)) {
            fixedDriverStrat = new SortMergeInnerJoinDescriptor(this.keys1, this.keys2);
        } else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_FIRST.equals(localStrategy)) {
            fixedDriverStrat = new HashJoinBuildFirstProperties(this.keys1, this.keys2);
        } else if (Optimizer.HINT_LOCAL_STRATEGY_HASH_BUILD_SECOND.equals(localStrategy)) {
            fixedDriverStrat = new HashJoinBuildSecondProperties(this.keys1, this.keys2);
        } else {
            throw new CompilerException("Invalid local strategy hint for match contract: " + localStrategy);
        }
        if (customPartitioner != null) {
            fixedDriverStrat.setCustomPartitioner(customPartitioner);
        }
        ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
        list.add(fixedDriverStrat);
        return list;
    } else {
        ArrayList<OperatorDescriptorDual> list = new ArrayList<OperatorDescriptorDual>();
        joinHint = joinHint == null ? JoinHint.OPTIMIZER_CHOOSES : joinHint;
        switch(joinHint) {
            case BROADCAST_HASH_FIRST:
                list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2, true, false, false));
                break;
            case BROADCAST_HASH_SECOND:
                list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2, false, true, false));
                break;
            case REPARTITION_HASH_FIRST:
                list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2, false, false, true));
                break;
            case REPARTITION_HASH_SECOND:
                list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2, false, false, true));
                break;
            case REPARTITION_SORT_MERGE:
                list.add(new SortMergeInnerJoinDescriptor(this.keys1, this.keys2, false, false, true));
                break;
            case OPTIMIZER_CHOOSES:
                list.add(new SortMergeInnerJoinDescriptor(this.keys1, this.keys2));
                list.add(new HashJoinBuildFirstProperties(this.keys1, this.keys2));
                list.add(new HashJoinBuildSecondProperties(this.keys1, this.keys2));
                break;
            default:
                throw new CompilerException("Unrecognized join hint: " + joinHint);
        }
        if (customPartitioner != null) {
            for (OperatorDescriptorDual descr : list) {
                ((AbstractJoinDescriptor) descr).setCustomPartitioner(customPartitioner);
            }
        }
        return list;
    }
}

Example 8 with OperatorDescriptorDual

use of org.apache.flink.optimizer.operators.OperatorDescriptorDual in project flink by apache.

the class TwoInputNode method addLocalCandidates.

protected void addLocalCandidates(Channel template1, Channel template2, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps1, RequestedGlobalProperties rgps2, List<PlanNode> target, LocalPropertiesPair[] validLocalCombinations, CostEstimator estimator) {
    for (RequestedLocalProperties ilp1 : this.input1.getInterestingProperties().getLocalProperties()) {
        final Channel in1 = template1.clone();
        ilp1.parameterizeChannel(in1);
        for (RequestedLocalProperties ilp2 : this.input2.getInterestingProperties().getLocalProperties()) {
            final Channel in2 = template2.clone();
            ilp2.parameterizeChannel(in2);
            for (OperatorDescriptorDual dps : getProperties()) {
                for (LocalPropertiesPair lpp : dps.getPossibleLocalProperties()) {
                    if (lpp.getProperties1().isMetBy(in1.getLocalProperties()) && lpp.getProperties2().isMetBy(in2.getLocalProperties())) {
                        // (such as when some sort order is requested, that both are the same sort order
                        if (dps.areCoFulfilled(lpp.getProperties1(), lpp.getProperties2(), in1.getLocalProperties(), in2.getLocalProperties())) {
                            // copy, because setting required properties and instantiation may
                            // change the channels and should not affect prior candidates
                            Channel in1Copy = in1.clone();
                            in1Copy.setRequiredLocalProps(lpp.getProperties1());
                            Channel in2Copy = in2.clone();
                            in2Copy.setRequiredLocalProps(lpp.getProperties2());
                            // all right, co compatible
                            instantiate(dps, in1Copy, in2Copy, broadcastPlanChannels, target, estimator, rgps1, rgps2, ilp1, ilp2);
                            break;
                        }
                    // else cannot use this pair, fall through the loop and try the next one
                    }
                }
            }
        }
    }
}

Example 9 with OperatorDescriptorDual

use of org.apache.flink.optimizer.operators.OperatorDescriptorDual in project flink by apache.

the class TwoInputNode method getAlternativePlans.

@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
    // check if we have a cached version
    if (this.cachedPlans != null) {
        return this.cachedPlans;
    }
    boolean childrenSkippedDueToReplicatedInput = false;
    // step down to all producer nodes and calculate alternative plans
    final List<? extends PlanNode> subPlans1 = getFirstPredecessorNode().getAlternativePlans(estimator);
    final List<? extends PlanNode> subPlans2 = getSecondPredecessorNode().getAlternativePlans(estimator);
    // calculate alternative sub-plans for predecessor
    final Set<RequestedGlobalProperties> intGlobal1 = this.input1.getInterestingProperties().getGlobalProperties();
    final Set<RequestedGlobalProperties> intGlobal2 = this.input2.getInterestingProperties().getGlobalProperties();
    // calculate alternative sub-plans for broadcast inputs
    final List<Set<? extends NamedChannel>> broadcastPlanChannels = new ArrayList<Set<? extends NamedChannel>>();
    List<DagConnection> broadcastConnections = getBroadcastConnections();
    List<String> broadcastConnectionNames = getBroadcastConnectionNames();
    for (int i = 0; i < broadcastConnections.size(); i++) {
        DagConnection broadcastConnection = broadcastConnections.get(i);
        String broadcastConnectionName = broadcastConnectionNames.get(i);
        List<PlanNode> broadcastPlanCandidates = broadcastConnection.getSource().getAlternativePlans(estimator);
        // wrap the plan candidates in named channels
        HashSet<NamedChannel> broadcastChannels = new HashSet<NamedChannel>(broadcastPlanCandidates.size());
        for (PlanNode plan : broadcastPlanCandidates) {
            final NamedChannel c = new NamedChannel(broadcastConnectionName, plan);
            DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(), ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline());
            c.setShipStrategy(ShipStrategyType.BROADCAST, exMode);
            broadcastChannels.add(c);
        }
        broadcastPlanChannels.add(broadcastChannels);
    }
    final GlobalPropertiesPair[] allGlobalPairs;
    final LocalPropertiesPair[] allLocalPairs;
    {
        Set<GlobalPropertiesPair> pairsGlob = new HashSet<GlobalPropertiesPair>();
        Set<LocalPropertiesPair> pairsLoc = new HashSet<LocalPropertiesPair>();
        for (OperatorDescriptorDual ods : getProperties()) {
            pairsGlob.addAll(ods.getPossibleGlobalProperties());
            pairsLoc.addAll(ods.getPossibleLocalProperties());
        }
        allGlobalPairs = pairsGlob.toArray(new GlobalPropertiesPair[pairsGlob.size()]);
        allLocalPairs = pairsLoc.toArray(new LocalPropertiesPair[pairsLoc.size()]);
    }
    final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>();
    final ExecutionMode input1Mode = this.input1.getDataExchangeMode();
    final ExecutionMode input2Mode = this.input2.getDataExchangeMode();
    final int parallelism = getParallelism();
    final int inParallelism1 = getFirstPredecessorNode().getParallelism();
    final int inParallelism2 = getSecondPredecessorNode().getParallelism();
    final boolean dopChange1 = parallelism != inParallelism1;
    final boolean dopChange2 = parallelism != inParallelism2;
    final boolean input1breaksPipeline = this.input1.isBreakingPipeline();
    final boolean input2breaksPipeline = this.input2.isBreakingPipeline();
    // create all candidates
    for (PlanNode child1 : subPlans1) {
        if (child1.getGlobalProperties().isFullyReplicated()) {
            // fully replicated input is always locally forwarded if parallelism is not changed
            if (dopChange1) {
                // can not continue with this child
                childrenSkippedDueToReplicatedInput = true;
                continue;
            } else {
                this.input1.setShipStrategy(ShipStrategyType.FORWARD);
            }
        }
        for (PlanNode child2 : subPlans2) {
            if (child2.getGlobalProperties().isFullyReplicated()) {
                // fully replicated input is always locally forwarded if parallelism is not changed
                if (dopChange2) {
                    // can not continue with this child
                    childrenSkippedDueToReplicatedInput = true;
                    continue;
                } else {
                    this.input2.setShipStrategy(ShipStrategyType.FORWARD);
                }
            }
            // candidate at the joined branch plan. 
            if (!areBranchCompatible(child1, child2)) {
                continue;
            }
            for (RequestedGlobalProperties igps1 : intGlobal1) {
                // create a candidate channel for the first input. mark it cached, if the connection says so
                final Channel c1 = new Channel(child1, this.input1.getMaterializationMode());
                if (this.input1.getShipStrategy() == null) {
                    // free to choose the ship strategy
                    igps1.parameterizeChannel(c1, dopChange1, input1Mode, input1breaksPipeline);
                    // ship strategy preserves/establishes them even under changing parallelisms
                    if (dopChange1 && !c1.getShipStrategy().isNetworkStrategy()) {
                        c1.getGlobalProperties().reset();
                    }
                } else {
                    // ship strategy fixed by compiler hint
                    ShipStrategyType shipType = this.input1.getShipStrategy();
                    DataExchangeMode exMode = DataExchangeMode.select(input1Mode, shipType, input1breaksPipeline);
                    if (this.keys1 != null) {
                        c1.setShipStrategy(shipType, this.keys1.toFieldList(), exMode);
                    } else {
                        c1.setShipStrategy(shipType, exMode);
                    }
                    if (dopChange1) {
                        c1.adjustGlobalPropertiesForFullParallelismChange();
                    }
                }
                for (RequestedGlobalProperties igps2 : intGlobal2) {
                    // create a candidate channel for the first input. mark it cached, if the connection says so
                    final Channel c2 = new Channel(child2, this.input2.getMaterializationMode());
                    if (this.input2.getShipStrategy() == null) {
                        // free to choose the ship strategy
                        igps2.parameterizeChannel(c2, dopChange2, input2Mode, input2breaksPipeline);
                        // ship strategy preserves/establishes them even under changing parallelisms
                        if (dopChange2 && !c2.getShipStrategy().isNetworkStrategy()) {
                            c2.getGlobalProperties().reset();
                        }
                    } else {
                        // ship strategy fixed by compiler hint
                        ShipStrategyType shipType = this.input2.getShipStrategy();
                        DataExchangeMode exMode = DataExchangeMode.select(input2Mode, shipType, input2breaksPipeline);
                        if (this.keys2 != null) {
                            c2.setShipStrategy(shipType, this.keys2.toFieldList(), exMode);
                        } else {
                            c2.setShipStrategy(shipType, exMode);
                        }
                        if (dopChange2) {
                            c2.adjustGlobalPropertiesForFullParallelismChange();
                        }
                    }
                    outer: for (GlobalPropertiesPair gpp : allGlobalPairs) {
                        if (gpp.getProperties1().isMetBy(c1.getGlobalProperties()) && gpp.getProperties2().isMetBy(c2.getGlobalProperties())) {
                            for (OperatorDescriptorDual desc : getProperties()) {
                                if (desc.areCompatible(gpp.getProperties1(), gpp.getProperties2(), c1.getGlobalProperties(), c2.getGlobalProperties())) {
                                    Channel c1Clone = c1.clone();
                                    c1Clone.setRequiredGlobalProps(gpp.getProperties1());
                                    c2.setRequiredGlobalProps(gpp.getProperties2());
                                    // we form a valid combination, so create the local candidates
                                    // for this
                                    addLocalCandidates(c1Clone, c2, broadcastPlanChannels, igps1, igps2, outputPlans, allLocalPairs, estimator);
                                    break outer;
                                }
                            }
                        }
                    }
                    // so we can stop after the first
                    if (this.input2.getShipStrategy() != null) {
                        break;
                    }
                }
                // so we can stop after the first
                if (this.input1.getShipStrategy() != null) {
                    break;
                }
            }
        }
    }
    if (outputPlans.isEmpty()) {
        if (childrenSkippedDueToReplicatedInput) {
            throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Invalid use of replicated input.");
        } else {
            throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Too restrictive plan hints.");
        }
    }
    // cost and prune the plans
    for (PlanNode node : outputPlans) {
        estimator.costOperator(node);
    }
    prunePlanAlternatives(outputPlans);
    outputPlans.trimToSize();
    this.cachedPlans = outputPlans;
    return outputPlans;
}

Example 10 with OperatorDescriptorDual

use of org.apache.flink.optimizer.operators.OperatorDescriptorDual in project flink by apache.

the class OuterJoinNode method createLeftOuterJoinDescriptors.

private List<OperatorDescriptorDual> createLeftOuterJoinDescriptors(JoinHint hint) {
    List<OperatorDescriptorDual> list = new ArrayList<>();
    switch(hint) {
        case OPTIMIZER_CHOOSES:
            list.add(new SortMergeLeftOuterJoinDescriptor(this.keys1, this.keys2, true));
            list.add(new HashLeftOuterJoinBuildSecondDescriptor(this.keys1, this.keys2, true, true));
            break;
        case REPARTITION_SORT_MERGE:
            list.add(new SortMergeLeftOuterJoinDescriptor(this.keys1, this.keys2, false));
            break;
        case REPARTITION_HASH_SECOND:
            list.add(new HashLeftOuterJoinBuildSecondDescriptor(this.keys1, this.keys2, false, true));
            break;
        case BROADCAST_HASH_SECOND:
            list.add(new HashLeftOuterJoinBuildSecondDescriptor(this.keys1, this.keys2, true, false));
            break;
        case REPARTITION_HASH_FIRST:
            list.add(new HashLeftOuterJoinBuildFirstDescriptor(this.keys1, this.keys2, false, true));
            break;
        case BROADCAST_HASH_FIRST:
        default:
            throw new CompilerException("Invalid join hint: " + hint + " for left outer join");
    }
    return list;
}