Examples with SourceAndDamReport org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport used on opensource projects

Example 1 with SourceAndDamReport

use of org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport in project flink by apache.

the class SingleInputNode method instantiateCandidate.

protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels, List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq) {
    final PlanNode inputSource = in.getSource();
    for (List<NamedChannel> broadcastChannelsCombination : Sets.cartesianProduct(broadcastPlanChannels)) {
        boolean validCombination = true;
        boolean requiresPipelinebreaker = false;
        // check whether the broadcast inputs use the same plan candidate at the branching point
        for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
            NamedChannel nc = broadcastChannelsCombination.get(i);
            PlanNode bcSource = nc.getSource();
            // check branch compatibility against input
            if (!areBranchCompatible(bcSource, inputSource)) {
                validCombination = false;
                break;
            }
            // check branch compatibility against all other broadcast variables
            for (int k = 0; k < i; k++) {
                PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
                if (!areBranchCompatible(bcSource, otherBcSource)) {
                    validCombination = false;
                    break;
                }
            }
            // check if there is a common predecessor and whether there is a dam on the way to all common predecessors
            if (in.isOnDynamicPath() && this.hereJoinedBranches != null) {
                for (OptimizerNode brancher : this.hereJoinedBranches) {
                    PlanNode candAtBrancher = in.getSource().getCandidateAtBranchPoint(brancher);
                    if (candAtBrancher == null) {
                        // closed branch between two broadcast variables
                        continue;
                    }
                    SourceAndDamReport res = in.getSource().hasDamOnPathDownTo(candAtBrancher);
                    if (res == NOT_FOUND) {
                        throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
                    } else if (res == FOUND_SOURCE) {
                        requiresPipelinebreaker = true;
                        break;
                    } else if (res == FOUND_SOURCE_AND_DAM) {
                    // good
                    } else {
                        throw new CompilerException();
                    }
                }
            }
        }
        if (!validCombination) {
            continue;
        }
        if (requiresPipelinebreaker) {
            in.setTempMode(in.getTempMode().makePipelineBreaker());
        }
        final SingleInputPlanNode node = dps.instantiate(in, this);
        node.setBroadcastInputs(broadcastChannelsCombination);
        // compute how the strategy affects the properties
        GlobalProperties gProps = in.getGlobalProperties().clone();
        LocalProperties lProps = in.getLocalProperties().clone();
        gProps = dps.computeGlobalProperties(gProps);
        lProps = dps.computeLocalProperties(lProps);
        // filter by the user code field copies
        gProps = gProps.filterBySemanticProperties(getSemanticPropertiesForGlobalPropertyFiltering(), 0);
        lProps = lProps.filterBySemanticProperties(getSemanticPropertiesForLocalPropertyFiltering(), 0);
        // apply
        node.initProperties(gProps, lProps);
        node.updatePropertiesWithUniqueSets(getUniqueFields());
        target.add(node);
    }
}

Example 2 with SourceAndDamReport

use of org.apache.flink.optimizer.plan.PlanNode.SourceAndDamReport in project flink by apache.

the class TwoInputNode method placePipelineBreakersIfNecessary.

protected void placePipelineBreakersIfNecessary(DriverStrategy strategy, Channel in1, Channel in2) {
    // whether either no input, or all of them have a dam
    if (in1.isOnDynamicPath() && in2.isOnDynamicPath() && this.hereJoinedBranches != null && this.hereJoinedBranches.size() > 0) {
        boolean someDamOnLeftPaths = false;
        boolean damOnAllLeftPaths = true;
        boolean someDamOnRightPaths = false;
        boolean damOnAllRightPaths = true;
        if (strategy.firstDam() == DamBehavior.FULL_DAM || in1.getLocalStrategy().dams() || in1.getTempMode().breaksPipeline()) {
            someDamOnLeftPaths = true;
        } else {
            for (OptimizerNode brancher : this.hereJoinedBranches) {
                PlanNode candAtBrancher = in1.getSource().getCandidateAtBranchPoint(brancher);
                // not all candidates are found, because this list includes joined branched from both regular inputs and broadcast vars
                if (candAtBrancher == null) {
                    continue;
                }
                SourceAndDamReport res = in1.getSource().hasDamOnPathDownTo(candAtBrancher);
                if (res == NOT_FOUND) {
                    throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
                } else if (res == FOUND_SOURCE) {
                    damOnAllLeftPaths = false;
                } else if (res == FOUND_SOURCE_AND_DAM) {
                    someDamOnLeftPaths = true;
                } else {
                    throw new CompilerException();
                }
            }
        }
        if (strategy.secondDam() == DamBehavior.FULL_DAM || in2.getLocalStrategy().dams() || in2.getTempMode().breaksPipeline()) {
            someDamOnRightPaths = true;
        } else {
            for (OptimizerNode brancher : this.hereJoinedBranches) {
                PlanNode candAtBrancher = in2.getSource().getCandidateAtBranchPoint(brancher);
                // not all candidates are found, because this list includes joined branched from both regular inputs and broadcast vars
                if (candAtBrancher == null) {
                    continue;
                }
                SourceAndDamReport res = in2.getSource().hasDamOnPathDownTo(candAtBrancher);
                if (res == NOT_FOUND) {
                    throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
                } else if (res == FOUND_SOURCE) {
                    damOnAllRightPaths = false;
                } else if (res == FOUND_SOURCE_AND_DAM) {
                    someDamOnRightPaths = true;
                } else {
                    throw new CompilerException();
                }
            }
        }
        // okay combinations are both all dam or both no dam
        if ((damOnAllLeftPaths & damOnAllRightPaths) | (!someDamOnLeftPaths & !someDamOnRightPaths)) {
        // good, either both materialize already on the way, or both fully pipeline
        } else {
            if (someDamOnLeftPaths & !damOnAllRightPaths) {
                // right needs a pipeline breaker
                in2.setTempMode(in2.getTempMode().makePipelineBreaker());
            }
            if (someDamOnRightPaths & !damOnAllLeftPaths) {
                // right needs a pipeline breaker
                in1.setTempMode(in1.getTempMode().makePipelineBreaker());
            }
        }
    }
}