Examples with RequestedGlobalProperties org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties used on opensource projects

Example 1 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class CoGroupGlobalPropertiesCompatibilityTest method checkCompatiblePartitionings.

@Test
public void checkCompatiblePartitionings() {
    try {
        final FieldList keysLeft = new FieldList(1, 4);
        final FieldList keysRight = new FieldList(3, 1);
        CoGroupDescriptor descr = new CoGroupDescriptor(keysLeft, keysRight);
        // test compatible hash partitioning
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setHashPartitioned(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setHashPartitioned(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setHashPartitioned(keysLeft);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setHashPartitioned(keysRight);
            assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test compatible custom partitioning
        {
            Partitioner<Object> part = new Partitioner<Object>() {

                @Override
                public int partition(Object key, int numPartitions) {
                    return 0;
                }
            };
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setCustomPartitioned(keysLeft, part);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setCustomPartitioned(keysRight, part);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setCustomPartitioned(keysLeft, part);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part);
            assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test custom partitioning matching any partitioning
        {
            Partitioner<Object> part = new Partitioner<Object>() {

                @Override
                public int partition(Object key, int numPartitions) {
                    return 0;
                }
            };
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setCustomPartitioned(keysLeft, part);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part);
            assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist1 = new TestDistribution(1);
        TestDistribution dist2 = new TestDistribution(1);
        // test compatible range partitioning with one ordering
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test compatible range partitioning with two orderings
        {
            Ordering ordering1 = new Ordering();
            ordering1.appendOrdering(keysLeft.get(0), null, Order.DESCENDING);
            ordering1.appendOrdering(keysLeft.get(1), null, Order.ASCENDING);
            Ordering ordering2 = new Ordering();
            ordering2.appendOrdering(keysRight.get(0), null, Order.DESCENDING);
            ordering2.appendOrdering(keysRight.get(1), null, Order.ASCENDING);
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class JoinGlobalPropertiesCompatibilityTest method checkCompatiblePartitionings.

@Test
public void checkCompatiblePartitionings() {
    try {
        final FieldList keysLeft = new FieldList(1, 4);
        final FieldList keysRight = new FieldList(3, 1);
        SortMergeInnerJoinDescriptor descr = new SortMergeInnerJoinDescriptor(keysLeft, keysRight);
        // test compatible hash partitioning
        {
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setHashPartitioned(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setHashPartitioned(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setHashPartitioned(keysLeft);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setHashPartitioned(keysRight);
            assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test compatible custom partitioning
        {
            Partitioner<Object> part = new Partitioner<Object>() {

                @Override
                public int partition(Object key, int numPartitions) {
                    return 0;
                }
            };
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setCustomPartitioned(keysLeft, part);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setCustomPartitioned(keysRight, part);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setCustomPartitioned(keysLeft, part);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part);
            assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test custom partitioning matching any partitioning
        {
            Partitioner<Object> part = new Partitioner<Object>() {

                @Override
                public int partition(Object key, int numPartitions) {
                    return 0;
                }
            };
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setAnyPartitioning(keysLeft);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setAnyPartitioning(keysRight);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setCustomPartitioned(keysLeft, part);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setCustomPartitioned(keysRight, part);
            assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        TestDistribution dist1 = new TestDistribution(1);
        TestDistribution dist2 = new TestDistribution(1);
        // test compatible range partitioning with one ordering
        {
            Ordering ordering1 = new Ordering();
            for (int field : keysLeft) {
                ordering1.appendOrdering(field, null, Order.ASCENDING);
            }
            Ordering ordering2 = new Ordering();
            for (int field : keysRight) {
                ordering2.appendOrdering(field, null, Order.ASCENDING);
            }
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
        // test compatible range partitioning with two orderings
        {
            Ordering ordering1 = new Ordering();
            ordering1.appendOrdering(keysLeft.get(0), null, Order.DESCENDING);
            ordering1.appendOrdering(keysLeft.get(1), null, Order.ASCENDING);
            Ordering ordering2 = new Ordering();
            ordering2.appendOrdering(keysRight.get(0), null, Order.DESCENDING);
            ordering2.appendOrdering(keysRight.get(1), null, Order.ASCENDING);
            RequestedGlobalProperties reqLeft = new RequestedGlobalProperties();
            reqLeft.setRangePartitioned(ordering1, dist1);
            RequestedGlobalProperties reqRight = new RequestedGlobalProperties();
            reqRight.setRangePartitioned(ordering2, dist2);
            GlobalProperties propsLeft = new GlobalProperties();
            propsLeft.setRangePartitioned(ordering1, dist1);
            GlobalProperties propsRight = new GlobalProperties();
            propsRight.setRangePartitioned(ordering2, dist2);
            assertTrue(descr.areCompatible(reqLeft, reqRight, propsLeft, propsRight));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class OptimizerNode method prunePlanAlternativesWithCommonBranching.

protected void prunePlanAlternativesWithCommonBranching(List<PlanNode> plans) {
    // for each interesting property, which plans are cheapest
    final RequestedGlobalProperties[] gps = this.intProps.getGlobalProperties().toArray(new RequestedGlobalProperties[this.intProps.getGlobalProperties().size()]);
    final RequestedLocalProperties[] lps = this.intProps.getLocalProperties().toArray(new RequestedLocalProperties[this.intProps.getLocalProperties().size()]);
    final PlanNode[][] toKeep = new PlanNode[gps.length][];
    final PlanNode[] cheapestForGlobal = new PlanNode[gps.length];
    // the overall cheapest plan
    PlanNode cheapest = null;
    // go over all plans from the list
    for (PlanNode candidate : plans) {
        // check if that plan is the overall cheapest
        if (cheapest == null || (cheapest.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) {
            cheapest = candidate;
        }
        // find the interesting global properties that this plan matches
        for (int i = 0; i < gps.length; i++) {
            if (gps[i].isMetBy(candidate.getGlobalProperties())) {
                if (cheapestForGlobal[i] == null || (cheapestForGlobal[i].getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0)) {
                    cheapestForGlobal[i] = candidate;
                }
                final PlanNode[] localMatches;
                if (toKeep[i] == null) {
                    localMatches = new PlanNode[lps.length];
                    toKeep[i] = localMatches;
                } else {
                    localMatches = toKeep[i];
                }
                for (int k = 0; k < lps.length; k++) {
                    if (lps[k].isMetBy(candidate.getLocalProperties())) {
                        final PlanNode previous = localMatches[k];
                        if (previous == null || previous.getCumulativeCosts().compareTo(candidate.getCumulativeCosts()) > 0) {
                            // this one is cheaper!
                            localMatches[k] = candidate;
                        }
                    }
                }
            }
        }
    }
    // all plans are set now
    plans.clear();
    // add the cheapest plan
    if (cheapest != null) {
        plans.add(cheapest);
        // remember that that plan is in the set
        cheapest.setPruningMarker();
    }
    // add all others, which are optimal for some interesting properties
    for (int i = 0; i < gps.length; i++) {
        if (toKeep[i] != null) {
            final PlanNode[] localMatches = toKeep[i];
            for (final PlanNode n : localMatches) {
                if (n != null && !n.isPruneMarkerSet()) {
                    n.setPruningMarker();
                    plans.add(n);
                }
            }
        }
        if (cheapestForGlobal[i] != null) {
            final PlanNode n = cheapestForGlobal[i];
            if (!n.isPruneMarkerSet()) {
                n.setPruningMarker();
                plans.add(n);
            }
        }
    }
}

Example 4 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class SingleInputNode method getAlternativePlans.

@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
    // check if we have a cached version
    if (this.cachedPlans != null) {
        return this.cachedPlans;
    }
    boolean childrenSkippedDueToReplicatedInput = false;
    // calculate alternative sub-plans for predecessor
    final List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
    final Set<RequestedGlobalProperties> intGlobal = this.inConn.getInterestingProperties().getGlobalProperties();
    // calculate alternative sub-plans for broadcast inputs
    final List<Set<? extends NamedChannel>> broadcastPlanChannels = new ArrayList<Set<? extends NamedChannel>>();
    List<DagConnection> broadcastConnections = getBroadcastConnections();
    List<String> broadcastConnectionNames = getBroadcastConnectionNames();
    for (int i = 0; i < broadcastConnections.size(); i++) {
        DagConnection broadcastConnection = broadcastConnections.get(i);
        String broadcastConnectionName = broadcastConnectionNames.get(i);
        List<PlanNode> broadcastPlanCandidates = broadcastConnection.getSource().getAlternativePlans(estimator);
        // wrap the plan candidates in named channels
        HashSet<NamedChannel> broadcastChannels = new HashSet<NamedChannel>(broadcastPlanCandidates.size());
        for (PlanNode plan : broadcastPlanCandidates) {
            NamedChannel c = new NamedChannel(broadcastConnectionName, plan);
            DataExchangeMode exMode = DataExchangeMode.select(broadcastConnection.getDataExchangeMode(), ShipStrategyType.BROADCAST, broadcastConnection.isBreakingPipeline());
            c.setShipStrategy(ShipStrategyType.BROADCAST, exMode);
            broadcastChannels.add(c);
        }
        broadcastPlanChannels.add(broadcastChannels);
    }
    final RequestedGlobalProperties[] allValidGlobals;
    {
        Set<RequestedGlobalProperties> pairs = new HashSet<RequestedGlobalProperties>();
        for (OperatorDescriptorSingle ods : getPossibleProperties()) {
            pairs.addAll(ods.getPossibleGlobalProperties());
        }
        allValidGlobals = pairs.toArray(new RequestedGlobalProperties[pairs.size()]);
    }
    final ArrayList<PlanNode> outputPlans = new ArrayList<PlanNode>();
    final ExecutionMode executionMode = this.inConn.getDataExchangeMode();
    final int parallelism = getParallelism();
    final int inParallelism = getPredecessorNode().getParallelism();
    final boolean parallelismChange = inParallelism != parallelism;
    final boolean breaksPipeline = this.inConn.isBreakingPipeline();
    // create all candidates
    for (PlanNode child : subPlans) {
        if (child.getGlobalProperties().isFullyReplicated()) {
            // fully replicated input is always locally forwarded if the parallelism is not changed
            if (parallelismChange) {
                // can not continue with this child
                childrenSkippedDueToReplicatedInput = true;
                continue;
            } else {
                this.inConn.setShipStrategy(ShipStrategyType.FORWARD);
            }
        }
        if (this.inConn.getShipStrategy() == null) {
            // pick the strategy ourselves
            for (RequestedGlobalProperties igps : intGlobal) {
                final Channel c = new Channel(child, this.inConn.getMaterializationMode());
                igps.parameterizeChannel(c, parallelismChange, executionMode, breaksPipeline);
                // ship strategy preserves/establishes them even under changing parallelisms
                if (parallelismChange && !c.getShipStrategy().isNetworkStrategy()) {
                    c.getGlobalProperties().reset();
                }
                // requested properties
                for (RequestedGlobalProperties rgps : allValidGlobals) {
                    if (rgps.isMetBy(c.getGlobalProperties())) {
                        c.setRequiredGlobalProps(rgps);
                        addLocalCandidates(c, broadcastPlanChannels, igps, outputPlans, estimator);
                        break;
                    }
                }
            }
        } else {
            // hint fixed the strategy
            final Channel c = new Channel(child, this.inConn.getMaterializationMode());
            final ShipStrategyType shipStrategy = this.inConn.getShipStrategy();
            final DataExchangeMode exMode = DataExchangeMode.select(executionMode, shipStrategy, breaksPipeline);
            if (this.keys != null) {
                c.setShipStrategy(shipStrategy, this.keys.toFieldList(), exMode);
            } else {
                c.setShipStrategy(shipStrategy, exMode);
            }
            if (parallelismChange) {
                c.adjustGlobalPropertiesForFullParallelismChange();
            }
            // check whether we meet any of the accepted properties
            for (RequestedGlobalProperties rgps : allValidGlobals) {
                if (rgps.isMetBy(c.getGlobalProperties())) {
                    addLocalCandidates(c, broadcastPlanChannels, rgps, outputPlans, estimator);
                    break;
                }
            }
        }
    }
    if (outputPlans.isEmpty()) {
        if (childrenSkippedDueToReplicatedInput) {
            throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Invalid use of replicated input.");
        } else {
            throw new CompilerException("No plan meeting the requirements could be created @ " + this + ". Most likely reason: Too restrictive plan hints.");
        }
    }
    // cost and prune the plans
    for (PlanNode node : outputPlans) {
        estimator.costOperator(node);
    }
    prunePlanAlternatives(outputPlans);
    outputPlans.trimToSize();
    this.cachedPlans = outputPlans;
    return outputPlans;
}

Example 5 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class SingleInputNode method instantiateCandidate.

protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels, List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq) {
    final PlanNode inputSource = in.getSource();
    for (List<NamedChannel> broadcastChannelsCombination : Sets.cartesianProduct(broadcastPlanChannels)) {
        boolean validCombination = true;
        boolean requiresPipelinebreaker = false;
        // check whether the broadcast inputs use the same plan candidate at the branching point
        for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
            NamedChannel nc = broadcastChannelsCombination.get(i);
            PlanNode bcSource = nc.getSource();
            // check branch compatibility against input
            if (!areBranchCompatible(bcSource, inputSource)) {
                validCombination = false;
                break;
            }
            // check branch compatibility against all other broadcast variables
            for (int k = 0; k < i; k++) {
                PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
                if (!areBranchCompatible(bcSource, otherBcSource)) {
                    validCombination = false;
                    break;
                }
            }
            // check if there is a common predecessor and whether there is a dam on the way to all common predecessors
            if (in.isOnDynamicPath() && this.hereJoinedBranches != null) {
                for (OptimizerNode brancher : this.hereJoinedBranches) {
                    PlanNode candAtBrancher = in.getSource().getCandidateAtBranchPoint(brancher);
                    if (candAtBrancher == null) {
                        // closed branch between two broadcast variables
                        continue;
                    }
                    SourceAndDamReport res = in.getSource().hasDamOnPathDownTo(candAtBrancher);
                    if (res == NOT_FOUND) {
                        throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
                    } else if (res == FOUND_SOURCE) {
                        requiresPipelinebreaker = true;
                        break;
                    } else if (res == FOUND_SOURCE_AND_DAM) {
                    // good
                    } else {
                        throw new CompilerException();
                    }
                }
            }
        }
        if (!validCombination) {
            continue;
        }
        if (requiresPipelinebreaker) {
            in.setTempMode(in.getTempMode().makePipelineBreaker());
        }
        final SingleInputPlanNode node = dps.instantiate(in, this);
        node.setBroadcastInputs(broadcastChannelsCombination);
        // compute how the strategy affects the properties
        GlobalProperties gProps = in.getGlobalProperties().clone();
        LocalProperties lProps = in.getLocalProperties().clone();
        gProps = dps.computeGlobalProperties(gProps);
        lProps = dps.computeLocalProperties(lProps);
        // filter by the user code field copies
        gProps = gProps.filterBySemanticProperties(getSemanticPropertiesForGlobalPropertyFiltering(), 0);
        lProps = lProps.filterBySemanticProperties(getSemanticPropertiesForLocalPropertyFiltering(), 0);
        // apply
        node.initProperties(gProps, lProps);
        node.updatePropertiesWithUniqueSets(getUniqueFields());
        target.add(node);
    }
}