Examples with RequestedLocalProperties org.apache.flink.optimizer.dataproperties.RequestedLocalProperties used on opensource projects

Example 16 with RequestedLocalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedLocalProperties in project flink by apache.

the class DataSinkNode method getAlternativePlans.

// --------------------------------------------------------------------------------------------
//                                   Recursive Optimization
// --------------------------------------------------------------------------------------------
@Override
public List<PlanNode> getAlternativePlans(CostEstimator estimator) {
    // check if we have a cached version
    if (this.cachedPlans != null) {
        return this.cachedPlans;
    }
    // calculate alternative sub-plans for predecessor
    List<? extends PlanNode> subPlans = getPredecessorNode().getAlternativePlans(estimator);
    List<PlanNode> outputPlans = new ArrayList<PlanNode>();
    final int parallelism = getParallelism();
    final int inDop = getPredecessorNode().getParallelism();
    final ExecutionMode executionMode = this.input.getDataExchangeMode();
    final boolean dopChange = parallelism != inDop;
    final boolean breakPipeline = this.input.isBreakingPipeline();
    InterestingProperties ips = this.input.getInterestingProperties();
    for (PlanNode p : subPlans) {
        for (RequestedGlobalProperties gp : ips.getGlobalProperties()) {
            for (RequestedLocalProperties lp : ips.getLocalProperties()) {
                Channel c = new Channel(p);
                gp.parameterizeChannel(c, dopChange, executionMode, breakPipeline);
                lp.parameterizeChannel(c);
                c.setRequiredLocalProps(lp);
                c.setRequiredGlobalProps(gp);
                // no need to check whether the created properties meet what we need in case
                // of ordering or global ordering, because the only interesting properties we have
                // are what we require
                outputPlans.add(new SinkPlanNode(this, "DataSink (" + this.getOperator().getName() + ")", c));
            }
        }
    }
    // cost and prune the plans
    for (PlanNode node : outputPlans) {
        estimator.costOperator(node);
    }
    prunePlanAlternatives(outputPlans);
    this.cachedPlans = outputPlans;
    return outputPlans;
}

Example 17 with RequestedLocalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedLocalProperties in project flink by apache.

the class DataSinkNode method computeInterestingPropertiesForInputs.

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
    final InterestingProperties iProps = new InterestingProperties();
    {
        final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties();
        iProps.addGlobalProperties(partitioningProps);
    }
    {
        final Ordering localOrder = getOperator().getLocalOrder();
        final RequestedLocalProperties orderProps = new RequestedLocalProperties();
        if (localOrder != null) {
            orderProps.setOrdering(localOrder);
        }
        iProps.addLocalProperties(orderProps);
    }
    this.input.setInterestingProperties(iProps);
}

Example 18 with RequestedLocalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedLocalProperties in project flink by apache.

the class TwoInputNode method addLocalCandidates.

protected void addLocalCandidates(Channel template1, Channel template2, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps1, RequestedGlobalProperties rgps2, List<PlanNode> target, LocalPropertiesPair[] validLocalCombinations, CostEstimator estimator) {
    for (RequestedLocalProperties ilp1 : this.input1.getInterestingProperties().getLocalProperties()) {
        final Channel in1 = template1.clone();
        ilp1.parameterizeChannel(in1);
        for (RequestedLocalProperties ilp2 : this.input2.getInterestingProperties().getLocalProperties()) {
            final Channel in2 = template2.clone();
            ilp2.parameterizeChannel(in2);
            for (OperatorDescriptorDual dps : getProperties()) {
                for (LocalPropertiesPair lpp : dps.getPossibleLocalProperties()) {
                    if (lpp.getProperties1().isMetBy(in1.getLocalProperties()) && lpp.getProperties2().isMetBy(in2.getLocalProperties())) {
                        // (such as when some sort order is requested, that both are the same sort order
                        if (dps.areCoFulfilled(lpp.getProperties1(), lpp.getProperties2(), in1.getLocalProperties(), in2.getLocalProperties())) {
                            // copy, because setting required properties and instantiation may
                            // change the channels and should not affect prior candidates
                            Channel in1Copy = in1.clone();
                            in1Copy.setRequiredLocalProps(lpp.getProperties1());
                            Channel in2Copy = in2.clone();
                            in2Copy.setRequiredLocalProps(lpp.getProperties2());
                            // all right, co compatible
                            instantiate(dps, in1Copy, in2Copy, broadcastPlanChannels, target, estimator, rgps1, rgps2, ilp1, ilp2);
                            break;
                        }
                    // else cannot use this pair, fall through the loop and try the next one
                    }
                }
            }
        }
    }
}

Example 19 with RequestedLocalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedLocalProperties in project flink by apache.

the class TwoInputNode method instantiate.

protected void instantiate(OperatorDescriptorDual operator, Channel in1, Channel in2, List<Set<? extends NamedChannel>> broadcastPlanChannels, List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq1, RequestedGlobalProperties globPropsReq2, RequestedLocalProperties locPropsReq1, RequestedLocalProperties locPropsReq2) {
    final PlanNode inputSource1 = in1.getSource();
    final PlanNode inputSource2 = in2.getSource();
    for (List<NamedChannel> broadcastChannelsCombination : Sets.cartesianProduct(broadcastPlanChannels)) {
        boolean validCombination = true;
        // check whether the broadcast inputs use the same plan candidate at the branching point
        for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
            NamedChannel nc = broadcastChannelsCombination.get(i);
            PlanNode bcSource = nc.getSource();
            if (!(areBranchCompatible(bcSource, inputSource1) || areBranchCompatible(bcSource, inputSource2))) {
                validCombination = false;
                break;
            }
            // check branch compatibility against all other broadcast variables
            for (int k = 0; k < i; k++) {
                PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
                if (!areBranchCompatible(bcSource, otherBcSource)) {
                    validCombination = false;
                    break;
                }
            }
        }
        if (!validCombination) {
            continue;
        }
        placePipelineBreakersIfNecessary(operator.getStrategy(), in1, in2);
        DualInputPlanNode node = operator.instantiate(in1, in2, this);
        node.setBroadcastInputs(broadcastChannelsCombination);
        SemanticProperties semPropsGlobalPropFiltering = getSemanticPropertiesForGlobalPropertyFiltering();
        GlobalProperties gp1 = in1.getGlobalProperties().clone().filterBySemanticProperties(semPropsGlobalPropFiltering, 0);
        GlobalProperties gp2 = in2.getGlobalProperties().clone().filterBySemanticProperties(semPropsGlobalPropFiltering, 1);
        GlobalProperties combined = operator.computeGlobalProperties(gp1, gp2);
        SemanticProperties semPropsLocalPropFiltering = getSemanticPropertiesForLocalPropertyFiltering();
        LocalProperties lp1 = in1.getLocalProperties().clone().filterBySemanticProperties(semPropsLocalPropFiltering, 0);
        LocalProperties lp2 = in2.getLocalProperties().clone().filterBySemanticProperties(semPropsLocalPropFiltering, 1);
        LocalProperties locals = operator.computeLocalProperties(lp1, lp2);
        node.initProperties(combined, locals);
        node.updatePropertiesWithUniqueSets(getUniqueFields());
        target.add(node);
    }
}

Example 20 with RequestedLocalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedLocalProperties in project flink by apache.

the class WorksetIterationNode method computeInterestingPropertiesForInputs.

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
    // our own solution (the solution set) is always partitioned and this cannot be adjusted
    // depending on what the successor to the workset iteration requests. for that reason,
    // we ignore incoming interesting properties.
    // in addition, we need to make 2 interesting property passes, because the root of the step function 
    // that computes the next workset needs the interesting properties as generated by the
    // workset source of the step function. the second pass concerns only the workset path.
    // as initial interesting properties, we have the trivial ones for the step function,
    // and partitioned on the solution set key for the solution set delta 
    RequestedGlobalProperties partitionedProperties = new RequestedGlobalProperties();
    partitionedProperties.setHashPartitioned(this.solutionSetKeyFields);
    InterestingProperties partitionedIP = new InterestingProperties();
    partitionedIP.addGlobalProperties(partitionedProperties);
    partitionedIP.addLocalProperties(new RequestedLocalProperties());
    this.nextWorksetRootConnection.setInterestingProperties(new InterestingProperties());
    this.solutionSetDeltaRootConnection.setInterestingProperties(partitionedIP.clone());
    InterestingPropertyVisitor ipv = new InterestingPropertyVisitor(estimator);
    this.nextWorkset.accept(ipv);
    this.solutionSetDelta.accept(ipv);
    // take the interesting properties of the partial solution and add them to the root interesting properties
    InterestingProperties worksetIntProps = this.worksetNode.getInterestingProperties();
    InterestingProperties intProps = new InterestingProperties();
    intProps.getGlobalProperties().addAll(worksetIntProps.getGlobalProperties());
    intProps.getLocalProperties().addAll(worksetIntProps.getLocalProperties());
    // clear all interesting properties to prepare the second traversal
    this.nextWorksetRootConnection.clearInterestingProperties();
    this.nextWorkset.accept(InterestingPropertiesClearer.INSTANCE);
    // 2nd pass
    this.nextWorksetRootConnection.setInterestingProperties(intProps);
    this.nextWorkset.accept(ipv);
    // now add the interesting properties of the workset to the workset input
    final InterestingProperties inProps = this.worksetNode.getInterestingProperties().clone();
    inProps.addGlobalProperties(new RequestedGlobalProperties());
    inProps.addLocalProperties(new RequestedLocalProperties());
    this.input2.setInterestingProperties(inProps);
    // the partial solution must be hash partitioned, so it has only that as interesting properties
    this.input1.setInterestingProperties(partitionedIP);
}