Examples with RequestedGlobalProperties org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties used on opensource projects

Example 11 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class WorksetIterationNode method instantiate.

@SuppressWarnings("unchecked")
@Override
protected void instantiate(OperatorDescriptorDual operator, Channel solutionSetIn, Channel worksetIn, List<Set<? extends NamedChannel>> broadcastPlanChannels, List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReqSolutionSet, RequestedGlobalProperties globPropsReqWorkset, RequestedLocalProperties locPropsReqSolutionSet, RequestedLocalProperties locPropsReqWorkset) {
    // check for pipeline breaking using hash join with build on the solution set side
    placePipelineBreakersIfNecessary(DriverStrategy.HYBRIDHASH_BUILD_FIRST, solutionSetIn, worksetIn);
    // NOTES ON THE ENUMERATION OF THE STEP FUNCTION PLANS:
    // Whenever we instantiate the iteration, we enumerate new candidates for the step function.
    // That way, we make sure we have an appropriate plan for each candidate for the initial partial solution,
    // we have a fitting candidate for the step function (often, work is pushed out of the step function).
    // Among the candidates of the step function, we keep only those that meet the requested properties of the
    // current candidate initial partial solution. That makes sure these properties exist at the beginning of
    // every iteration.
    // 1) Because we enumerate multiple times, we may need to clean the cached plans
    //    before starting another enumeration
    this.nextWorkset.accept(PlanCacheCleaner.INSTANCE);
    this.solutionSetDelta.accept(PlanCacheCleaner.INSTANCE);
    // 2) Give the partial solution the properties of the current candidate for the initial partial solution
    //    This concerns currently only the workset.
    this.worksetNode.setCandidateProperties(worksetIn.getGlobalProperties(), worksetIn.getLocalProperties(), worksetIn);
    this.solutionSetNode.setCandidateProperties(this.partitionedProperties, new LocalProperties(), solutionSetIn);
    final SolutionSetPlanNode sspn = this.solutionSetNode.getCurrentSolutionSetPlanNode();
    final WorksetPlanNode wspn = this.worksetNode.getCurrentWorksetPlanNode();
    // 3) Get the alternative plans
    List<PlanNode> solutionSetDeltaCandidates = this.solutionSetDelta.getAlternativePlans(estimator);
    List<PlanNode> worksetCandidates = this.nextWorkset.getAlternativePlans(estimator);
    // 4) Throw away all that are not compatible with the properties currently requested to the
    //    initial partial solution
    // Make sure that the workset candidates fulfill the input requirements
    {
        List<PlanNode> newCandidates = new ArrayList<PlanNode>();
        for (Iterator<PlanNode> planDeleter = worksetCandidates.iterator(); planDeleter.hasNext(); ) {
            PlanNode candidate = planDeleter.next();
            GlobalProperties atEndGlobal = candidate.getGlobalProperties();
            LocalProperties atEndLocal = candidate.getLocalProperties();
            FeedbackPropertiesMeetRequirementsReport report = candidate.checkPartialSolutionPropertiesMet(wspn, atEndGlobal, atEndLocal);
            if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
            // depends only through broadcast variable on the workset solution
            } else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
                // attach a no-op node through which we create the properties of the original input
                Channel toNoOp = new Channel(candidate);
                globPropsReqWorkset.parameterizeChannel(toNoOp, false, nextWorksetRootConnection.getDataExchangeMode(), false);
                locPropsReqWorkset.parameterizeChannel(toNoOp);
                NoOpUnaryUdfOp noOpUnaryUdfOp = new NoOpUnaryUdfOp<>();
                noOpUnaryUdfOp.setInput(candidate.getProgramOperator());
                UnaryOperatorNode rebuildWorksetPropertiesNode = new UnaryOperatorNode("Rebuild Workset Properties", noOpUnaryUdfOp, true);
                rebuildWorksetPropertiesNode.setParallelism(candidate.getParallelism());
                SingleInputPlanNode rebuildWorksetPropertiesPlanNode = new SingleInputPlanNode(rebuildWorksetPropertiesNode, "Rebuild Workset Properties", toNoOp, DriverStrategy.UNARY_NO_OP);
                rebuildWorksetPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties());
                estimator.costOperator(rebuildWorksetPropertiesPlanNode);
                GlobalProperties atEndGlobalModified = rebuildWorksetPropertiesPlanNode.getGlobalProperties();
                LocalProperties atEndLocalModified = rebuildWorksetPropertiesPlanNode.getLocalProperties();
                if (!(atEndGlobalModified.equals(atEndGlobal) && atEndLocalModified.equals(atEndLocal))) {
                    FeedbackPropertiesMeetRequirementsReport report2 = candidate.checkPartialSolutionPropertiesMet(wspn, atEndGlobalModified, atEndLocalModified);
                    if (report2 != FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
                        newCandidates.add(rebuildWorksetPropertiesPlanNode);
                    }
                }
                // remove the original operator and add the modified candidate
                planDeleter.remove();
            }
        }
        worksetCandidates.addAll(newCandidates);
    }
    if (worksetCandidates.isEmpty()) {
        return;
    }
    // sanity check the solution set delta
    for (PlanNode solutionSetDeltaCandidate : solutionSetDeltaCandidates) {
        SingleInputPlanNode candidate = (SingleInputPlanNode) solutionSetDeltaCandidate;
        GlobalProperties gp = candidate.getGlobalProperties();
        if (gp.getPartitioning() != PartitioningProperty.HASH_PARTITIONED || gp.getPartitioningFields() == null || !gp.getPartitioningFields().equals(this.solutionSetKeyFields)) {
            throw new CompilerException("Bug: The solution set delta is not partitioned.");
        }
    }
    // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
    final GlobalProperties gp = new GlobalProperties();
    gp.setHashPartitioned(this.solutionSetKeyFields);
    gp.addUniqueFieldCombination(this.solutionSetKeyFields);
    LocalProperties lp = LocalProperties.EMPTY.addUniqueFields(this.solutionSetKeyFields);
    // take all combinations of solution set delta and workset plans
    for (PlanNode solutionSetCandidate : solutionSetDeltaCandidates) {
        for (PlanNode worksetCandidate : worksetCandidates) {
            // check whether they have the same operator at their latest branching point
            if (this.singleRoot.areBranchCompatible(solutionSetCandidate, worksetCandidate)) {
                SingleInputPlanNode siSolutionDeltaCandidate = (SingleInputPlanNode) solutionSetCandidate;
                boolean immediateDeltaUpdate;
                // check whether we need a dedicated solution set delta operator, or whether we can update on the fly
                if (siSolutionDeltaCandidate.getInput().getShipStrategy() == ShipStrategyType.FORWARD && this.solutionDeltaImmediatelyAfterSolutionJoin) {
                    // sanity check the node and connection
                    if (siSolutionDeltaCandidate.getDriverStrategy() != DriverStrategy.UNARY_NO_OP || siSolutionDeltaCandidate.getInput().getLocalStrategy() != LocalStrategy.NONE) {
                        throw new CompilerException("Invalid Solution set delta node.");
                    }
                    solutionSetCandidate = siSolutionDeltaCandidate.getInput().getSource();
                    immediateDeltaUpdate = true;
                } else {
                    // was not partitioned, we need to keep this node.
                    // mark that we materialize the input
                    siSolutionDeltaCandidate.getInput().setTempMode(TempMode.PIPELINE_BREAKER);
                    immediateDeltaUpdate = false;
                }
                WorksetIterationPlanNode wsNode = new WorksetIterationPlanNode(this, this.getOperator().getName(), solutionSetIn, worksetIn, sspn, wspn, worksetCandidate, solutionSetCandidate);
                wsNode.setImmediateSolutionSetUpdate(immediateDeltaUpdate);
                wsNode.initProperties(gp, lp);
                target.add(wsNode);
            }
        }
    }
}

Example 12 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class AbstractJoinDescriptor method createPossibleGlobalProperties.

@Override
protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
    ArrayList<GlobalPropertiesPair> pairs = new ArrayList<GlobalPropertiesPair>();
    if (repartitionAllowed) {
        // partition both (hash or custom)
        if (this.customPartitioner == null) {
            // we accept compatible partitionings of any type
            RequestedGlobalProperties partitioned_left_any = new RequestedGlobalProperties();
            RequestedGlobalProperties partitioned_right_any = new RequestedGlobalProperties();
            partitioned_left_any.setAnyPartitioning(this.keys1);
            partitioned_right_any.setAnyPartitioning(this.keys2);
            pairs.add(new GlobalPropertiesPair(partitioned_left_any, partitioned_right_any));
            // add strict hash partitioning of both inputs on their full key sets
            RequestedGlobalProperties partitioned_left_hash = new RequestedGlobalProperties();
            RequestedGlobalProperties partitioned_right_hash = new RequestedGlobalProperties();
            partitioned_left_hash.setHashPartitioned(this.keys1);
            partitioned_right_hash.setHashPartitioned(this.keys2);
            pairs.add(new GlobalPropertiesPair(partitioned_left_hash, partitioned_right_hash));
        } else {
            RequestedGlobalProperties partitioned_left = new RequestedGlobalProperties();
            partitioned_left.setCustomPartitioned(this.keys1, this.customPartitioner);
            RequestedGlobalProperties partitioned_right = new RequestedGlobalProperties();
            partitioned_right.setCustomPartitioned(this.keys2, this.customPartitioner);
            return Collections.singletonList(new GlobalPropertiesPair(partitioned_left, partitioned_right));
        }
        RequestedGlobalProperties partitioned1 = new RequestedGlobalProperties();
        if (customPartitioner == null) {
            partitioned1.setAnyPartitioning(this.keys1);
        } else {
            partitioned1.setCustomPartitioned(this.keys1, this.customPartitioner);
        }
        RequestedGlobalProperties partitioned2 = new RequestedGlobalProperties();
        if (customPartitioner == null) {
            partitioned2.setAnyPartitioning(this.keys2);
        } else {
            partitioned2.setCustomPartitioned(this.keys2, this.customPartitioner);
        }
        pairs.add(new GlobalPropertiesPair(partitioned1, partitioned2));
    }
    if (broadcastSecondAllowed) {
        // replicate second
        RequestedGlobalProperties any1 = new RequestedGlobalProperties();
        RequestedGlobalProperties replicated2 = new RequestedGlobalProperties();
        replicated2.setFullyReplicated();
        pairs.add(new GlobalPropertiesPair(any1, replicated2));
    }
    if (broadcastFirstAllowed) {
        // replicate first
        RequestedGlobalProperties replicated1 = new RequestedGlobalProperties();
        replicated1.setFullyReplicated();
        RequestedGlobalProperties any2 = new RequestedGlobalProperties();
        pairs.add(new GlobalPropertiesPair(replicated1, any2));
    }
    return pairs;
}

Example 13 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class MapDescriptor method createPossibleGlobalProperties.

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
    RequestedGlobalProperties rgp = new RequestedGlobalProperties();
    rgp.setAnyDistribution();
    return Collections.singletonList(rgp);
}

Example 14 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class MapPartitionDescriptor method createPossibleGlobalProperties.

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
    RequestedGlobalProperties rgp = new RequestedGlobalProperties();
    rgp.setAnyDistribution();
    return Collections.singletonList(rgp);
}

Example 15 with RequestedGlobalProperties

use of org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties in project flink by apache.

the class SolutionSetDeltaOperator method createPossibleGlobalProperties.

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
    RequestedGlobalProperties partProps = new RequestedGlobalProperties();
    partProps.setHashPartitioned(this.keyList);
    return Collections.singletonList(partProps);
}