Examples with CompilerException org.apache.flink.optimizer.CompilerException used on opensource projects

Example 11 with CompilerException

use of org.apache.flink.optimizer.CompilerException in project flink by apache.

the class JobGraphGenerator method finalizeBulkIteration.

private void finalizeBulkIteration(IterationDescriptor descr) {
    final BulkIterationPlanNode bulkNode = (BulkIterationPlanNode) descr.getIterationNode();
    final JobVertex headVertex = descr.getHeadTask();
    final TaskConfig headConfig = new TaskConfig(headVertex.getConfiguration());
    final TaskConfig headFinalOutputConfig = descr.getHeadFinalResultConfig();
    // ------------ finalize the head config with the final outputs and the sync gate ------------
    final int numStepFunctionOuts = headConfig.getNumOutputs();
    final int numFinalOuts = headFinalOutputConfig.getNumOutputs();
    if (numStepFunctionOuts == 0) {
        throw new CompilerException("The iteration has no operation inside the step function.");
    }
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutputConfig);
    headConfig.setIterationHeadIndexOfSyncOutput(numStepFunctionOuts + numFinalOuts);
    final double relativeMemForBackChannel = bulkNode.getRelativeMemoryPerSubTask();
    if (relativeMemForBackChannel <= 0) {
        throw new CompilerException("Bug: No memory has been assigned to the iteration back channel.");
    }
    headConfig.setRelativeBackChannelMemory(relativeMemForBackChannel);
    // --------------------------- create the sync task ---------------------------
    final JobVertex sync = new JobVertex("Sync(" + bulkNode.getNodeName() + ")");
    sync.setResources(bulkNode.getMinResources(), bulkNode.getPreferredResources());
    sync.setInvokableClass(IterationSynchronizationSinkTask.class);
    sync.setParallelism(1);
    sync.setMaxParallelism(1);
    this.auxVertices.add(sync);
    final TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setGateIterativeWithNumberOfEventsUntilInterrupt(0, headVertex.getParallelism());
    // set the number of iteration / convergence criterion for the sync
    final int maxNumIterations = bulkNode.getIterationNode().getIterationContract().getMaximumNumberOfIterations();
    if (maxNumIterations < 1) {
        throw new CompilerException("Cannot create bulk iteration with unspecified maximum number of iterations.");
    }
    syncConfig.setNumberOfIterations(maxNumIterations);
    // connect the sync task
    sync.connectNewDataSetAsInput(headVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
    // ----------------------------- create the iteration tail ------------------------------
    final PlanNode rootOfTerminationCriterion = bulkNode.getRootOfTerminationCriterion();
    final PlanNode rootOfStepFunction = bulkNode.getRootOfStepFunction();
    final TaskConfig tailConfig;
    JobVertex rootOfStepFunctionVertex = this.vertices.get(rootOfStepFunction);
    if (rootOfStepFunctionVertex == null) {
        // last op is chained
        final TaskInChain taskInChain = this.chainedTasks.get(rootOfStepFunction);
        if (taskInChain == null) {
            throw new CompilerException("Bug: Tail of step function not found as vertex or chained task.");
        }
        rootOfStepFunctionVertex = taskInChain.getContainingVertex();
        // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
        tailConfig = taskInChain.getTaskConfig();
    } else {
        tailConfig = new TaskConfig(rootOfStepFunctionVertex.getConfiguration());
    }
    tailConfig.setIsWorksetUpdate();
    // No following termination criterion
    if (rootOfStepFunction.getOutgoingChannels().isEmpty()) {
        rootOfStepFunctionVertex.setInvokableClass(IterationTailTask.class);
        tailConfig.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
    }
    // create the fake output task for termination criterion, if needed
    final TaskConfig tailConfigOfTerminationCriterion;
    // If we have a termination criterion and it is not an intermediate node
    if (rootOfTerminationCriterion != null && rootOfTerminationCriterion.getOutgoingChannels().isEmpty()) {
        JobVertex rootOfTerminationCriterionVertex = this.vertices.get(rootOfTerminationCriterion);
        if (rootOfTerminationCriterionVertex == null) {
            // last op is chained
            final TaskInChain taskInChain = this.chainedTasks.get(rootOfTerminationCriterion);
            if (taskInChain == null) {
                throw new CompilerException("Bug: Tail of termination criterion not found as vertex or chained task.");
            }
            rootOfTerminationCriterionVertex = taskInChain.getContainingVertex();
            // the fake channel is statically typed to pact record. no data is sent over this channel anyways.
            tailConfigOfTerminationCriterion = taskInChain.getTaskConfig();
        } else {
            tailConfigOfTerminationCriterion = new TaskConfig(rootOfTerminationCriterionVertex.getConfiguration());
        }
        rootOfTerminationCriterionVertex.setInvokableClass(IterationTailTask.class);
        // Hack
        tailConfigOfTerminationCriterion.setIsSolutionSetUpdate();
        tailConfigOfTerminationCriterion.setOutputSerializer(bulkNode.getSerializerForIterationChannel());
        // tell the head that it needs to wait for the solution set updates
        headConfig.setWaitForSolutionSetUpdate();
    }
    // ------------------- register the aggregators -------------------
    AggregatorRegistry aggs = bulkNode.getIterationNode().getIterationContract().getAggregators();
    Collection<AggregatorWithName<?>> allAggregators = aggs.getAllRegisteredAggregators();
    headConfig.addIterationAggregators(allAggregators);
    syncConfig.addIterationAggregators(allAggregators);
    String convAggName = aggs.getConvergenceCriterionAggregatorName();
    ConvergenceCriterion<?> convCriterion = aggs.getConvergenceCriterion();
    if (convCriterion != null || convAggName != null) {
        if (convCriterion == null) {
            throw new CompilerException("Error: Convergence criterion aggregator set, but criterion is null.");
        }
        if (convAggName == null) {
            throw new CompilerException("Error: Aggregator convergence criterion set, but aggregator is null.");
        }
        syncConfig.setConvergenceCriterion(convAggName, convCriterion);
    }
}

Example 12 with CompilerException

use of org.apache.flink.optimizer.CompilerException in project flink by apache.

the class JobGraphGenerator method connectJobVertices.

// ------------------------------------------------------------------------
// Connecting Vertices
// ------------------------------------------------------------------------
/**
	 * NOTE: The channel for global and local strategies are different if we connect a union. The global strategy
	 * channel is then the channel into the union node, the local strategy channel the one from the union to the
	 * actual target operator.
	 *
	 * @throws CompilerException
	 */
private DistributionPattern connectJobVertices(Channel channel, int inputNumber, final JobVertex sourceVertex, final TaskConfig sourceConfig, final JobVertex targetVertex, final TaskConfig targetConfig, boolean isBroadcast) throws CompilerException {
    // ------------ connect the vertices to the job graph --------------
    final DistributionPattern distributionPattern;
    switch(channel.getShipStrategy()) {
        case FORWARD:
            distributionPattern = DistributionPattern.POINTWISE;
            break;
        case PARTITION_RANDOM:
        case BROADCAST:
        case PARTITION_HASH:
        case PARTITION_CUSTOM:
        case PARTITION_RANGE:
        case PARTITION_FORCED_REBALANCE:
            distributionPattern = DistributionPattern.ALL_TO_ALL;
            break;
        default:
            throw new RuntimeException("Unknown runtime ship strategy: " + channel.getShipStrategy());
    }
    final ResultPartitionType resultType;
    switch(channel.getDataExchangeMode()) {
        case PIPELINED:
            resultType = ResultPartitionType.PIPELINED;
            break;
        case BATCH:
            // BLOCKING results are currently not supported in closed loop iterations
            //
            // See https://issues.apache.org/jira/browse/FLINK-1713 for details
            resultType = channel.getSource().isOnDynamicPath() ? ResultPartitionType.PIPELINED : ResultPartitionType.BLOCKING;
            break;
        case PIPELINE_WITH_BATCH_FALLBACK:
            throw new UnsupportedOperationException("Data exchange mode " + channel.getDataExchangeMode() + " currently not supported.");
        default:
            throw new UnsupportedOperationException("Unknown data exchange mode.");
    }
    JobEdge edge = targetVertex.connectNewDataSetAsInput(sourceVertex, distributionPattern, resultType);
    // -------------- configure the source task's ship strategy strategies in task config --------------
    final int outputIndex = sourceConfig.getNumOutputs();
    sourceConfig.addOutputShipStrategy(channel.getShipStrategy());
    if (outputIndex == 0) {
        sourceConfig.setOutputSerializer(channel.getSerializer());
    }
    if (channel.getShipStrategyComparator() != null) {
        sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
    }
    if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
        final DataDistribution dataDistribution = channel.getDataDistribution();
        if (dataDistribution != null) {
            sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
        } else {
            throw new RuntimeException("Range partitioning requires data distribution.");
        }
    }
    if (channel.getShipStrategy() == ShipStrategyType.PARTITION_CUSTOM) {
        if (channel.getPartitioner() != null) {
            sourceConfig.setOutputPartitioner(channel.getPartitioner(), outputIndex);
        } else {
            throw new CompilerException("The ship strategy was set to custom partitioning, but no partitioner was set.");
        }
    }
    // ---------------- configure the receiver -------------------
    if (isBroadcast) {
        targetConfig.addBroadcastInputToGroup(inputNumber);
    } else {
        targetConfig.addInputToGroup(inputNumber);
    }
    // ---------------- attach the additional infos to the job edge -------------------
    String shipStrategy = JsonMapper.getShipStrategyString(channel.getShipStrategy());
    if (channel.getShipStrategyKeys() != null && channel.getShipStrategyKeys().size() > 0) {
        shipStrategy += " on " + (channel.getShipStrategySortOrder() == null ? channel.getShipStrategyKeys().toString() : Utils.createOrdering(channel.getShipStrategyKeys(), channel.getShipStrategySortOrder()).toString());
    }
    String localStrategy;
    if (channel.getLocalStrategy() == null || channel.getLocalStrategy() == LocalStrategy.NONE) {
        localStrategy = null;
    } else {
        localStrategy = JsonMapper.getLocalStrategyString(channel.getLocalStrategy());
        if (localStrategy != null && channel.getLocalStrategyKeys() != null && channel.getLocalStrategyKeys().size() > 0) {
            localStrategy += " on " + (channel.getLocalStrategySortOrder() == null ? channel.getLocalStrategyKeys().toString() : Utils.createOrdering(channel.getLocalStrategyKeys(), channel.getLocalStrategySortOrder()).toString());
        }
    }
    String caching = channel.getTempMode() == TempMode.NONE ? null : channel.getTempMode().toString();
    edge.setShipStrategyName(shipStrategy);
    edge.setPreProcessingOperationName(localStrategy);
    edge.setOperatorLevelCachingDescription(caching);
    return distributionPattern;
}

Example 13 with CompilerException

use of org.apache.flink.optimizer.CompilerException in project flink by apache.

the class JobGraphGenerator method preVisit.

/**
	 * This methods implements the pre-visiting during a depth-first traversal. It create the job vertex and
	 * sets local strategy.
	 * 
	 * @param node
	 *        The node that is currently processed.
	 * @return True, if the visitor should descend to the node's children, false if not.
	 * @see org.apache.flink.util.Visitor#preVisit(org.apache.flink.util.Visitable)
	 */
@Override
public boolean preVisit(PlanNode node) {
    // check if we have visited this node before. in non-tree graphs, this happens
    if (this.vertices.containsKey(node) || this.chainedTasks.containsKey(node) || this.iterations.containsKey(node)) {
        // return false to prevent further descend
        return false;
    }
    // the vertex to be created for the current node
    final JobVertex vertex;
    try {
        if (node instanceof SinkPlanNode) {
            vertex = createDataSinkVertex((SinkPlanNode) node);
        } else if (node instanceof SourcePlanNode) {
            vertex = createDataSourceVertex((SourcePlanNode) node);
        } else if (node instanceof BulkIterationPlanNode) {
            BulkIterationPlanNode iterationNode = (BulkIterationPlanNode) node;
            // for the bulk iteration, we skip creating anything for now. we create the graph
            // for the step function in the post visit.
            // check that the root of the step function has the same parallelism as the iteration.
            // because the tail must have the same parallelism as the head, we can only merge the last
            // operator with the tail, if they have the same parallelism. not merging is currently not
            // implemented
            PlanNode root = iterationNode.getRootOfStepFunction();
            if (root.getParallelism() != node.getParallelism()) {
                throw new CompilerException("Error: The final operator of the step " + "function has a different parallelism than the iteration operator itself.");
            }
            IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
            this.iterations.put(iterationNode, descr);
            vertex = null;
        } else if (node instanceof WorksetIterationPlanNode) {
            WorksetIterationPlanNode iterationNode = (WorksetIterationPlanNode) node;
            // we have the same constraints as for the bulk iteration
            PlanNode nextWorkSet = iterationNode.getNextWorkSetPlanNode();
            PlanNode solutionSetDelta = iterationNode.getSolutionSetDeltaPlanNode();
            if (nextWorkSet.getParallelism() != node.getParallelism()) {
                throw new CompilerException("It is currently not supported that the final operator of the step " + "function has a different parallelism than the iteration operator itself.");
            }
            if (solutionSetDelta.getParallelism() != node.getParallelism()) {
                throw new CompilerException("It is currently not supported that the final operator of the step " + "function has a different parallelism than the iteration operator itself.");
            }
            IterationDescriptor descr = new IterationDescriptor(iterationNode, this.iterationIdEnumerator++);
            this.iterations.put(iterationNode, descr);
            vertex = null;
        } else if (node instanceof SingleInputPlanNode) {
            vertex = createSingleInputVertex((SingleInputPlanNode) node);
        } else if (node instanceof DualInputPlanNode) {
            vertex = createDualInputVertex((DualInputPlanNode) node);
        } else if (node instanceof NAryUnionPlanNode) {
            // skip the union for now
            vertex = null;
        } else if (node instanceof BulkPartialSolutionPlanNode) {
            // create a head node (or not, if it is merged into its successor)
            vertex = createBulkIterationHead((BulkPartialSolutionPlanNode) node);
        } else if (node instanceof SolutionSetPlanNode) {
            // we adjust the joins / cogroups that go into the solution set here
            for (Channel c : node.getOutgoingChannels()) {
                DualInputPlanNode target = (DualInputPlanNode) c.getTarget();
                JobVertex accessingVertex = this.vertices.get(target);
                TaskConfig conf = new TaskConfig(accessingVertex.getConfiguration());
                int inputNum = c == target.getInput1() ? 0 : c == target.getInput2() ? 1 : -1;
                // sanity checks
                if (inputNum == -1) {
                    throw new CompilerException();
                }
                // adjust the driver
                if (conf.getDriver().equals(JoinDriver.class)) {
                    conf.setDriver(inputNum == 0 ? JoinWithSolutionSetFirstDriver.class : JoinWithSolutionSetSecondDriver.class);
                } else if (conf.getDriver().equals(CoGroupDriver.class)) {
                    conf.setDriver(inputNum == 0 ? CoGroupWithSolutionSetFirstDriver.class : CoGroupWithSolutionSetSecondDriver.class);
                } else {
                    throw new CompilerException("Found join with solution set using incompatible operator (only Join/CoGroup are valid).");
                }
            }
            // make sure we do not visit this node again. for that, we add a 'already seen' entry into one of the sets
            this.chainedTasks.put(node, ALREADY_VISITED_PLACEHOLDER);
            vertex = null;
        } else if (node instanceof WorksetPlanNode) {
            // create the iteration head here
            vertex = createWorksetIterationHead((WorksetPlanNode) node);
        } else {
            throw new CompilerException("Unrecognized node type: " + node.getClass().getName());
        }
    } catch (Exception e) {
        throw new CompilerException("Error translating node '" + node + "': " + e.getMessage(), e);
    }
    // check if a vertex was created, or if it was chained or skipped
    if (vertex != null) {
        // set parallelism
        int pd = node.getParallelism();
        vertex.setParallelism(pd);
        vertex.setMaxParallelism(pd);
        vertex.setSlotSharingGroup(sharingGroup);
        // check whether this vertex is part of an iteration step function
        if (this.currentIteration != null) {
            // check that the task has the same parallelism as the iteration as such
            PlanNode iterationNode = (PlanNode) this.currentIteration;
            if (iterationNode.getParallelism() < pd) {
                throw new CompilerException("Error: All functions that are part of an iteration must have the same, or a lower, parallelism than the iteration operator.");
            }
            // store the id of the iterations the step functions participate in
            IterationDescriptor descr = this.iterations.get(this.currentIteration);
            new TaskConfig(vertex.getConfiguration()).setIterationId(descr.getId());
        }
        // store in the map
        this.vertices.put(node, vertex);
    }
    // returning true causes deeper descend
    return true;
}

Example 14 with CompilerException

use of org.apache.flink.optimizer.CompilerException in project flink by apache.

the class WorksetIterationNode method instantiate.

@SuppressWarnings("unchecked")
@Override
protected void instantiate(OperatorDescriptorDual operator, Channel solutionSetIn, Channel worksetIn, List<Set<? extends NamedChannel>> broadcastPlanChannels, List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReqSolutionSet, RequestedGlobalProperties globPropsReqWorkset, RequestedLocalProperties locPropsReqSolutionSet, RequestedLocalProperties locPropsReqWorkset) {
    // check for pipeline breaking using hash join with build on the solution set side
    placePipelineBreakersIfNecessary(DriverStrategy.HYBRIDHASH_BUILD_FIRST, solutionSetIn, worksetIn);
    // NOTES ON THE ENUMERATION OF THE STEP FUNCTION PLANS:
    // Whenever we instantiate the iteration, we enumerate new candidates for the step function.
    // That way, we make sure we have an appropriate plan for each candidate for the initial partial solution,
    // we have a fitting candidate for the step function (often, work is pushed out of the step function).
    // Among the candidates of the step function, we keep only those that meet the requested properties of the
    // current candidate initial partial solution. That makes sure these properties exist at the beginning of
    // every iteration.
    // 1) Because we enumerate multiple times, we may need to clean the cached plans
    //    before starting another enumeration
    this.nextWorkset.accept(PlanCacheCleaner.INSTANCE);
    this.solutionSetDelta.accept(PlanCacheCleaner.INSTANCE);
    // 2) Give the partial solution the properties of the current candidate for the initial partial solution
    //    This concerns currently only the workset.
    this.worksetNode.setCandidateProperties(worksetIn.getGlobalProperties(), worksetIn.getLocalProperties(), worksetIn);
    this.solutionSetNode.setCandidateProperties(this.partitionedProperties, new LocalProperties(), solutionSetIn);
    final SolutionSetPlanNode sspn = this.solutionSetNode.getCurrentSolutionSetPlanNode();
    final WorksetPlanNode wspn = this.worksetNode.getCurrentWorksetPlanNode();
    // 3) Get the alternative plans
    List<PlanNode> solutionSetDeltaCandidates = this.solutionSetDelta.getAlternativePlans(estimator);
    List<PlanNode> worksetCandidates = this.nextWorkset.getAlternativePlans(estimator);
    // 4) Throw away all that are not compatible with the properties currently requested to the
    //    initial partial solution
    // Make sure that the workset candidates fulfill the input requirements
    {
        List<PlanNode> newCandidates = new ArrayList<PlanNode>();
        for (Iterator<PlanNode> planDeleter = worksetCandidates.iterator(); planDeleter.hasNext(); ) {
            PlanNode candidate = planDeleter.next();
            GlobalProperties atEndGlobal = candidate.getGlobalProperties();
            LocalProperties atEndLocal = candidate.getLocalProperties();
            FeedbackPropertiesMeetRequirementsReport report = candidate.checkPartialSolutionPropertiesMet(wspn, atEndGlobal, atEndLocal);
            if (report == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
            // depends only through broadcast variable on the workset solution
            } else if (report == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
                // attach a no-op node through which we create the properties of the original input
                Channel toNoOp = new Channel(candidate);
                globPropsReqWorkset.parameterizeChannel(toNoOp, false, nextWorksetRootConnection.getDataExchangeMode(), false);
                locPropsReqWorkset.parameterizeChannel(toNoOp);
                NoOpUnaryUdfOp noOpUnaryUdfOp = new NoOpUnaryUdfOp<>();
                noOpUnaryUdfOp.setInput(candidate.getProgramOperator());
                UnaryOperatorNode rebuildWorksetPropertiesNode = new UnaryOperatorNode("Rebuild Workset Properties", noOpUnaryUdfOp, true);
                rebuildWorksetPropertiesNode.setParallelism(candidate.getParallelism());
                SingleInputPlanNode rebuildWorksetPropertiesPlanNode = new SingleInputPlanNode(rebuildWorksetPropertiesNode, "Rebuild Workset Properties", toNoOp, DriverStrategy.UNARY_NO_OP);
                rebuildWorksetPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties());
                estimator.costOperator(rebuildWorksetPropertiesPlanNode);
                GlobalProperties atEndGlobalModified = rebuildWorksetPropertiesPlanNode.getGlobalProperties();
                LocalProperties atEndLocalModified = rebuildWorksetPropertiesPlanNode.getLocalProperties();
                if (!(atEndGlobalModified.equals(atEndGlobal) && atEndLocalModified.equals(atEndLocal))) {
                    FeedbackPropertiesMeetRequirementsReport report2 = candidate.checkPartialSolutionPropertiesMet(wspn, atEndGlobalModified, atEndLocalModified);
                    if (report2 != FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
                        newCandidates.add(rebuildWorksetPropertiesPlanNode);
                    }
                }
                // remove the original operator and add the modified candidate
                planDeleter.remove();
            }
        }
        worksetCandidates.addAll(newCandidates);
    }
    if (worksetCandidates.isEmpty()) {
        return;
    }
    // sanity check the solution set delta
    for (PlanNode solutionSetDeltaCandidate : solutionSetDeltaCandidates) {
        SingleInputPlanNode candidate = (SingleInputPlanNode) solutionSetDeltaCandidate;
        GlobalProperties gp = candidate.getGlobalProperties();
        if (gp.getPartitioning() != PartitioningProperty.HASH_PARTITIONED || gp.getPartitioningFields() == null || !gp.getPartitioningFields().equals(this.solutionSetKeyFields)) {
            throw new CompilerException("Bug: The solution set delta is not partitioned.");
        }
    }
    // 5) Create a candidate for the Iteration Node for every remaining plan of the step function.
    final GlobalProperties gp = new GlobalProperties();
    gp.setHashPartitioned(this.solutionSetKeyFields);
    gp.addUniqueFieldCombination(this.solutionSetKeyFields);
    LocalProperties lp = LocalProperties.EMPTY.addUniqueFields(this.solutionSetKeyFields);
    // take all combinations of solution set delta and workset plans
    for (PlanNode solutionSetCandidate : solutionSetDeltaCandidates) {
        for (PlanNode worksetCandidate : worksetCandidates) {
            // check whether they have the same operator at their latest branching point
            if (this.singleRoot.areBranchCompatible(solutionSetCandidate, worksetCandidate)) {
                SingleInputPlanNode siSolutionDeltaCandidate = (SingleInputPlanNode) solutionSetCandidate;
                boolean immediateDeltaUpdate;
                // check whether we need a dedicated solution set delta operator, or whether we can update on the fly
                if (siSolutionDeltaCandidate.getInput().getShipStrategy() == ShipStrategyType.FORWARD && this.solutionDeltaImmediatelyAfterSolutionJoin) {
                    // sanity check the node and connection
                    if (siSolutionDeltaCandidate.getDriverStrategy() != DriverStrategy.UNARY_NO_OP || siSolutionDeltaCandidate.getInput().getLocalStrategy() != LocalStrategy.NONE) {
                        throw new CompilerException("Invalid Solution set delta node.");
                    }
                    solutionSetCandidate = siSolutionDeltaCandidate.getInput().getSource();
                    immediateDeltaUpdate = true;
                } else {
                    // was not partitioned, we need to keep this node.
                    // mark that we materialize the input
                    siSolutionDeltaCandidate.getInput().setTempMode(TempMode.PIPELINE_BREAKER);
                    immediateDeltaUpdate = false;
                }
                WorksetIterationPlanNode wsNode = new WorksetIterationPlanNode(this, this.getOperator().getName(), solutionSetIn, worksetIn, sspn, wspn, worksetCandidate, solutionSetCandidate);
                wsNode.setImmediateSolutionSetUpdate(immediateDeltaUpdate);
                wsNode.initProperties(gp, lp);
                target.add(wsNode);
            }
        }
    }
}

Example 15 with CompilerException

use of org.apache.flink.optimizer.CompilerException in project flink by apache.

the class GlobalProperties method parameterizeChannel.

public void parameterizeChannel(Channel channel, boolean globalDopChange, ExecutionMode exchangeMode, boolean breakPipeline) {
    ShipStrategyType shipType;
    FieldList partitionKeys;
    boolean[] sortDirection;
    Partitioner<?> partitioner;
    switch(this.partitioning) {
        case RANDOM_PARTITIONED:
            shipType = globalDopChange ? ShipStrategyType.PARTITION_RANDOM : ShipStrategyType.FORWARD;
            partitionKeys = null;
            sortDirection = null;
            partitioner = null;
            break;
        case FULL_REPLICATION:
            shipType = ShipStrategyType.BROADCAST;
            partitionKeys = null;
            sortDirection = null;
            partitioner = null;
            break;
        case ANY_PARTITIONING:
        case HASH_PARTITIONED:
            shipType = ShipStrategyType.PARTITION_HASH;
            partitionKeys = Utils.createOrderedFromSet(this.partitioningFields);
            sortDirection = null;
            partitioner = null;
            break;
        case RANGE_PARTITIONED:
            shipType = ShipStrategyType.PARTITION_RANGE;
            partitionKeys = this.ordering.getInvolvedIndexes();
            sortDirection = this.ordering.getFieldSortDirections();
            partitioner = null;
            break;
        case FORCED_REBALANCED:
            shipType = ShipStrategyType.PARTITION_RANDOM;
            partitionKeys = null;
            sortDirection = null;
            partitioner = null;
            break;
        case CUSTOM_PARTITIONING:
            shipType = ShipStrategyType.PARTITION_CUSTOM;
            partitionKeys = this.partitioningFields;
            sortDirection = null;
            partitioner = this.customPartitioner;
            break;
        default:
            throw new CompilerException("Unsupported partitioning strategy");
    }
    channel.setDataDistribution(this.distribution);
    DataExchangeMode exMode = DataExchangeMode.select(exchangeMode, shipType, breakPipeline);
    channel.setShipStrategy(shipType, partitionKeys, sortDirection, partitioner, exMode);
}