Examples with DistributionPattern org.apache.flink.runtime.jobgraph.DistributionPattern used on opensource projects

Example 6 with DistributionPattern

use of org.apache.flink.runtime.jobgraph.DistributionPattern in project flink by apache.

the class DefaultCheckpointPlanCalculator method someTasksMustBeTriggered.

private boolean someTasksMustBeTriggered(Map<JobVertexID, BitSet> runningTasksByVertex, List<JobEdge> prevJobEdges) {
    for (JobEdge jobEdge : prevJobEdges) {
        DistributionPattern distributionPattern = jobEdge.getDistributionPattern();
        BitSet upstreamRunningStatus = runningTasksByVertex.get(jobEdge.getSource().getProducer().getID());
        if (hasActiveUpstreamVertex(distributionPattern, upstreamRunningStatus)) {
            return false;
        }
    }
    return true;
}

Example 7 with DistributionPattern

use of org.apache.flink.runtime.jobgraph.DistributionPattern in project flink by apache.

the class VertexFinishedStateChecker method checkPredecessorsOfPartiallyFinishedVertex.

private void checkPredecessorsOfPartiallyFinishedVertex(ExecutionJobVertex vertex, VerticesFinishedStatusCache verticesFinishedStatusCache) {
    // Computes the distribution pattern from each predecessor. If there are multiple edges
    // from a single predecessor, ALL_TO_ALL edges would have a higher priority since it
    // implies stricter limitation (must be fully finished).
    Map<JobVertexID, DistributionPattern> predecessorDistribution = new HashMap<>();
    for (JobEdge jobEdge : vertex.getJobVertex().getInputs()) {
        predecessorDistribution.compute(jobEdge.getSource().getProducer().getID(), (k, v) -> v == DistributionPattern.ALL_TO_ALL ? v : jobEdge.getDistributionPattern());
    }
    for (IntermediateResult dataset : vertex.getInputs()) {
        ExecutionJobVertex predecessor = dataset.getProducer();
        VertexFinishedState predecessorState = verticesFinishedStatusCache.getOrUpdate(predecessor);
        DistributionPattern distribution = predecessorDistribution.get(predecessor.getJobVertexId());
        if (distribution == DistributionPattern.ALL_TO_ALL && predecessorState != VertexFinishedState.FULLY_FINISHED) {
            throw new FlinkRuntimeException("Illegal JobGraph modification. Cannot run a program with partially finished" + " vertices predeceased with running or partially finished ones and" + " connected via the ALL_TO_ALL edges. Task vertex " + vertex.getName() + "(" + vertex.getJobVertexId() + ")" + " has a " + (predecessorState == VertexFinishedState.ALL_RUNNING ? "all running" : "partially finished") + " predecessor");
        } else if (distribution == DistributionPattern.POINTWISE && predecessorState == VertexFinishedState.ALL_RUNNING) {
            throw new FlinkRuntimeException("Illegal JobGraph modification. Cannot run a program with partially finished" + " vertices predeceased with all running ones. Task vertex " + vertex.getName() + "(" + vertex.getJobVertexId() + ")" + " has a all running predecessor");
        }
    }
}