Examples with SchedulingResultPartition org.apache.flink.runtime.scheduler.strategy.SchedulingResultPartition used on opensource projects

Example 1 with SchedulingResultPartition

use of org.apache.flink.runtime.scheduler.strategy.SchedulingResultPartition in project flink by apache.

the class RestartPipelinedRegionFailoverStrategy method getConsumerVerticesToVisit.

private Iterable<ExecutionVertexID> getConsumerVerticesToVisit(SchedulingPipelinedRegion regionToRestart, Set<ConsumerVertexGroup> visitedConsumerVertexGroups) {
    final List<ConsumerVertexGroup> consumerVertexGroupsToVisit = new ArrayList<>();
    for (SchedulingExecutionVertex vertex : regionToRestart.getVertices()) {
        for (SchedulingResultPartition producedPartition : vertex.getProducedResults()) {
            final Optional<ConsumerVertexGroup> consumerVertexGroup = producedPartition.getConsumerVertexGroup();
            if (consumerVertexGroup.isPresent() && !visitedConsumerVertexGroups.contains(consumerVertexGroup.get())) {
                visitedConsumerVertexGroups.add(consumerVertexGroup.get());
                consumerVertexGroupsToVisit.add(consumerVertexGroup.get());
            }
        }
    }
    return IterableUtils.flatMap(consumerVertexGroupsToVisit, Function.identity());
}

Example 2 with SchedulingResultPartition

use of org.apache.flink.runtime.scheduler.strategy.SchedulingResultPartition in project flink by apache.

the class RestartPipelinedRegionFailoverStrategy method getRegionsToRestart.

/**
 * All 'involved' regions are proposed to be restarted. The 'involved' regions are calculated
 * with rules below: 1. The region containing the failed task is always involved 2. If an input
 * result partition of an involved region is not available, i.e. Missing or Corrupted, the
 * region containing the partition producer task is involved 3. If a region is involved, all of
 * its consumer regions are involved
 */
private Set<SchedulingPipelinedRegion> getRegionsToRestart(SchedulingPipelinedRegion failedRegion) {
    Set<SchedulingPipelinedRegion> regionsToRestart = Collections.newSetFromMap(new IdentityHashMap<>());
    Set<SchedulingPipelinedRegion> visitedRegions = Collections.newSetFromMap(new IdentityHashMap<>());
    Set<ConsumedPartitionGroup> visitedConsumedResultGroups = Collections.newSetFromMap(new IdentityHashMap<>());
    Set<ConsumerVertexGroup> visitedConsumerVertexGroups = Collections.newSetFromMap(new IdentityHashMap<>());
    // start from the failed region to visit all involved regions
    Queue<SchedulingPipelinedRegion> regionsToVisit = new ArrayDeque<>();
    visitedRegions.add(failedRegion);
    regionsToVisit.add(failedRegion);
    while (!regionsToVisit.isEmpty()) {
        SchedulingPipelinedRegion regionToRestart = regionsToVisit.poll();
        // an involved region should be restarted
        regionsToRestart.add(regionToRestart);
        // if a needed input result partition is not available, its producer region is involved
        for (IntermediateResultPartitionID consumedPartitionId : getConsumedPartitionsToVisit(regionToRestart, visitedConsumedResultGroups)) {
            if (!resultPartitionAvailabilityChecker.isAvailable(consumedPartitionId)) {
                SchedulingResultPartition consumedPartition = topology.getResultPartition(consumedPartitionId);
                SchedulingPipelinedRegion producerRegion = topology.getPipelinedRegionOfVertex(consumedPartition.getProducer().getId());
                if (!visitedRegions.contains(producerRegion)) {
                    visitedRegions.add(producerRegion);
                    regionsToVisit.add(producerRegion);
                }
            }
        }
        // all consumer regions of an involved region should be involved
        for (ExecutionVertexID consumerVertexId : getConsumerVerticesToVisit(regionToRestart, visitedConsumerVertexGroups)) {
            SchedulingPipelinedRegion consumerRegion = topology.getPipelinedRegionOfVertex(consumerVertexId);
            if (!visitedRegions.contains(consumerRegion)) {
                visitedRegions.add(consumerRegion);
                regionsToVisit.add(consumerRegion);
            }
        }
    }
    return regionsToRestart;
}

Example 3 with SchedulingResultPartition

use of org.apache.flink.runtime.scheduler.strategy.SchedulingResultPartition in project flink by apache.

the class SchedulingPipelinedRegionComputeUtil method buildOutEdgesDesc.

private static List<List<Integer>> buildOutEdgesDesc(final Map<SchedulingExecutionVertex, Set<SchedulingExecutionVertex>> vertexToRegion, final List<Set<SchedulingExecutionVertex>> regionList, final Function<ExecutionVertexID, ? extends SchedulingExecutionVertex> executionVertexRetriever) {
    final Map<Set<SchedulingExecutionVertex>, Integer> regionIndices = new IdentityHashMap<>();
    for (int i = 0; i < regionList.size(); i++) {
        regionIndices.put(regionList.get(i), i);
    }
    final List<List<Integer>> outEdges = new ArrayList<>(regionList.size());
    for (Set<SchedulingExecutionVertex> currentRegion : regionList) {
        final List<Integer> currentRegionOutEdges = new ArrayList<>();
        for (SchedulingExecutionVertex vertex : currentRegion) {
            for (SchedulingResultPartition producedResult : vertex.getProducedResults()) {
                if (!producedResult.getResultType().isReconnectable()) {
                    continue;
                }
                final Optional<ConsumerVertexGroup> consumerVertexGroup = producedResult.getConsumerVertexGroup();
                if (!consumerVertexGroup.isPresent()) {
                    continue;
                }
                for (ExecutionVertexID consumerVertexId : consumerVertexGroup.get()) {
                    SchedulingExecutionVertex consumerVertex = executionVertexRetriever.apply(consumerVertexId);
                    // regions and cannot be merged
                    if (!vertexToRegion.containsKey(consumerVertex)) {
                        break;
                    }
                    if (!currentRegion.contains(consumerVertex)) {
                        currentRegionOutEdges.add(regionIndices.get(vertexToRegion.get(consumerVertex)));
                    }
                }
            }
        }
        outEdges.add(currentRegionOutEdges);
    }
    return outEdges;
}

Example 4 with SchedulingResultPartition

use of org.apache.flink.runtime.scheduler.strategy.SchedulingResultPartition in project flink by apache.

the class DefaultSchedulingPipelinedRegion method initializeAllBlockingConsumedPartitionGroups.

private void initializeAllBlockingConsumedPartitionGroups() {
    final Set<ConsumedPartitionGroup> consumedPartitionGroupSet = new HashSet<>();
    for (DefaultExecutionVertex executionVertex : executionVertices.values()) {
        for (ConsumedPartitionGroup consumedPartitionGroup : executionVertex.getConsumedPartitionGroups()) {
            SchedulingResultPartition consumedPartition = resultPartitionRetriever.apply(consumedPartitionGroup.getFirst());
            if (consumedPartition.getResultType().isBlocking()) {
                consumedPartitionGroupSet.add(consumedPartitionGroup);
            }
        }
    }
    this.blockingConsumedPartitionGroups = Collections.unmodifiableSet(consumedPartitionGroupSet);
}

Example 5 with SchedulingResultPartition

use of org.apache.flink.runtime.scheduler.strategy.SchedulingResultPartition in project flink by apache.

the class DefaultExecutionVertexTest method setUp.

@Before
public void setUp() throws Exception {
    intermediateResultPartitionId = new IntermediateResultPartitionID();
    DefaultResultPartition schedulingResultPartition = new DefaultResultPartition(intermediateResultPartitionId, new IntermediateDataSetID(), BLOCKING, () -> ResultPartitionState.CREATED, () -> {
        throw new UnsupportedOperationException();
    }, () -> {
        throw new UnsupportedOperationException();
    });
    producerVertex = new DefaultExecutionVertex(new ExecutionVertexID(new JobVertexID(), 0), Collections.singletonList(schedulingResultPartition), stateSupplier, Collections.emptyList(), partitionID -> {
        throw new UnsupportedOperationException();
    });
    schedulingResultPartition.setProducer(producerVertex);
    List<ConsumedPartitionGroup> consumedPartitionGroups = Collections.singletonList(ConsumedPartitionGroup.fromSinglePartition(intermediateResultPartitionId));
    Map<IntermediateResultPartitionID, DefaultResultPartition> resultPartitionById = Collections.singletonMap(intermediateResultPartitionId, schedulingResultPartition);
    consumerVertex = new DefaultExecutionVertex(new ExecutionVertexID(new JobVertexID(), 0), Collections.emptyList(), stateSupplier, consumedPartitionGroups, resultPartitionById::get);
}