Examples with ConsumerVertexGroup org.apache.flink.runtime.scheduler.strategy.ConsumerVertexGroup used on opensource projects

Example 1 with ConsumerVertexGroup

use of org.apache.flink.runtime.scheduler.strategy.ConsumerVertexGroup in project flink by apache.

the class EdgeManagerBuildUtil method connectAllToAll.

private static void connectAllToAll(ExecutionVertex[] taskVertices, IntermediateResult intermediateResult) {
    List<IntermediateResultPartitionID> consumedPartitions = Arrays.stream(intermediateResult.getPartitions()).map(IntermediateResultPartition::getPartitionId).collect(Collectors.toList());
    ConsumedPartitionGroup consumedPartitionGroup = createAndRegisterConsumedPartitionGroupToEdgeManager(consumedPartitions, intermediateResult);
    for (ExecutionVertex ev : taskVertices) {
        ev.addConsumedPartitionGroup(consumedPartitionGroup);
    }
    List<ExecutionVertexID> consumerVertices = Arrays.stream(taskVertices).map(ExecutionVertex::getID).collect(Collectors.toList());
    ConsumerVertexGroup consumerVertexGroup = ConsumerVertexGroup.fromMultipleVertices(consumerVertices);
    for (IntermediateResultPartition partition : intermediateResult.getPartitions()) {
        partition.addConsumers(consumerVertexGroup);
    }
}

Example 2 with ConsumerVertexGroup

use of org.apache.flink.runtime.scheduler.strategy.ConsumerVertexGroup in project flink by apache.

the class Execution method updatePartitionConsumers.

private void updatePartitionConsumers(final IntermediateResultPartition partition) {
    final Optional<ConsumerVertexGroup> consumerVertexGroup = partition.getConsumerVertexGroupOptional();
    if (!consumerVertexGroup.isPresent()) {
        return;
    }
    for (ExecutionVertexID consumerVertexId : consumerVertexGroup.get()) {
        final ExecutionVertex consumerVertex = vertex.getExecutionGraphAccessor().getExecutionVertexOrThrow(consumerVertexId);
        final Execution consumer = consumerVertex.getCurrentExecutionAttempt();
        final ExecutionState consumerState = consumer.getState();
        // ----------------------------------------------------------------
        if (consumerState == DEPLOYING || consumerState == RUNNING || consumerState == INITIALIZING) {
            final PartitionInfo partitionInfo = createPartitionInfo(partition);
            if (consumerState == DEPLOYING) {
                consumerVertex.cachePartitionInfo(partitionInfo);
            } else {
                consumer.sendUpdatePartitionInfoRpcCall(Collections.singleton(partitionInfo));
            }
        }
    }
}

Example 3 with ConsumerVertexGroup

use of org.apache.flink.runtime.scheduler.strategy.ConsumerVertexGroup in project flink by apache.

the class EdgeManagerBuildUtilTest method testGetMaxNumEdgesToTarget.

private void testGetMaxNumEdgesToTarget(int upstream, int downstream, DistributionPattern pattern) throws Exception {
    Pair<ExecutionJobVertex, ExecutionJobVertex> pair = setupExecutionGraph(upstream, downstream, pattern);
    ExecutionJobVertex upstreamEJV = pair.getLeft();
    ExecutionJobVertex downstreamEJV = pair.getRight();
    int calculatedMaxForUpstream = EdgeManagerBuildUtil.computeMaxEdgesToTargetExecutionVertex(upstream, downstream, pattern);
    int actualMaxForUpstream = -1;
    for (ExecutionVertex ev : upstreamEJV.getTaskVertices()) {
        assertEquals(1, ev.getProducedPartitions().size());
        IntermediateResultPartition partition = ev.getProducedPartitions().values().iterator().next();
        ConsumerVertexGroup consumerVertexGroup = partition.getConsumerVertexGroup();
        int actual = consumerVertexGroup.size();
        if (actual > actualMaxForUpstream) {
            actualMaxForUpstream = actual;
        }
    }
    assertEquals(actualMaxForUpstream, calculatedMaxForUpstream);
    int calculatedMaxForDownstream = EdgeManagerBuildUtil.computeMaxEdgesToTargetExecutionVertex(downstream, upstream, pattern);
    int actualMaxForDownstream = -1;
    for (ExecutionVertex ev : downstreamEJV.getTaskVertices()) {
        assertEquals(1, ev.getNumberOfInputs());
        int actual = ev.getConsumedPartitionGroup(0).size();
        if (actual > actualMaxForDownstream) {
            actualMaxForDownstream = actual;
        }
    }
    assertEquals(actualMaxForDownstream, calculatedMaxForDownstream);
}

Example 4 with ConsumerVertexGroup

use of org.apache.flink.runtime.scheduler.strategy.ConsumerVertexGroup in project flink by apache.

the class RestartPipelinedRegionFailoverStrategy method getConsumerVerticesToVisit.

private Iterable<ExecutionVertexID> getConsumerVerticesToVisit(SchedulingPipelinedRegion regionToRestart, Set<ConsumerVertexGroup> visitedConsumerVertexGroups) {
    final List<ConsumerVertexGroup> consumerVertexGroupsToVisit = new ArrayList<>();
    for (SchedulingExecutionVertex vertex : regionToRestart.getVertices()) {
        for (SchedulingResultPartition producedPartition : vertex.getProducedResults()) {
            final Optional<ConsumerVertexGroup> consumerVertexGroup = producedPartition.getConsumerVertexGroup();
            if (consumerVertexGroup.isPresent() && !visitedConsumerVertexGroups.contains(consumerVertexGroup.get())) {
                visitedConsumerVertexGroups.add(consumerVertexGroup.get());
                consumerVertexGroupsToVisit.add(consumerVertexGroup.get());
            }
        }
    }
    return IterableUtils.flatMap(consumerVertexGroupsToVisit, Function.identity());
}

Example 5 with ConsumerVertexGroup

use of org.apache.flink.runtime.scheduler.strategy.ConsumerVertexGroup in project flink by apache.

the class RestartPipelinedRegionFailoverStrategy method getRegionsToRestart.

/**
 * All 'involved' regions are proposed to be restarted. The 'involved' regions are calculated
 * with rules below: 1. The region containing the failed task is always involved 2. If an input
 * result partition of an involved region is not available, i.e. Missing or Corrupted, the
 * region containing the partition producer task is involved 3. If a region is involved, all of
 * its consumer regions are involved
 */
private Set<SchedulingPipelinedRegion> getRegionsToRestart(SchedulingPipelinedRegion failedRegion) {
    Set<SchedulingPipelinedRegion> regionsToRestart = Collections.newSetFromMap(new IdentityHashMap<>());
    Set<SchedulingPipelinedRegion> visitedRegions = Collections.newSetFromMap(new IdentityHashMap<>());
    Set<ConsumedPartitionGroup> visitedConsumedResultGroups = Collections.newSetFromMap(new IdentityHashMap<>());
    Set<ConsumerVertexGroup> visitedConsumerVertexGroups = Collections.newSetFromMap(new IdentityHashMap<>());
    // start from the failed region to visit all involved regions
    Queue<SchedulingPipelinedRegion> regionsToVisit = new ArrayDeque<>();
    visitedRegions.add(failedRegion);
    regionsToVisit.add(failedRegion);
    while (!regionsToVisit.isEmpty()) {
        SchedulingPipelinedRegion regionToRestart = regionsToVisit.poll();
        // an involved region should be restarted
        regionsToRestart.add(regionToRestart);
        // if a needed input result partition is not available, its producer region is involved
        for (IntermediateResultPartitionID consumedPartitionId : getConsumedPartitionsToVisit(regionToRestart, visitedConsumedResultGroups)) {
            if (!resultPartitionAvailabilityChecker.isAvailable(consumedPartitionId)) {
                SchedulingResultPartition consumedPartition = topology.getResultPartition(consumedPartitionId);
                SchedulingPipelinedRegion producerRegion = topology.getPipelinedRegionOfVertex(consumedPartition.getProducer().getId());
                if (!visitedRegions.contains(producerRegion)) {
                    visitedRegions.add(producerRegion);
                    regionsToVisit.add(producerRegion);
                }
            }
        }
        // all consumer regions of an involved region should be involved
        for (ExecutionVertexID consumerVertexId : getConsumerVerticesToVisit(regionToRestart, visitedConsumerVertexGroups)) {
            SchedulingPipelinedRegion consumerRegion = topology.getPipelinedRegionOfVertex(consumerVertexId);
            if (!visitedRegions.contains(consumerRegion)) {
                visitedRegions.add(consumerRegion);
                regionsToVisit.add(consumerRegion);
            }
        }
    }
    return regionsToRestart;
}