Examples with IntermediateResultPartitionID org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID used on opensource projects

Example 46 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class RegionPartitionGroupReleaseStrategyTest method releasePartitionsIfDownstreamRegionWithMultipleOperatorsIsFinished.

@Test
public void releasePartitionsIfDownstreamRegionWithMultipleOperatorsIsFinished() {
    final List<TestingSchedulingExecutionVertex> sourceVertices = testingSchedulingTopology.addExecutionVertices().finish();
    final List<TestingSchedulingExecutionVertex> intermediateVertices = testingSchedulingTopology.addExecutionVertices().finish();
    final List<TestingSchedulingExecutionVertex> sinkVertices = testingSchedulingTopology.addExecutionVertices().finish();
    final List<TestingSchedulingResultPartition> sourceResultPartitions = testingSchedulingTopology.connectAllToAll(sourceVertices, intermediateVertices).finish();
    testingSchedulingTopology.connectAllToAll(intermediateVertices, sinkVertices).withResultPartitionType(ResultPartitionType.PIPELINED).finish();
    final ExecutionVertexID onlyIntermediateVertexId = intermediateVertices.get(0).getId();
    final ExecutionVertexID onlySinkVertexId = sinkVertices.get(0).getId();
    final IntermediateResultPartitionID onlySourceResultPartitionId = sourceResultPartitions.get(0).getId();
    final RegionPartitionGroupReleaseStrategy regionPartitionGroupReleaseStrategy = new RegionPartitionGroupReleaseStrategy(testingSchedulingTopology);
    regionPartitionGroupReleaseStrategy.vertexFinished(onlyIntermediateVertexId);
    final List<IntermediateResultPartitionID> partitionsToRelease = getReleasablePartitions(regionPartitionGroupReleaseStrategy, onlySinkVertexId);
    assertThat(partitionsToRelease, contains(onlySourceResultPartitionId));
}

Example 47 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class RegionPartitionGroupReleaseStrategyTest method notReleasePartitionsIfDownstreamRegionIsNotFinished.

@Test
public void notReleasePartitionsIfDownstreamRegionIsNotFinished() {
    final List<TestingSchedulingExecutionVertex> producers = testingSchedulingTopology.addExecutionVertices().finish();
    final List<TestingSchedulingExecutionVertex> consumers = testingSchedulingTopology.addExecutionVertices().withParallelism(2).finish();
    testingSchedulingTopology.connectAllToAll(producers, consumers).finish();
    final ExecutionVertexID consumerVertex1 = consumers.get(0).getId();
    final RegionPartitionGroupReleaseStrategy regionPartitionGroupReleaseStrategy = new RegionPartitionGroupReleaseStrategy(testingSchedulingTopology);
    final List<IntermediateResultPartitionID> partitionsToRelease = getReleasablePartitions(regionPartitionGroupReleaseStrategy, consumerVertex1);
    assertThat(partitionsToRelease, is(empty()));
}

Example 48 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class RegionPartitionGroupReleaseStrategyTest method releasePartitionsIfDownstreamRegionIsFinished.

@Test
public void releasePartitionsIfDownstreamRegionIsFinished() {
    final List<TestingSchedulingExecutionVertex> producers = testingSchedulingTopology.addExecutionVertices().finish();
    final List<TestingSchedulingExecutionVertex> consumers = testingSchedulingTopology.addExecutionVertices().finish();
    final List<TestingSchedulingResultPartition> resultPartitions = testingSchedulingTopology.connectPointwise(producers, consumers).finish();
    final ExecutionVertexID onlyConsumerVertexId = consumers.get(0).getId();
    final IntermediateResultPartitionID onlyResultPartitionId = resultPartitions.get(0).getId();
    final RegionPartitionGroupReleaseStrategy regionPartitionGroupReleaseStrategy = new RegionPartitionGroupReleaseStrategy(testingSchedulingTopology);
    final List<IntermediateResultPartitionID> partitionsToRelease = getReleasablePartitions(regionPartitionGroupReleaseStrategy, onlyConsumerVertexId);
    assertThat(partitionsToRelease, contains(onlyResultPartitionId));
}

Example 49 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class DefaultExecutionTopologyTest method assertGraphEquals.

private static void assertGraphEquals(ExecutionGraph originalGraph, DefaultExecutionTopology adaptedTopology) {
    Iterator<ExecutionVertex> originalVertices = originalGraph.getAllExecutionVertices().iterator();
    Iterator<DefaultExecutionVertex> adaptedVertices = adaptedTopology.getVertices().iterator();
    while (originalVertices.hasNext()) {
        ExecutionVertex originalVertex = originalVertices.next();
        DefaultExecutionVertex adaptedVertex = adaptedVertices.next();
        assertEquals(originalVertex.getID(), adaptedVertex.getId());
        List<IntermediateResultPartition> originalConsumedPartitions = new ArrayList<>();
        for (ConsumedPartitionGroup consumedPartitionGroup : originalVertex.getAllConsumedPartitionGroups()) {
            for (IntermediateResultPartitionID partitionId : consumedPartitionGroup) {
                IntermediateResultPartition partition = originalVertex.getExecutionGraphAccessor().getResultPartitionOrThrow(partitionId);
                originalConsumedPartitions.add(partition);
            }
        }
        Iterable<DefaultResultPartition> adaptedConsumedPartitions = adaptedVertex.getConsumedResults();
        assertPartitionsEquals(originalConsumedPartitions, adaptedConsumedPartitions);
        Collection<IntermediateResultPartition> originalProducedPartitions = originalVertex.getProducedPartitions().values();
        Iterable<DefaultResultPartition> adaptedProducedPartitions = adaptedVertex.getProducedResults();
        assertPartitionsEquals(originalProducedPartitions, adaptedProducedPartitions);
    }
    assertFalse("Number of adapted vertices exceeds number of original vertices.", adaptedVertices.hasNext());
}

Example 50 with IntermediateResultPartitionID

use of org.apache.flink.runtime.jobgraph.IntermediateResultPartitionID in project flink by apache.

the class DefaultExecutionTopologyTest method testGetResultPartition.

@Test
public void testGetResultPartition() {
    for (ExecutionVertex vertex : executionGraph.getAllExecutionVertices()) {
        for (Map.Entry<IntermediateResultPartitionID, IntermediateResultPartition> entry : vertex.getProducedPartitions().entrySet()) {
            IntermediateResultPartition partition = entry.getValue();
            DefaultResultPartition schedulingResultPartition = adapter.getResultPartition(entry.getKey());
            assertPartitionEquals(partition, schedulingResultPartition);
        }
    }
}