Examples with ConsumedPartitionGroup org.apache.flink.runtime.scheduler.strategy.ConsumedPartitionGroup used on opensource projects

Example 11 with ConsumedPartitionGroup

use of org.apache.flink.runtime.scheduler.strategy.ConsumedPartitionGroup in project flink by apache.

the class PointwisePatternTest method test3NToN.

@Test
public void test3NToN() throws Exception {
    final int N = 17;
    ExecutionJobVertex target = setUpExecutionGraphAndGetDownstreamVertex(3 * N, N);
    for (ExecutionVertex ev : target.getTaskVertices()) {
        assertEquals(1, ev.getNumberOfInputs());
        ConsumedPartitionGroup consumedPartitionGroup = ev.getConsumedPartitionGroup(0);
        assertEquals(3, consumedPartitionGroup.size());
        int idx = 0;
        for (IntermediateResultPartitionID partitionId : consumedPartitionGroup) {
            assertEquals(ev.getParallelSubtaskIndex() * 3L + idx++, partitionId.getPartitionNumber());
        }
    }
}

Example 12 with ConsumedPartitionGroup

use of org.apache.flink.runtime.scheduler.strategy.ConsumedPartitionGroup in project flink by apache.

the class PointwisePatternTest method testLowToHigh.

private void testLowToHigh(int lowDop, int highDop) throws Exception {
    if (highDop < lowDop) {
        throw new IllegalArgumentException();
    }
    final int factor = highDop / lowDop;
    final int delta = highDop % lowDop == 0 ? 0 : 1;
    ExecutionJobVertex target = setUpExecutionGraphAndGetDownstreamVertex(lowDop, highDop);
    int[] timesUsed = new int[lowDop];
    for (ExecutionVertex ev : target.getTaskVertices()) {
        assertEquals(1, ev.getNumberOfInputs());
        ConsumedPartitionGroup consumedPartitionGroup = ev.getConsumedPartitionGroup(0);
        assertEquals(1, consumedPartitionGroup.size());
        timesUsed[consumedPartitionGroup.getFirst().getPartitionNumber()]++;
    }
    for (int used : timesUsed) {
        assertTrue(used >= factor && used <= factor + delta);
    }
}

Example 13 with ConsumedPartitionGroup

use of org.apache.flink.runtime.scheduler.strategy.ConsumedPartitionGroup in project flink by apache.

the class PointwisePatternTest method test2NToN.

@Test
public void test2NToN() throws Exception {
    final int N = 17;
    ExecutionJobVertex target = setUpExecutionGraphAndGetDownstreamVertex(2 * N, N);
    for (ExecutionVertex ev : target.getTaskVertices()) {
        assertEquals(1, ev.getNumberOfInputs());
        ConsumedPartitionGroup consumedPartitionGroup = ev.getConsumedPartitionGroup(0);
        assertEquals(2, consumedPartitionGroup.size());
        int idx = 0;
        for (IntermediateResultPartitionID partitionId : consumedPartitionGroup) {
            assertEquals(ev.getParallelSubtaskIndex() * 2L + idx++, partitionId.getPartitionNumber());
        }
    }
}

Example 14 with ConsumedPartitionGroup

use of org.apache.flink.runtime.scheduler.strategy.ConsumedPartitionGroup in project flink by apache.

the class PointwisePatternTest method testNTo7N.

@Test
public void testNTo7N() throws Exception {
    final int N = 11;
    ExecutionJobVertex target = setUpExecutionGraphAndGetDownstreamVertex(N, 7 * N);
    for (ExecutionVertex ev : target.getTaskVertices()) {
        assertEquals(1, ev.getNumberOfInputs());
        ConsumedPartitionGroup consumedPartitionGroup = ev.getConsumedPartitionGroup(0);
        assertEquals(1, consumedPartitionGroup.size());
        assertEquals(ev.getParallelSubtaskIndex() / 7, consumedPartitionGroup.getFirst().getPartitionNumber());
    }
}

Example 15 with ConsumedPartitionGroup

use of org.apache.flink.runtime.scheduler.strategy.ConsumedPartitionGroup in project flink by apache.

the class IntermediateResultPartitionTest method testBlockingPartitionConsumable.

@Test
public void testBlockingPartitionConsumable() throws Exception {
    IntermediateResult result = createResult(ResultPartitionType.BLOCKING, 2);
    IntermediateResultPartition partition1 = result.getPartitions()[0];
    IntermediateResultPartition partition2 = result.getPartitions()[1];
    ConsumedPartitionGroup consumedPartitionGroup = partition1.getConsumedPartitionGroups().get(0);
    // Not consumable on init
    assertFalse(partition1.isConsumable());
    assertFalse(partition2.isConsumable());
    assertFalse(consumedPartitionGroup.areAllPartitionsFinished());
    // Not consumable if only one partition is FINISHED
    partition1.markFinished();
    assertTrue(partition1.isConsumable());
    assertFalse(partition2.isConsumable());
    assertFalse(consumedPartitionGroup.areAllPartitionsFinished());
    // Consumable after all partitions are FINISHED
    partition2.markFinished();
    assertTrue(partition1.isConsumable());
    assertTrue(partition2.isConsumable());
    assertTrue(consumedPartitionGroup.areAllPartitionsFinished());
    // Not consumable if failover happens
    result.resetForNewExecution();
    assertFalse(partition1.isConsumable());
    assertFalse(partition2.isConsumable());
    assertFalse(consumedPartitionGroup.areAllPartitionsFinished());
}