Examples with ResultSubpartitionView org.apache.flink.runtime.io.network.partition.ResultSubpartitionView used on opensource projects

Example 16 with ResultSubpartitionView

use of org.apache.flink.runtime.io.network.partition.ResultSubpartitionView in project flink by apache.

the class LocalInputChannelTest method testGetNextAfterPartitionReleased.

/**
 * Tests that reading from a channel when after the partition has been released are handled and
 * don't lead to NPEs.
 */
@Test
public void testGetNextAfterPartitionReleased() throws Exception {
    ResultSubpartitionView subpartitionView = InputChannelTestUtils.createResultSubpartitionView(false);
    TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(subpartitionView);
    LocalInputChannel channel = createLocalInputChannel(new SingleInputGateBuilder().build(), partitionManager);
    channel.requestSubpartition();
    assertFalse(channel.getNextBuffer().isPresent());
    // release the subpartition view
    subpartitionView.releaseAllResources();
    try {
        channel.getNextBuffer();
        fail("Did not throw expected CancelTaskException");
    } catch (CancelTaskException ignored) {
    }
    channel.releaseAllResources();
    assertFalse(channel.getNextBuffer().isPresent());
}

Example 17 with ResultSubpartitionView

use of org.apache.flink.runtime.io.network.partition.ResultSubpartitionView in project flink by apache.

the class LocalInputChannelTest method testReceivingBuffersInUseBeforeSubpartitionViewInitialization.

@Test
public void testReceivingBuffersInUseBeforeSubpartitionViewInitialization() throws Exception {
    // given: Local input channel without initialized subpartition view.
    ResultSubpartitionView subpartitionView = InputChannelTestUtils.createResultSubpartitionView(createFilledFinishedBufferConsumer(4096), createFilledFinishedBufferConsumer(4096), createFilledFinishedBufferConsumer(4096));
    TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(subpartitionView);
    final SingleInputGate inputGate = createSingleInputGate(1);
    final LocalInputChannel localChannel = createLocalInputChannel(inputGate, partitionManager);
    inputGate.setInputChannels(localChannel);
    // then: Buffers in use should be equal to 0 until subpartition view initialization.
    assertEquals(0, localChannel.getBuffersInUseCount());
    // when: The subpartition view is initialized.
    localChannel.requestSubpartition();
    // then: Buffers in use should show correct value.
    assertEquals(3, localChannel.getBuffersInUseCount());
}

Example 18 with ResultSubpartitionView

use of org.apache.flink.runtime.io.network.partition.ResultSubpartitionView in project flink by apache.

the class LocalInputChannelTest method testCheckpointingInflightData.

@Test
public void testCheckpointingInflightData() throws Exception {
    SingleInputGate inputGate = new SingleInputGateBuilder().build();
    PipelinedResultPartition parent = (PipelinedResultPartition) PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED, NoOpFileChannelManager.INSTANCE);
    ResultSubpartition subpartition = parent.getAllPartitions()[0];
    ResultSubpartitionView subpartitionView = subpartition.createReadView(() -> {
    });
    TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(subpartitionView);
    final RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    LocalInputChannel channel = createLocalInputChannel(inputGate, partitionManager, 0, 0, b -> b.setStateWriter(stateWriter));
    inputGate.setInputChannels(channel);
    channel.requestSubpartition();
    final CheckpointStorageLocationReference location = getDefault();
    CheckpointOptions options = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, location);
    stateWriter.start(0, options);
    final CheckpointBarrier barrier = new CheckpointBarrier(0, 123L, options);
    channel.checkpointStarted(barrier);
    // add 1 buffer before barrier and 1 buffer afterwards. Only the first buffer should be
    // written.
    subpartition.add(createFilledFinishedBufferConsumer(1));
    assertTrue(channel.getNextBuffer().isPresent());
    subpartition.add(EventSerializer.toBufferConsumer(barrier, true));
    assertTrue(channel.getNextBuffer().isPresent());
    subpartition.add(createFilledFinishedBufferConsumer(2));
    assertTrue(channel.getNextBuffer().isPresent());
    assertArrayEquals(stateWriter.getAddedInput().get(channel.getChannelInfo()).stream().mapToInt(Buffer::getSize).toArray(), new int[] { 1 });
}

Example 19 with ResultSubpartitionView

use of org.apache.flink.runtime.io.network.partition.ResultSubpartitionView in project flink by apache.

the class LocalInputChannelTest method testProducerFailedException.

@Test(expected = CancelTaskException.class)
public void testProducerFailedException() throws Exception {
    ResultSubpartitionView view = mock(ResultSubpartitionView.class);
    when(view.isReleased()).thenReturn(true);
    when(view.getFailureCause()).thenReturn(new Exception("Expected test exception"));
    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    when(partitionManager.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class))).thenReturn(view);
    SingleInputGate inputGate = mock(SingleInputGate.class);
    BufferProvider bufferProvider = mock(BufferProvider.class);
    when(inputGate.getBufferProvider()).thenReturn(bufferProvider);
    LocalInputChannel ch = createLocalInputChannel(inputGate, partitionManager);
    ch.requestSubpartition();
    // Should throw an instance of CancelTaskException.
    ch.getNextBuffer();
}

Example 20 with ResultSubpartitionView

use of org.apache.flink.runtime.io.network.partition.ResultSubpartitionView in project flink by apache.

the class LocalInputChannelTest method testConcurrentReleaseAndRetriggerPartitionRequest.

/**
 * Verifies that concurrent release via the SingleInputGate and re-triggering of a partition
 * request works smoothly.
 *
 * <ul>
 *   <li>SingleInputGate acquires its request lock and tries to release all registered channels.
 *       When releasing a channel, it needs to acquire the channel's shared request-release
 *       lock.
 *   <li>If a LocalInputChannel concurrently retriggers a partition request via a Timer Thread
 *       it acquires the channel's request-release lock and calls the retrigger callback on the
 *       SingleInputGate, which again tries to acquire the gate's request lock.
 * </ul>
 *
 * <p>For certain timings this obviously leads to a deadlock. This test reliably reproduced such
 * a timing (reported in FLINK-5228). This test is pretty much testing the buggy implementation
 * and has not much more general value. If it becomes obsolete at some point (future greatness
 * ;)), feel free to remove it.
 *
 * <p>The fix in the end was to to not acquire the channels lock when releasing it and/or not
 * doing any input gate callbacks while holding the channel's lock. I decided to do both.
 */
@Test
public void testConcurrentReleaseAndRetriggerPartitionRequest() throws Exception {
    final SingleInputGate gate = createSingleInputGate(1);
    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    when(partitionManager.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class))).thenAnswer((Answer<ResultSubpartitionView>) invocationOnMock -> {
        Thread.sleep(100);
        throw new PartitionNotFoundException(new ResultPartitionID());
    });
    final LocalInputChannel channel = createLocalInputChannel(gate, partitionManager, 1, 1);
    Thread releaser = new Thread(() -> {
        try {
            gate.close();
        } catch (IOException ignored) {
        }
    });
    Thread requester = new Thread(() -> {
        try {
            channel.requestSubpartition();
        } catch (IOException ignored) {
        }
    });
    requester.start();
    releaser.start();
    releaser.join();
    requester.join();
}