Examples with ResultPartitionManager org.apache.flink.runtime.io.network.partition.ResultPartitionManager used on opensource projects

Example 11 with ResultPartitionManager

use of org.apache.flink.runtime.io.network.partition.ResultPartitionManager in project flink by apache.

the class LocalInputChannelTest method testPartitionNotFoundExceptionWhileRequestingPartition.

/**
 * Tests that {@link LocalInputChannel#requestSubpartition()} throws {@link
 * PartitionNotFoundException} if the result partition was not registered in {@link
 * ResultPartitionManager} and no backoff.
 */
@Test
public void testPartitionNotFoundExceptionWhileRequestingPartition() throws Exception {
    final SingleInputGate inputGate = createSingleInputGate(1);
    final LocalInputChannel localChannel = createLocalInputChannel(inputGate, new ResultPartitionManager());
    try {
        localChannel.requestSubpartition();
        fail("Should throw a PartitionNotFoundException.");
    } catch (PartitionNotFoundException notFound) {
        assertThat(localChannel.getPartitionId(), Matchers.is(notFound.getPartitionId()));
    }
}

Example 12 with ResultPartitionManager

use of org.apache.flink.runtime.io.network.partition.ResultPartitionManager in project flink by apache.

the class LocalInputChannelTest method testChannelErrorWhileRetriggeringRequest.

/**
 * Tests that {@link LocalInputChannel#retriggerSubpartitionRequest(Timer)} would throw {@link
 * PartitionNotFoundException} which is set onto the input channel then.
 */
@Test
public void testChannelErrorWhileRetriggeringRequest() {
    final SingleInputGate inputGate = createSingleInputGate(1);
    final LocalInputChannel localChannel = createLocalInputChannel(inputGate, new ResultPartitionManager());
    final Timer timer = new Timer(true) {

        @Override
        public void schedule(TimerTask task, long delay) {
            task.run();
            try {
                localChannel.checkError();
                fail("Should throw a PartitionNotFoundException.");
            } catch (PartitionNotFoundException notFound) {
                assertThat(localChannel.partitionId, Matchers.is(notFound.getPartitionId()));
            } catch (IOException ex) {
                fail("Should throw a PartitionNotFoundException.");
            }
        }
    };
    try {
        localChannel.retriggerSubpartitionRequest(timer);
    } finally {
        timer.cancel();
    }
}

Example 13 with ResultPartitionManager

use of org.apache.flink.runtime.io.network.partition.ResultPartitionManager in project flink by apache.

the class LocalInputChannelTest method testPartitionRequestExponentialBackoff.

@Test
public void testPartitionRequestExponentialBackoff() throws Exception {
    // Config
    int initialBackoff = 500;
    int maxBackoff = 3000;
    // Start with initial backoff, then keep doubling, and cap at max.
    int[] expectedDelays = { initialBackoff, 1000, 2000, maxBackoff };
    // Setup
    SingleInputGate inputGate = mock(SingleInputGate.class);
    BufferProvider bufferProvider = mock(BufferProvider.class);
    when(inputGate.getBufferProvider()).thenReturn(bufferProvider);
    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    LocalInputChannel ch = createLocalInputChannel(inputGate, partitionManager, initialBackoff, maxBackoff);
    when(partitionManager.createSubpartitionView(eq(ch.partitionId), eq(0), any(BufferAvailabilityListener.class))).thenThrow(new PartitionNotFoundException(ch.partitionId));
    Timer timer = mock(Timer.class);
    doAnswer((Answer<Void>) invocation -> {
        ((TimerTask) invocation.getArguments()[0]).run();
        return null;
    }).when(timer).schedule(any(TimerTask.class), anyLong());
    // Initial request
    ch.requestSubpartition();
    verify(partitionManager).createSubpartitionView(eq(ch.partitionId), eq(0), any(BufferAvailabilityListener.class));
    // Request subpartition and verify that the actual requests are delayed.
    for (long expected : expectedDelays) {
        ch.retriggerSubpartitionRequest(timer);
        verify(timer).schedule(any(TimerTask.class), eq(expected));
    }
    // Exception after backoff is greater than the maximum backoff.
    try {
        ch.retriggerSubpartitionRequest(timer);
        ch.getNextBuffer();
        fail("Did not throw expected exception.");
    } catch (Exception expected) {
    }
}

Example 14 with ResultPartitionManager

use of org.apache.flink.runtime.io.network.partition.ResultPartitionManager in project flink by apache.

the class LocalInputChannelTest method testProducerFailedException.

@Test(expected = CancelTaskException.class)
public void testProducerFailedException() throws Exception {
    ResultSubpartitionView view = mock(ResultSubpartitionView.class);
    when(view.isReleased()).thenReturn(true);
    when(view.getFailureCause()).thenReturn(new Exception("Expected test exception"));
    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    when(partitionManager.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class))).thenReturn(view);
    SingleInputGate inputGate = mock(SingleInputGate.class);
    BufferProvider bufferProvider = mock(BufferProvider.class);
    when(inputGate.getBufferProvider()).thenReturn(bufferProvider);
    LocalInputChannel ch = createLocalInputChannel(inputGate, partitionManager);
    ch.requestSubpartition();
    // Should throw an instance of CancelTaskException.
    ch.getNextBuffer();
}

Example 15 with ResultPartitionManager

use of org.apache.flink.runtime.io.network.partition.ResultPartitionManager in project flink by apache.

the class LocalInputChannelTest method testConcurrentReleaseAndRetriggerPartitionRequest.

/**
 * Verifies that concurrent release via the SingleInputGate and re-triggering of a partition
 * request works smoothly.
 *
 * <ul>
 *   <li>SingleInputGate acquires its request lock and tries to release all registered channels.
 *       When releasing a channel, it needs to acquire the channel's shared request-release
 *       lock.
 *   <li>If a LocalInputChannel concurrently retriggers a partition request via a Timer Thread
 *       it acquires the channel's request-release lock and calls the retrigger callback on the
 *       SingleInputGate, which again tries to acquire the gate's request lock.
 * </ul>
 *
 * <p>For certain timings this obviously leads to a deadlock. This test reliably reproduced such
 * a timing (reported in FLINK-5228). This test is pretty much testing the buggy implementation
 * and has not much more general value. If it becomes obsolete at some point (future greatness
 * ;)), feel free to remove it.
 *
 * <p>The fix in the end was to to not acquire the channels lock when releasing it and/or not
 * doing any input gate callbacks while holding the channel's lock. I decided to do both.
 */
@Test
public void testConcurrentReleaseAndRetriggerPartitionRequest() throws Exception {
    final SingleInputGate gate = createSingleInputGate(1);
    ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    when(partitionManager.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class))).thenAnswer((Answer<ResultSubpartitionView>) invocationOnMock -> {
        Thread.sleep(100);
        throw new PartitionNotFoundException(new ResultPartitionID());
    });
    final LocalInputChannel channel = createLocalInputChannel(gate, partitionManager, 1, 1);
    Thread releaser = new Thread(() -> {
        try {
            gate.close();
        } catch (IOException ignored) {
        }
    });
    Thread requester = new Thread(() -> {
        try {
            channel.requestSubpartition();
        } catch (IOException ignored) {
        }
    });
    requester.start();
    releaser.start();
    releaser.join();
    requester.join();
}