Examples with ResultPartitionID org.apache.flink.runtime.io.network.partition.ResultPartitionID used on opensource projects

Example 31 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class PartitionRequestQueueTest method testProducerFailedException.

@Test
public void testProducerFailedException() throws Exception {
    PartitionRequestQueue queue = new PartitionRequestQueue();
    ResultPartitionProvider partitionProvider = mock(ResultPartitionProvider.class);
    ResultPartitionID rpid = new ResultPartitionID();
    BufferProvider bufferProvider = mock(BufferProvider.class);
    ResultSubpartitionView view = mock(ResultSubpartitionView.class);
    when(view.isReleased()).thenReturn(true);
    when(view.getFailureCause()).thenReturn(new RuntimeException("Expected test exception"));
    when(partitionProvider.createSubpartitionView(eq(rpid), eq(0), eq(bufferProvider), any(BufferAvailabilityListener.class))).thenReturn(view);
    EmbeddedChannel ch = new EmbeddedChannel(queue);
    SequenceNumberingViewReader seqView = new SequenceNumberingViewReader(new InputChannelID(), queue);
    seqView.requestSubpartitionView(partitionProvider, rpid, 0, bufferProvider);
    // Enqueue the erroneous view
    queue.notifyReaderNonEmpty(seqView);
    ch.runPendingTasks();
    // Read the enqueued msg
    Object msg = ch.readOutbound();
    assertEquals(msg.getClass(), NettyMessage.ErrorResponse.class);
    NettyMessage.ErrorResponse err = (NettyMessage.ErrorResponse) msg;
    assertTrue(err.cause instanceof CancelTaskException);
}

Example 32 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class ServerTransportErrorHandlingTest method testRemoteClose.

/**
	 * Verifies remote closes trigger the release of all resources.
	 */
@Test
public void testRemoteClose() throws Exception {
    final TestPooledBufferProvider outboundBuffers = new TestPooledBufferProvider(16);
    final CountDownLatch sync = new CountDownLatch(1);
    final ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    when(partitionManager.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferProvider.class), any(BufferAvailabilityListener.class))).thenAnswer(new Answer<ResultSubpartitionView>() {

        @Override
        public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
            BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[3];
            listener.notifyBuffersAvailable(Long.MAX_VALUE);
            return new CancelPartitionRequestTest.InfiniteSubpartitionView(outboundBuffers, sync);
        }
    });
    NettyProtocol protocol = new NettyProtocol() {

        @Override
        public ChannelHandler[] getServerChannelHandlers() {
            return new PartitionRequestProtocol(partitionManager, mock(TaskEventDispatcher.class), mock(NetworkBufferPool.class)).getServerChannelHandlers();
        }

        @Override
        public ChannelHandler[] getClientChannelHandlers() {
            return new ChannelHandler[] { new NettyMessage.NettyMessageEncoder(), // Close on read
            new ChannelInboundHandlerAdapter() {

                @Override
                public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
                    ctx.channel().close();
                }
            } };
        }
    };
    NettyTestUtil.NettyServerAndClient serverAndClient = null;
    try {
        serverAndClient = initServerAndClient(protocol, createConfig());
        Channel ch = connect(serverAndClient);
        // Write something to trigger close by server
        ch.writeAndFlush(new NettyMessage.PartitionRequest(new ResultPartitionID(), 0, new InputChannelID()));
        // Wait for the notification
        if (!sync.await(TestingUtils.TESTING_DURATION().toMillis(), TimeUnit.MILLISECONDS)) {
            fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION().toMillis() + " ms to be notified about released partition.");
        }
    } finally {
        shutdown(serverAndClient);
    }
}

Example 33 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class RemoteInputChannelTest method testOnFailedPartitionRequest.

@Test
public void testOnFailedPartitionRequest() throws Exception {
    final ConnectionManager connectionManager = mock(ConnectionManager.class);
    when(connectionManager.createPartitionRequestClient(any(ConnectionID.class))).thenReturn(mock(PartitionRequestClient.class));
    final ResultPartitionID partitionId = new ResultPartitionID();
    final SingleInputGate inputGate = mock(SingleInputGate.class);
    final RemoteInputChannel ch = new RemoteInputChannel(inputGate, 0, partitionId, mock(ConnectionID.class), connectionManager, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    ch.onFailedPartitionRequest();
    verify(inputGate).triggerPartitionStateCheck(eq(partitionId));
}

Example 34 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class SingleInputGateTest method testReleaseWhilePollingChannel.

/**
	 * Tests that the release of the input gate is noticed while polling the
	 * channels for available data.
	 */
@Test
public void testReleaseWhilePollingChannel() throws Exception {
    final AtomicReference<Exception> asyncException = new AtomicReference<>();
    // Setup the input gate with a single channel that does nothing
    final SingleInputGate inputGate = new SingleInputGate("InputGate", new JobID(), new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, 1, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    InputChannel unknown = new UnknownInputChannel(inputGate, 0, new ResultPartitionID(), new ResultPartitionManager(), new TaskEventDispatcher(), new LocalConnectionManager(), 0, 0, new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    inputGate.setInputChannel(unknown.partitionId.getPartitionId(), unknown);
    // Start the consumer in a separate Thread
    Thread asyncConsumer = new Thread() {

        @Override
        public void run() {
            try {
                inputGate.getNextBufferOrEvent();
            } catch (Exception e) {
                asyncException.set(e);
            }
        }
    };
    asyncConsumer.start();
    // Wait for blocking queue poll call and release input gate
    boolean success = false;
    for (int i = 0; i < 50; i++) {
        if (asyncConsumer.isAlive()) {
            success = asyncConsumer.getState() == Thread.State.WAITING;
        }
        if (success) {
            break;
        } else {
            // Retry
            Thread.sleep(100);
        }
    }
    // Verify that async consumer is in blocking request
    assertTrue("Did not trigger blocking buffer request.", success);
    // Release the input gate
    inputGate.releaseAllResources();
    // Wait for Thread to finish and verify expected Exceptions. If the
    // input gate status is not properly checked during requests, this
    // call will never return.
    asyncConsumer.join();
    assertNotNull(asyncException.get());
    assertEquals(IllegalStateException.class, asyncException.get().getClass());
}

Example 35 with ResultPartitionID

use of org.apache.flink.runtime.io.network.partition.ResultPartitionID in project flink by apache.

the class SingleInputGateTest method testRequestBackoffConfiguration.

/**
	 * Tests request back off configuration is correctly forwarded to the channels.
	 */
@Test
public void testRequestBackoffConfiguration() throws Exception {
    ResultPartitionID[] partitionIds = new ResultPartitionID[] { new ResultPartitionID(), new ResultPartitionID(), new ResultPartitionID() };
    InputChannelDeploymentDescriptor[] channelDescs = new InputChannelDeploymentDescriptor[] { // Local
    new InputChannelDeploymentDescriptor(partitionIds[0], ResultPartitionLocation.createLocal()), // Remote
    new InputChannelDeploymentDescriptor(partitionIds[1], ResultPartitionLocation.createRemote(new ConnectionID(new InetSocketAddress("localhost", 5000), 0))), // Unknown
    new InputChannelDeploymentDescriptor(partitionIds[2], ResultPartitionLocation.createUnknown()) };
    InputGateDeploymentDescriptor gateDesc = new InputGateDeploymentDescriptor(new IntermediateDataSetID(), ResultPartitionType.PIPELINED, 0, channelDescs);
    int initialBackoff = 137;
    int maxBackoff = 1001;
    NetworkEnvironment netEnv = mock(NetworkEnvironment.class);
    when(netEnv.getResultPartitionManager()).thenReturn(new ResultPartitionManager());
    when(netEnv.getTaskEventDispatcher()).thenReturn(new TaskEventDispatcher());
    when(netEnv.getPartitionRequestInitialBackoff()).thenReturn(initialBackoff);
    when(netEnv.getPartitionRequestMaxBackoff()).thenReturn(maxBackoff);
    when(netEnv.getConnectionManager()).thenReturn(new LocalConnectionManager());
    SingleInputGate gate = SingleInputGate.create("TestTask", new JobID(), new ExecutionAttemptID(), gateDesc, netEnv, mock(TaskActions.class), new UnregisteredTaskMetricsGroup.DummyTaskIOMetricGroup());
    assertEquals(gateDesc.getConsumedPartitionType(), gate.getConsumedPartitionType());
    Map<IntermediateResultPartitionID, InputChannel> channelMap = gate.getInputChannels();
    assertEquals(3, channelMap.size());
    InputChannel localChannel = channelMap.get(partitionIds[0].getPartitionId());
    assertEquals(LocalInputChannel.class, localChannel.getClass());
    InputChannel remoteChannel = channelMap.get(partitionIds[1].getPartitionId());
    assertEquals(RemoteInputChannel.class, remoteChannel.getClass());
    InputChannel unknownChannel = channelMap.get(partitionIds[2].getPartitionId());
    assertEquals(UnknownInputChannel.class, unknownChannel.getClass());
    InputChannel[] channels = new InputChannel[] { localChannel, remoteChannel, unknownChannel };
    for (InputChannel ch : channels) {
        assertEquals(0, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(initialBackoff, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(initialBackoff * 2, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(initialBackoff * 2 * 2, ch.getCurrentBackoff());
        assertTrue(ch.increaseBackoff());
        assertEquals(maxBackoff, ch.getCurrentBackoff());
        assertFalse(ch.increaseBackoff());
    }
}