Examples with ResultPartitionManager org.apache.flink.runtime.io.network.partition.ResultPartitionManager used on opensource projects

Example 21 with ResultPartitionManager

use of org.apache.flink.runtime.io.network.partition.ResultPartitionManager in project flink by apache.

the class TaskExecutorPartitionLifecycleTest method testBlockingLocalPartitionReleaseDoesNotBlockTaskExecutor.

@Test
public void testBlockingLocalPartitionReleaseDoesNotBlockTaskExecutor() throws Exception {
    BlockerSync sync = new BlockerSync();
    ResultPartitionManager blockingResultPartitionManager = new ResultPartitionManager() {

        @Override
        public void releasePartition(ResultPartitionID partitionId, Throwable cause) {
            sync.blockNonInterruptible();
            super.releasePartition(partitionId, cause);
        }
    };
    NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().setResultPartitionManager(blockingResultPartitionManager).setIoExecutor(TEST_EXECUTOR_SERVICE_RESOURCE.getExecutor()).build();
    final CompletableFuture<ResultPartitionID> startTrackingFuture = new CompletableFuture<>();
    final TaskExecutorPartitionTracker partitionTracker = new TaskExecutorPartitionTrackerImpl(shuffleEnvironment) {

        @Override
        public void startTrackingPartition(JobID producingJobId, TaskExecutorPartitionInfo partitionInfo) {
            super.startTrackingPartition(producingJobId, partitionInfo);
            startTrackingFuture.complete(partitionInfo.getResultPartitionId());
        }
    };
    try {
        internalTestPartitionRelease(partitionTracker, shuffleEnvironment, startTrackingFuture, (jobId, resultPartitionDeploymentDescriptor, taskExecutor, taskExecutorGateway) -> {
            final IntermediateDataSetID dataSetId = resultPartitionDeploymentDescriptor.getResultId();
            taskExecutorGateway.releaseClusterPartitions(Collections.singleton(dataSetId), timeout);
            // execute some operation to check whether the TaskExecutor is blocked
            taskExecutorGateway.canBeReleased().get(5, TimeUnit.SECONDS);
        });
    } finally {
        sync.releaseBlocker();
    }
}

Example 22 with ResultPartitionManager

use of org.apache.flink.runtime.io.network.partition.ResultPartitionManager in project flink by apache.

the class CancelPartitionRequestTest method testCancelPartitionRequest.

/**
 * Verifies that requests for non-existing (failed/cancelled) input channels are properly
 * cancelled. The receiver receives data, but there is no input channel to receive the data.
 * This should cancel the request.
 */
@Test
public void testCancelPartitionRequest() throws Exception {
    NettyServerAndClient serverAndClient = null;
    try {
        TestPooledBufferProvider outboundBuffers = new TestPooledBufferProvider(16);
        ResultPartitionManager partitions = mock(ResultPartitionManager.class);
        ResultPartitionID pid = new ResultPartitionID();
        CountDownLatch sync = new CountDownLatch(1);
        final ResultSubpartitionView view = spy(new InfiniteSubpartitionView(outboundBuffers, sync));
        // Return infinite subpartition
        when(partitions.createSubpartitionView(eq(pid), eq(0), any(BufferAvailabilityListener.class))).thenAnswer(new Answer<ResultSubpartitionView>() {

            @Override
            public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
                BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[2];
                listener.notifyDataAvailable();
                return view;
            }
        });
        NettyProtocol protocol = new NettyProtocol(partitions, mock(TaskEventDispatcher.class));
        serverAndClient = initServerAndClient(protocol);
        Channel ch = connect(serverAndClient);
        // Request for non-existing input channel => results in cancel request
        ch.writeAndFlush(new PartitionRequest(pid, 0, new InputChannelID(), Integer.MAX_VALUE)).await();
        // Wait for the notification
        if (!sync.await(TestingUtils.TESTING_DURATION.toMillis(), TimeUnit.MILLISECONDS)) {
            fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION.toMillis() + " ms to be notified about cancelled partition.");
        }
        verify(view, times(1)).releaseAllResources();
    } finally {
        shutdown(serverAndClient);
    }
}

Example 23 with ResultPartitionManager

use of org.apache.flink.runtime.io.network.partition.ResultPartitionManager in project flink by apache.

the class NettyShuffleEnvironmentTest method testSlowIODoesNotBlockRelease.

@Test
public void testSlowIODoesNotBlockRelease() throws Exception {
    BlockerSync sync = new BlockerSync();
    ResultPartitionManager blockingResultPartitionManager = new ResultPartitionManager() {

        @Override
        public void releasePartition(ResultPartitionID partitionId, Throwable cause) {
            sync.blockNonInterruptible();
            super.releasePartition(partitionId, cause);
        }
    };
    NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().setResultPartitionManager(blockingResultPartitionManager).setIoExecutor(Executors.newFixedThreadPool(1)).build();
    shuffleEnvironment.releasePartitionsLocally(Collections.singleton(new ResultPartitionID()));
    sync.awaitBlocker();
    sync.releaseBlocker();
}

Example 24 with ResultPartitionManager

use of org.apache.flink.runtime.io.network.partition.ResultPartitionManager in project flink by apache.

the class PartitionRequestQueueTest method testCancelPartitionRequest.

private void testCancelPartitionRequest(boolean isAvailableView) throws Exception {
    // setup
    final ResultPartitionManager partitionManager = new ResultPartitionManager();
    final ResultPartition partition = createFinishedPartitionWithFilledData(partitionManager);
    final InputChannelID receiverId = new InputChannelID();
    final PartitionRequestQueue queue = new PartitionRequestQueue();
    final CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
    final EmbeddedChannel channel = new EmbeddedChannel(queue);
    reader.requestSubpartitionView(partitionManager, partition.getPartitionId(), 0);
    // add this reader into allReaders queue
    queue.notifyReaderCreated(reader);
    // block the channel so that we see an intermediate state in the test
    blockChannel(channel);
    // add credit to make this reader available for adding into availableReaders queue
    if (isAvailableView) {
        queue.addCreditOrResumeConsumption(receiverId, viewReader -> viewReader.addCredit(1));
        assertTrue(queue.getAvailableReaders().contains(reader));
    }
    // cancel this subpartition view
    queue.cancel(receiverId);
    channel.runPendingTasks();
    assertFalse(queue.getAvailableReaders().contains(reader));
    // the reader view should be released (the partition is not, though, blocking partitions
    // support multiple successive readers for recovery and caching)
    assertTrue(reader.isReleased());
    // cleanup
    partition.release();
    channel.close();
}

Example 25 with ResultPartitionManager

use of org.apache.flink.runtime.io.network.partition.ResultPartitionManager in project flink by apache.

the class ServerTransportErrorHandlingTest method testRemoteClose.

/**
 * Verifies remote closes trigger the release of all resources.
 */
@Test
public void testRemoteClose() throws Exception {
    final TestPooledBufferProvider outboundBuffers = new TestPooledBufferProvider(16);
    final CountDownLatch sync = new CountDownLatch(1);
    final ResultPartitionManager partitionManager = mock(ResultPartitionManager.class);
    when(partitionManager.createSubpartitionView(any(ResultPartitionID.class), anyInt(), any(BufferAvailabilityListener.class))).thenAnswer(new Answer<ResultSubpartitionView>() {

        @Override
        public ResultSubpartitionView answer(InvocationOnMock invocationOnMock) throws Throwable {
            BufferAvailabilityListener listener = (BufferAvailabilityListener) invocationOnMock.getArguments()[2];
            listener.notifyDataAvailable();
            return new CancelPartitionRequestTest.InfiniteSubpartitionView(outboundBuffers, sync);
        }
    });
    NettyProtocol protocol = new NettyProtocol(partitionManager, mock(TaskEventDispatcher.class)) {

        @Override
        public ChannelHandler[] getClientChannelHandlers() {
            return new ChannelHandler[] { new NettyMessage.NettyMessageEncoder(), // Close on read
            new ChannelInboundHandlerAdapter() {

                @Override
                public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
                    ctx.channel().close();
                }
            } };
        }
    };
    NettyTestUtil.NettyServerAndClient serverAndClient = null;
    try {
        for (int retry = 0; retry < NETTY_INIT_MAX_RETRY_TIMES; retry++) {
            try {
                serverAndClient = initServerAndClient(protocol, createConfig());
                break;
            } catch (Exception e) {
                if (retry >= NETTY_INIT_MAX_RETRY_TIMES - 1) {
                    throw new RuntimeException("Failed to initialize netty server and client, retried " + retry + " times.", e);
                }
                if (e instanceof BindException || ExceptionUtils.findThrowableWithMessage(e, "Address already in use").isPresent()) {
                    continue;
                }
                throw e;
            }
        }
        Channel ch = connect(serverAndClient);
        // Write something to trigger close by server
        ch.writeAndFlush(new NettyMessage.PartitionRequest(new ResultPartitionID(), 0, new InputChannelID(), Integer.MAX_VALUE));
        // Wait for the notification
        if (!sync.await(TestingUtils.TESTING_DURATION.toMillis(), TimeUnit.MILLISECONDS)) {
            fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION.toMillis() + " ms to be notified about released partition.");
        }
    } finally {
        shutdown(serverAndClient);
    }
}