Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 41 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class RecordWriterTest method testClearBuffersAfterInterruptDuringBlockingBufferRequest.

// ---------------------------------------------------------------------------------------------
// Resource release tests
// ---------------------------------------------------------------------------------------------
/**
	 * Tests a fix for FLINK-2089.
	 *
	 * @see <a href="https://issues.apache.org/jira/browse/FLINK-2089">FLINK-2089</a>
	 */
@Test
public void testClearBuffersAfterInterruptDuringBlockingBufferRequest() throws Exception {
    ExecutorService executor = null;
    try {
        executor = Executors.newSingleThreadExecutor();
        final CountDownLatch sync = new CountDownLatch(2);
        final Buffer buffer = spy(TestBufferFactory.createBuffer(4));
        // Return buffer for first request, but block for all following requests.
        Answer<Buffer> request = new Answer<Buffer>() {

            @Override
            public Buffer answer(InvocationOnMock invocation) throws Throwable {
                sync.countDown();
                if (sync.getCount() == 1) {
                    return buffer;
                }
                final Object o = new Object();
                synchronized (o) {
                    while (true) {
                        o.wait();
                    }
                }
            }
        };
        BufferProvider bufferProvider = mock(BufferProvider.class);
        when(bufferProvider.requestBufferBlocking()).thenAnswer(request);
        ResultPartitionWriter partitionWriter = createResultPartitionWriter(bufferProvider);
        final RecordWriter<IntValue> recordWriter = new RecordWriter<IntValue>(partitionWriter);
        Future<?> result = executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                IntValue val = new IntValue(0);
                try {
                    recordWriter.emit(val);
                    recordWriter.flush();
                    recordWriter.emit(val);
                } catch (InterruptedException e) {
                    recordWriter.clearBuffers();
                }
                return null;
            }
        });
        sync.await();
        // Interrupt the Thread.
        //
        // The second emit call requests a new buffer and blocks the thread.
        // When interrupting the thread at this point, clearing the buffers
        // should not recycle any buffer.
        result.cancel(true);
        recordWriter.clearBuffers();
        // Verify that buffer have been requested, but only one has been written out.
        verify(bufferProvider, times(2)).requestBufferBlocking();
        verify(partitionWriter, times(1)).writeBuffer(any(Buffer.class), anyInt());
        // Verify that the written out buffer has only been recycled once
        // (by the partition writer).
        assertTrue("Buffer not recycled.", buffer.isRecycled());
        verify(buffer, times(1)).recycle();
    } finally {
        if (executor != null) {
            executor.shutdown();
        }
    }
}

Example 42 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class ResultPartitionWriterTest method testWriteBufferToAllChannelsReferenceCounting.

// ---------------------------------------------------------------------------------------------
// Resource release tests
// ---------------------------------------------------------------------------------------------
/**
	 * Tests that event buffers are properly recycled when broadcasting events
	 * to multiple channels.
	 *
	 * @throws Exception
	 */
@Test
public void testWriteBufferToAllChannelsReferenceCounting() throws Exception {
    Buffer buffer = EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE);
    ResultPartition partition = new ResultPartition("TestTask", mock(TaskActions.class), new JobID(), new ResultPartitionID(), ResultPartitionType.PIPELINED, 2, 2, mock(ResultPartitionManager.class), mock(ResultPartitionConsumableNotifier.class), mock(IOManager.class), false);
    ResultPartitionWriter partitionWriter = new ResultPartitionWriter(partition);
    partitionWriter.writeBufferToAllChannels(buffer);
    // Verify added to all queues, i.e. two buffers in total
    assertEquals(2, partition.getTotalNumberOfBuffers());
    // release the buffers in the partition
    partition.release();
    assertTrue(buffer.isRecycled());
}

Example 43 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class NettyMessageSerializationTest method testEncodeDecode.

@Test
public void testEncodeDecode() {
    {
        Buffer buffer = spy(new Buffer(MemorySegmentFactory.allocateUnpooledSegment(1024), mock(BufferRecycler.class)));
        ByteBuffer nioBuffer = buffer.getNioBuffer();
        for (int i = 0; i < 1024; i += 4) {
            nioBuffer.putInt(i);
        }
        NettyMessage.BufferResponse expected = new NettyMessage.BufferResponse(buffer, random.nextInt(), new InputChannelID());
        NettyMessage.BufferResponse actual = encodeAndDecode(expected);
        // Verify recycle has been called on buffer instance
        verify(buffer, times(1)).recycle();
        final ByteBuf retainedSlice = actual.getNettyBuffer();
        // Ensure not recycled and same size as original buffer
        assertEquals(1, retainedSlice.refCnt());
        assertEquals(1024, retainedSlice.readableBytes());
        nioBuffer = retainedSlice.nioBuffer();
        for (int i = 0; i < 1024; i += 4) {
            assertEquals(i, nioBuffer.getInt());
        }
        // Release the retained slice
        actual.releaseBuffer();
        assertEquals(0, retainedSlice.refCnt());
        assertEquals(expected.sequenceNumber, actual.sequenceNumber);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        {
            IllegalStateException expectedError = new IllegalStateException();
            InputChannelID receiverId = new InputChannelID();
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError, receiverId);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertEquals(receiverId, actual.receiverId);
        }
        {
            IllegalStateException expectedError = new IllegalStateException("Illegal illegal illegal");
            InputChannelID receiverId = new InputChannelID();
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError, receiverId);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertEquals(receiverId, actual.receiverId);
        }
        {
            IllegalStateException expectedError = new IllegalStateException("Illegal illegal illegal");
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertNull(actual.receiverId);
            assertTrue(actual.isFatalError());
        }
    }
    {
        NettyMessage.PartitionRequest expected = new NettyMessage.PartitionRequest(new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID()), random.nextInt(), new InputChannelID());
        NettyMessage.PartitionRequest actual = encodeAndDecode(expected);
        assertEquals(expected.partitionId, actual.partitionId);
        assertEquals(expected.queueIndex, actual.queueIndex);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.TaskEventRequest expected = new NettyMessage.TaskEventRequest(new IntegerTaskEvent(random.nextInt()), new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID()), new InputChannelID());
        NettyMessage.TaskEventRequest actual = encodeAndDecode(expected);
        assertEquals(expected.event, actual.event);
        assertEquals(expected.partitionId, actual.partitionId);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.CancelPartitionRequest expected = new NettyMessage.CancelPartitionRequest(new InputChannelID());
        NettyMessage.CancelPartitionRequest actual = encodeAndDecode(expected);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.CloseRequest expected = new NettyMessage.CloseRequest();
        NettyMessage.CloseRequest actual = encodeAndDecode(expected);
        assertEquals(expected.getClass(), actual.getClass());
    }
}

Example 44 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class PartitionRequestClientHandlerTest method testAutoReadAfterUnsuccessfulStagedMessage.

/**
	 * Tests that an unsuccessful message decode call for a staged message
	 * does not leave the channel with auto read set to false.
	 */
@Test
@SuppressWarnings("unchecked")
public void testAutoReadAfterUnsuccessfulStagedMessage() throws Exception {
    PartitionRequestClientHandler handler = new PartitionRequestClientHandler();
    EmbeddedChannel channel = new EmbeddedChannel(handler);
    final AtomicReference<EventListener<Buffer>> listener = new AtomicReference<>();
    BufferProvider bufferProvider = mock(BufferProvider.class);
    when(bufferProvider.addListener(any(EventListener.class))).thenAnswer(new Answer<Boolean>() {

        @Override
        @SuppressWarnings("unchecked")
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            listener.set((EventListener<Buffer>) invocation.getArguments()[0]);
            return true;
        }
    });
    when(bufferProvider.requestBuffer()).thenReturn(null);
    InputChannelID channelId = new InputChannelID(0, 0);
    RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(channelId);
    // The 3rd staged msg has a null buffer provider
    when(inputChannel.getBufferProvider()).thenReturn(bufferProvider, bufferProvider, null);
    handler.addInputChannel(inputChannel);
    BufferResponse msg = createBufferResponse(createBuffer(true), 0, channelId);
    // Write 1st buffer msg. No buffer is available, therefore the buffer
    // should be staged and auto read should be set to false.
    assertTrue(channel.config().isAutoRead());
    channel.writeInbound(msg);
    // No buffer available, auto read false
    assertFalse(channel.config().isAutoRead());
    // Write more buffers... all staged.
    msg = createBufferResponse(createBuffer(true), 1, channelId);
    channel.writeInbound(msg);
    msg = createBufferResponse(createBuffer(true), 2, channelId);
    channel.writeInbound(msg);
    // Notify about buffer => handle 1st msg
    Buffer availableBuffer = createBuffer(false);
    listener.get().onEvent(availableBuffer);
    // Start processing of staged buffers (in run pending tasks). Make
    // sure that the buffer provider acts like it's destroyed.
    when(bufferProvider.addListener(any(EventListener.class))).thenReturn(false);
    when(bufferProvider.isDestroyed()).thenReturn(true);
    // Execute all tasks that are scheduled in the event loop. Further
    // eventLoop().execute() calls are directly executed, if they are
    // called in the scope of this call.
    channel.runPendingTasks();
    assertTrue(channel.config().isAutoRead());
}

Example 45 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class AsynchronousFileIOChannelTest method testAllRequestsProcessedListenerNotification.

@Test
public void testAllRequestsProcessedListenerNotification() throws Exception {
    // -- Config ----------------------------------------------------------
    final int numberOfRuns = 10;
    final int numberOfRequests = 100;
    // -- Setup -----------------------------------------------------------
    final IOManagerAsync ioManager = new IOManagerAsync();
    final ExecutorService executor = Executors.newFixedThreadPool(3);
    final Random random = new Random();
    final RequestQueue<WriteRequest> requestQueue = new RequestQueue<WriteRequest>();
    final RequestDoneCallback<Buffer> ioChannelCallback = mock(RequestDoneCallback.class);
    final TestNotificationListener listener = new TestNotificationListener();
    // -- The Test --------------------------------------------------------
    try {
        for (int run = 0; run < numberOfRuns; run++) {
            final TestAsyncFileIOChannel ioChannel = new TestAsyncFileIOChannel(ioManager.createChannel(), requestQueue, ioChannelCallback, true);
            final CountDownLatch sync = new CountDownLatch(3);
            // The mock requests
            final Buffer buffer = mock(Buffer.class);
            final WriteRequest request = mock(WriteRequest.class);
            // Add requests task
            Callable<Void> addRequestsTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    for (int i = 0; i < numberOfRuns; i++) {
                        LOG.debug("Starting run {}.", i + 1);
                        for (int j = 0; j < numberOfRequests; j++) {
                            ioChannel.addRequest(request);
                        }
                        LOG.debug("Added all ({}) requests of run {}.", numberOfRequests, i + 1);
                        int sleep = random.nextInt(10);
                        LOG.debug("Sleeping for {} ms before next run.", sleep);
                        Thread.sleep(sleep);
                    }
                    LOG.debug("Done. Closing channel.");
                    ioChannel.close();
                    sync.countDown();
                    return null;
                }
            };
            // Process requests task
            Callable<Void> processRequestsTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    int total = numberOfRequests * numberOfRuns;
                    for (int i = 0; i < total; i++) {
                        requestQueue.take();
                        ioChannel.handleProcessedBuffer(buffer, null);
                    }
                    LOG.debug("Processed all ({}) requests.", numberOfRequests);
                    sync.countDown();
                    return null;
                }
            };
            // Listener
            Callable<Void> registerListenerTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    while (true) {
                        int current = listener.getNumberOfNotifications();
                        if (ioChannel.registerAllRequestsProcessedListener(listener)) {
                            listener.waitForNotification(current);
                        } else if (ioChannel.isClosed()) {
                            break;
                        }
                    }
                    LOG.debug("Stopping listener. Channel closed.");
                    sync.countDown();
                    return null;
                }
            };
            // Run tasks in random order
            final List<Callable<?>> tasks = new LinkedList<Callable<?>>();
            tasks.add(addRequestsTask);
            tasks.add(processRequestsTask);
            tasks.add(registerListenerTask);
            Collections.shuffle(tasks);
            for (Callable<?> task : tasks) {
                executor.submit(task);
            }
            if (!sync.await(2, TimeUnit.MINUTES)) {
                fail("Test failed due to a timeout. This indicates a deadlock due to the way" + "that listeners are registered/notified in the asynchronous file I/O" + "channel.");
            }
            listener.reset();
        }
    } finally {
        ioManager.shutdown();
        executor.shutdown();
    }
}