Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 16 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class AsynchronousFileIOChannelTest method testClosedButAddRequestAndRegisterListenerRace.

@Test
public void testClosedButAddRequestAndRegisterListenerRace() throws Exception {
    // -- Config ----------------------------------------------------------
    final int numberOfRuns = 1024;
    // -- Setup -----------------------------------------------------------
    final IOManagerAsync ioManager = new IOManagerAsync();
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    final RequestQueue<WriteRequest> requestQueue = new RequestQueue<WriteRequest>();
    @SuppressWarnings("unchecked") final RequestDoneCallback<Buffer> ioChannelCallback = mock(RequestDoneCallback.class);
    final TestNotificationListener listener = new TestNotificationListener();
    // -- The Test --------------------------------------------------------
    try {
        // Repeatedly close the channel and add a request.
        for (int i = 0; i < numberOfRuns; i++) {
            final TestAsyncFileIOChannel ioChannel = new TestAsyncFileIOChannel(ioManager.createChannel(), requestQueue, ioChannelCallback, true);
            final CountDownLatch sync = new CountDownLatch(2);
            final WriteRequest request = mock(WriteRequest.class);
            ioChannel.close();
            // Add request task
            Callable<Void> addRequestTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    try {
                        ioChannel.addRequest(request);
                    } catch (Throwable expected) {
                    } finally {
                        sync.countDown();
                    }
                    return null;
                }
            };
            // Listener
            Callable<Void> registerListenerTask = new Callable<Void>() {

                @Override
                public Void call() throws Exception {
                    try {
                        while (true) {
                            int current = listener.getNumberOfNotifications();
                            if (ioChannel.registerAllRequestsProcessedListener(listener)) {
                                listener.waitForNotification(current);
                            } else if (ioChannel.isClosed()) {
                                break;
                            }
                        }
                    } finally {
                        sync.countDown();
                    }
                    return null;
                }
            };
            executor.submit(addRequestTask);
            executor.submit(registerListenerTask);
            if (!sync.await(2, TimeUnit.MINUTES)) {
                fail("Test failed due to a timeout. This indicates a deadlock due to the way" + "that listeners are registered/notified in the asynchronous file I/O" + "channel.");
            }
        }
    } finally {
        ioManager.shutdown();
        executor.shutdown();
    }
}

Example 17 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class BufferFileWriterFileSegmentReaderTest method testWriteRead.

@Test
public void testWriteRead() throws IOException, InterruptedException {
    int numBuffers = 1024;
    int currentNumber = 0;
    final int minBufferSize = BUFFER_SIZE / 4;
    // Write buffers filled with ascending numbers...
    for (int i = 0; i < numBuffers; i++) {
        final Buffer buffer = createBuffer();
        int size = getNextMultipleOf(getRandomNumberInRange(minBufferSize, BUFFER_SIZE), 4);
        buffer.setSize(size);
        currentNumber = fillBufferWithAscendingNumbers(buffer, currentNumber);
        writer.writeBlock(buffer);
    }
    // Make sure that the writes are finished
    writer.close();
    // Read buffers back in...
    for (int i = 0; i < numBuffers; i++) {
        assertFalse(reader.hasReachedEndOfFile());
        reader.read();
    }
    // Wait for all requests to be finished
    final CountDownLatch sync = new CountDownLatch(1);
    final NotificationListener listener = new NotificationListener() {

        @Override
        public void onNotification() {
            sync.countDown();
        }
    };
    if (reader.registerAllRequestsProcessedListener(listener)) {
        sync.await();
    }
    assertTrue(reader.hasReachedEndOfFile());
    // Verify that the content is the same
    assertEquals("Read less buffers than written.", numBuffers, returnedFileSegments.size());
    currentNumber = 0;
    FileSegment fileSegment;
    ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
    while ((fileSegment = returnedFileSegments.poll()) != null) {
        buffer.position(0);
        buffer.limit(fileSegment.getLength());
        fileSegment.getFileChannel().read(buffer, fileSegment.getPosition());
        currentNumber = verifyBufferFilledWithAscendingNumbers(new Buffer(MemorySegmentFactory.wrap(buffer.array()), BUFFER_RECYCLER), currentNumber, fileSegment.getLength());
    }
    reader.close();
}

Example 18 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class BufferFileWriterReaderTest method testWriteSkipRead.

@Test
public void testWriteSkipRead() throws IOException {
    int numBuffers = 1024;
    int currentNumber = 0;
    // Write buffers filled with ascending numbers...
    for (int i = 0; i < numBuffers; i++) {
        final Buffer buffer = createBuffer();
        currentNumber = fillBufferWithAscendingNumbers(buffer, currentNumber);
        writer.writeBlock(buffer);
    }
    // Make sure that the writes are finished
    writer.close();
    final int toSkip = 32;
    // Skip first buffers...
    reader.seekToPosition((8 + BUFFER_SIZE) * toSkip);
    numBuffers -= toSkip;
    // Read buffers back in...
    for (int i = 0; i < numBuffers; i++) {
        assertFalse(reader.hasReachedEndOfFile());
        reader.readInto(createBuffer());
    }
    reader.close();
    assertTrue(reader.hasReachedEndOfFile());
    // Verify that the content is the same
    assertEquals("Read less buffers than written.", numBuffers, returnedBuffers.size());
    // Start number after skipped buffers...
    currentNumber = (BUFFER_SIZE / 4) * toSkip;
    Buffer buffer;
    while ((buffer = returnedBuffers.poll()) != null) {
        currentNumber = verifyBufferFilledWithAscendingNumbers(buffer, currentNumber);
    }
}

Example 19 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class ResultPartitionTest method testAddOnFinishedPartition.

/**
	 * Tests {@link ResultPartition#add} on a partition which has already finished.
	 *
	 * @param pipelined the result partition type to set up
	 */
protected void testAddOnFinishedPartition(final ResultPartitionType pipelined) throws Exception {
    Buffer buffer = TestBufferFactory.createBuffer();
    ResultPartitionConsumableNotifier notifier = mock(ResultPartitionConsumableNotifier.class);
    try {
        ResultPartition partition = createPartition(notifier, pipelined, true);
        partition.finish();
        reset(notifier);
        // partition.add() should fail
        partition.add(buffer, 0);
        Assert.fail("exception expected");
    } catch (IllegalStateException e) {
    // expected => ignored
    } finally {
        if (!buffer.isRecycled()) {
            Assert.fail("buffer not recycled");
            buffer.recycle();
        }
        // should not have notified either
        verify(notifier, never()).notifyPartitionConsumable(any(JobID.class), any(ResultPartitionID.class), any(TaskActions.class));
    }
}

Example 20 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SubpartitionTestBase method verifyViewReleasedAfterParentRelease.

private void verifyViewReleasedAfterParentRelease(ResultSubpartition partition) throws Exception {
    // Add a buffer
    Buffer buffer = TestBufferFactory.createBuffer();
    partition.add(buffer);
    partition.finish();
    TestInfiniteBufferProvider buffers = new TestInfiniteBufferProvider();
    // Create the view
    BufferAvailabilityListener listener = mock(BufferAvailabilityListener.class);
    ResultSubpartitionView view = partition.createReadView(buffers, listener);
    // The added buffer and end-of-partition event
    assertNotNull(view.getNextBuffer());
    assertNotNull(view.getNextBuffer());
    // Release the parent
    assertFalse(view.isReleased());
    partition.release();
    // Verify that parent release is reflected at partition view
    assertTrue(view.isReleased());
}