Examples with BufferBuilder org.apache.flink.runtime.io.network.buffer.BufferBuilder used on opensource projects

Example 46 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class RecordWriterTest method testIsAvailableOrNot.

/**
 * Tests that the RecordWriter is available iif the respective LocalBufferPool has at-least one
 * available buffer.
 */
@Test
public void testIsAvailableOrNot() throws Exception {
    // setup
    final NetworkBufferPool globalPool = new NetworkBufferPool(10, 128);
    final BufferPool localPool = globalPool.createBufferPool(1, 1, 1, Integer.MAX_VALUE);
    final ResultPartitionWriter resultPartition = new ResultPartitionBuilder().setBufferPoolFactory(() -> localPool).build();
    resultPartition.setup();
    final RecordWriter<?> recordWriter = createRecordWriter(resultPartition);
    try {
        // record writer is available because of initial available global pool
        assertTrue(recordWriter.getAvailableFuture().isDone());
        // request one buffer from the local pool to make it unavailable afterwards
        try (BufferBuilder bufferBuilder = localPool.requestBufferBuilder(0)) {
            assertNotNull(bufferBuilder);
            assertFalse(recordWriter.getAvailableFuture().isDone());
            // recycle the buffer to make the local pool available again
            final Buffer buffer = BufferBuilderTestUtils.buildSingleBuffer(bufferBuilder);
            buffer.recycleBuffer();
        }
        assertTrue(recordWriter.getAvailableFuture().isDone());
        assertEquals(recordWriter.AVAILABLE, recordWriter.getAvailableFuture());
    } finally {
        localPool.lazyDestroy();
        globalPool.destroy();
    }
}

Example 47 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class RemoteInputChannelTest method testConcurrentRecycleAndRelease2.

/**
 * Tests to verify that there is no race condition with two things running in parallel:
 * recycling exclusive buffers and recycling external buffers to the buffer pool while the
 * recycling of the exclusive buffer triggers recycling a floating buffer (FLINK-9676).
 */
@Test
public void testConcurrentRecycleAndRelease2() throws Exception {
    // Setup
    final int retries = 1_000;
    final int numExclusiveBuffers = 2;
    final int numFloatingBuffers = 2;
    final int numTotalBuffers = numExclusiveBuffers + numFloatingBuffers;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(numTotalBuffers, 32);
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
    inputGate.setInputChannels(inputChannel);
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        final Callable<Void> bufferPoolInteractionsTask = () -> {
            for (int i = 0; i < retries; ++i) {
                try (BufferBuilder bufferBuilder = bufferPool.requestBufferBuilderBlocking()) {
                    Buffer buffer = buildSingleBuffer(bufferBuilder);
                    buffer.recycleBuffer();
                }
            }
            return null;
        };
        final Callable<Void> channelInteractionsTask = () -> {
            ArrayList<Buffer> exclusiveBuffers = new ArrayList<>(numExclusiveBuffers);
            ArrayList<Buffer> floatingBuffers = new ArrayList<>(numExclusiveBuffers);
            try {
                for (int i = 0; i < retries; ++i) {
                    // floating buffers as soon as we take exclusive ones
                    for (int j = 0; j < numTotalBuffers; ++j) {
                        Buffer buffer = inputChannel.requestBuffer();
                        if (buffer == null) {
                            break;
                        } else {
                            // noinspection ObjectEquality
                            if (buffer.getRecycler() == inputChannel.getBufferManager()) {
                                exclusiveBuffers.add(buffer);
                            } else {
                                floatingBuffers.add(buffer);
                            }
                        }
                    }
                    // recycle excess floating buffers (will go back into the channel)
                    floatingBuffers.forEach(Buffer::recycleBuffer);
                    floatingBuffers.clear();
                    assertEquals(numExclusiveBuffers, exclusiveBuffers.size());
                    inputChannel.onSenderBacklog(// trigger subscription to buffer pool
                    0);
                    // note: if we got a floating buffer by increasing the backlog, it
                    // will be released again when recycling the exclusive buffer, if
                    // not, we should release it once we get it
                    exclusiveBuffers.forEach(Buffer::recycleBuffer);
                    exclusiveBuffers.clear();
                }
            } finally {
                inputChannel.releaseAllResources();
            }
            return null;
        };
        // Submit tasks and wait to finish
        submitTasksAndWaitForResults(executor, new Callable[] { bufferPoolInteractionsTask, channelInteractionsTask });
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, executor, null, thrown, inputChannel);
    }
}

Example 48 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class StreamTaskNetworkInputTest method createDataBuffer.

private BufferOrEvent createDataBuffer() throws IOException {
    try (BufferBuilder bufferBuilder = BufferBuilderTestUtils.createEmptyBufferBuilder(PAGE_SIZE)) {
        BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
        serializeRecord(42L, bufferBuilder);
        serializeRecord(44L, bufferBuilder);
        return new BufferOrEvent(bufferConsumer.build(), new InputChannelInfo(0, 0));
    }
}

Example 49 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by apache.

the class ChannelStateSerializerImplTest method testReadToBufferBuilder.

@Test
public void testReadToBufferBuilder() throws IOException {
    byte[] data = generateData(100);
    BufferBuilder bufferBuilder = new BufferBuilder(MemorySegmentFactory.allocateUnpooledSegment(data.length, null), FreeingBufferRecycler.INSTANCE);
    BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
    new ChannelStateSerializerImpl().readData(new ByteArrayInputStream(data), wrap(bufferBuilder), Integer.MAX_VALUE);
    assertFalse(bufferBuilder.isFinished());
    bufferBuilder.finish();
    Buffer buffer = bufferConsumer.build();
    assertEquals(data.length, buffer.readableBytes());
    byte[] actual = new byte[buffer.readableBytes()];
    buffer.asByteBuf().readBytes(actual);
    assertArrayEquals(data, actual);
}

Example 50 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by apache.

the class PipelinedSubpartitionWithReadViewTest method testAddEmptyNonFinishedBuffer.

@Test
public void testAddEmptyNonFinishedBuffer() throws IOException {
    assertEquals(0, availablityListener.getNumNotifications());
    BufferBuilder bufferBuilder = createBufferBuilder();
    subpartition.add(bufferBuilder.createBufferConsumer());
    assertEquals(0, availablityListener.getNumNotifications());
    assertNull(readView.getNextBuffer());
    bufferBuilder.finish();
    bufferBuilder = createBufferBuilder();
    subpartition.add(bufferBuilder.createBufferConsumer());
    assertEquals(1, subpartition.getBuffersInBacklogUnsafe());
    assertEquals(1, availablityListener.getNumNotifications());
    assertNull(readView.getNextBuffer());
    assertEquals(0, subpartition.getBuffersInBacklogUnsafe());
}