Examples with BufferBuilder org.apache.flink.runtime.io.network.buffer.BufferBuilder used on opensource projects

Example 21 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class SpanningRecordSerializationTest method appendLeftOverBytes.

private static Buffer appendLeftOverBytes(Buffer buffer, byte[] leftOverBytes) {
    try (BufferBuilder bufferBuilder = new BufferBuilder(MemorySegmentFactory.allocateUnpooledSegment(buffer.readableBytes() + leftOverBytes.length), FreeingBufferRecycler.INSTANCE)) {
        try (BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer()) {
            bufferBuilder.append(buffer.getNioBufferReadable());
            bufferBuilder.appendAndCommit(ByteBuffer.wrap(leftOverBytes));
            return bufferConsumer.build();
        }
    }
}

Example 22 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class RecordWriterTest method testIsAvailableOrNot.

/**
 * Tests that the RecordWriter is available iif the respective LocalBufferPool has at-least one
 * available buffer.
 */
@Test
public void testIsAvailableOrNot() throws Exception {
    // setup
    final NetworkBufferPool globalPool = new NetworkBufferPool(10, 128);
    final BufferPool localPool = globalPool.createBufferPool(1, 1, 1, Integer.MAX_VALUE);
    final ResultPartitionWriter resultPartition = new ResultPartitionBuilder().setBufferPoolFactory(() -> localPool).build();
    resultPartition.setup();
    final RecordWriter<?> recordWriter = createRecordWriter(resultPartition);
    try {
        // record writer is available because of initial available global pool
        assertTrue(recordWriter.getAvailableFuture().isDone());
        // request one buffer from the local pool to make it unavailable afterwards
        try (BufferBuilder bufferBuilder = localPool.requestBufferBuilder(0)) {
            assertNotNull(bufferBuilder);
            assertFalse(recordWriter.getAvailableFuture().isDone());
            // recycle the buffer to make the local pool available again
            final Buffer buffer = BufferBuilderTestUtils.buildSingleBuffer(bufferBuilder);
            buffer.recycleBuffer();
        }
        assertTrue(recordWriter.getAvailableFuture().isDone());
        assertEquals(recordWriter.AVAILABLE, recordWriter.getAvailableFuture());
    } finally {
        localPool.lazyDestroy();
        globalPool.destroy();
    }
}

Example 23 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class RemoteInputChannelTest method testConcurrentRecycleAndRelease2.

/**
 * Tests to verify that there is no race condition with two things running in parallel:
 * recycling exclusive buffers and recycling external buffers to the buffer pool while the
 * recycling of the exclusive buffer triggers recycling a floating buffer (FLINK-9676).
 */
@Test
public void testConcurrentRecycleAndRelease2() throws Exception {
    // Setup
    final int retries = 1_000;
    final int numExclusiveBuffers = 2;
    final int numFloatingBuffers = 2;
    final int numTotalBuffers = numExclusiveBuffers + numFloatingBuffers;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(numTotalBuffers, 32);
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
    inputGate.setInputChannels(inputChannel);
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        final Callable<Void> bufferPoolInteractionsTask = () -> {
            for (int i = 0; i < retries; ++i) {
                try (BufferBuilder bufferBuilder = bufferPool.requestBufferBuilderBlocking()) {
                    Buffer buffer = buildSingleBuffer(bufferBuilder);
                    buffer.recycleBuffer();
                }
            }
            return null;
        };
        final Callable<Void> channelInteractionsTask = () -> {
            ArrayList<Buffer> exclusiveBuffers = new ArrayList<>(numExclusiveBuffers);
            ArrayList<Buffer> floatingBuffers = new ArrayList<>(numExclusiveBuffers);
            try {
                for (int i = 0; i < retries; ++i) {
                    // floating buffers as soon as we take exclusive ones
                    for (int j = 0; j < numTotalBuffers; ++j) {
                        Buffer buffer = inputChannel.requestBuffer();
                        if (buffer == null) {
                            break;
                        } else {
                            // noinspection ObjectEquality
                            if (buffer.getRecycler() == inputChannel.getBufferManager()) {
                                exclusiveBuffers.add(buffer);
                            } else {
                                floatingBuffers.add(buffer);
                            }
                        }
                    }
                    // recycle excess floating buffers (will go back into the channel)
                    floatingBuffers.forEach(Buffer::recycleBuffer);
                    floatingBuffers.clear();
                    assertEquals(numExclusiveBuffers, exclusiveBuffers.size());
                    inputChannel.onSenderBacklog(// trigger subscription to buffer pool
                    0);
                    // note: if we got a floating buffer by increasing the backlog, it
                    // will be released again when recycling the exclusive buffer, if
                    // not, we should release it once we get it
                    exclusiveBuffers.forEach(Buffer::recycleBuffer);
                    exclusiveBuffers.clear();
                }
            } finally {
                inputChannel.releaseAllResources();
            }
            return null;
        };
        // Submit tasks and wait to finish
        submitTasksAndWaitForResults(executor, new Callable[] { bufferPoolInteractionsTask, channelInteractionsTask });
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, executor, null, thrown, inputChannel);
    }
}

Example 24 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class StreamTestSingleInputGate method setupInputChannels.

private TestInputChannel[] setupInputChannels() {
    TestInputChannel[] inputChannels = new TestInputChannel[numInputChannels];
    for (int i = 0; i < numInputChannels; i++) {
        final int channelIndex = i;
        final DataOutputSerializer dataOutputSerializer = new DataOutputSerializer(128);
        final SerializationDelegate<StreamElement> delegate = new SerializationDelegate<>(new StreamElementSerializer<T>(serializer));
        inputQueues[channelIndex] = new ConcurrentLinkedQueue<>();
        inputChannels[channelIndex] = new TestInputChannel(inputGate, i);
        final BufferAndAvailabilityProvider answer = () -> {
            ConcurrentLinkedQueue<InputValue<Object>> inputQueue = inputQueues[channelIndex];
            InputValue<Object> input;
            Buffer.DataType nextType;
            synchronized (inputQueue) {
                input = inputQueue.poll();
                nextType = !inputQueue.isEmpty() ? Buffer.DataType.DATA_BUFFER : Buffer.DataType.NONE;
            }
            if (input != null && input.isStreamEnd()) {
                inputChannels[channelIndex].setReleased();
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE, false), nextType, 0, 0));
            } else if (input != null && input.isDataEnd()) {
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(new EndOfData(StopMode.DRAIN), false), nextType, 0, 0));
            } else if (input != null && input.isStreamRecord()) {
                StreamElement inputElement = input.getStreamRecord();
                delegate.setInstance(inputElement);
                ByteBuffer serializedRecord = RecordWriter.serializeRecord(dataOutputSerializer, delegate);
                BufferBuilder bufferBuilder = createBufferBuilder(bufferSize);
                BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
                bufferBuilder.appendAndCommit(serializedRecord);
                bufferBuilder.finish();
                bufferBuilder.close();
                // Call getCurrentBuffer to ensure size is set
                return Optional.of(new BufferAndAvailability(bufferConsumer.build(), nextType, 0, 0));
            } else if (input != null && input.isEvent()) {
                AbstractEvent event = input.getEvent();
                if (event instanceof EndOfPartitionEvent) {
                    inputChannels[channelIndex].setReleased();
                }
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(event, false), nextType, 0, 0));
            } else {
                return Optional.empty();
            }
        };
        inputChannels[channelIndex].addBufferAndAvailability(answer);
    }
    return inputChannels;
}

Example 25 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class BufferWritingResultPartition method finishUnicastBufferBuilder.

private void finishUnicastBufferBuilder(int targetSubpartition) {
    final BufferBuilder bufferBuilder = unicastBufferBuilders[targetSubpartition];
    if (bufferBuilder != null) {
        int bytes = bufferBuilder.finish();
        numBytesProduced.inc(bytes);
        numBytesOut.inc(bytes);
        numBuffersOut.inc();
        unicastBufferBuilders[targetSubpartition] = null;
        bufferBuilder.close();
    }
}