Examples with BufferBuilder org.apache.flink.runtime.io.network.buffer.BufferBuilder used on opensource projects

Example 11 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by apache.

the class BufferWritingResultPartition method requestNewBufferBuilderFromPool.

private BufferBuilder requestNewBufferBuilderFromPool(int targetSubpartition) throws IOException {
    BufferBuilder bufferBuilder = bufferPool.requestBufferBuilder(targetSubpartition);
    if (bufferBuilder != null) {
        return bufferBuilder;
    }
    hardBackPressuredTimeMsPerSecond.markStart();
    try {
        bufferBuilder = bufferPool.requestBufferBuilderBlocking(targetSubpartition);
        hardBackPressuredTimeMsPerSecond.markEnd();
        return bufferBuilder;
    } catch (InterruptedException e) {
        throw new IOException("Interrupted while waiting for buffer");
    }
}

Example 12 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by apache.

the class BufferWritingResultPartition method requestNewBroadcastBufferBuilder.

private BufferBuilder requestNewBroadcastBufferBuilder() throws IOException {
    checkInProduceState();
    ensureBroadcastMode();
    final BufferBuilder bufferBuilder = requestNewBufferBuilderFromPool(0);
    broadcastBufferBuilder = bufferBuilder;
    return bufferBuilder;
}

Example 13 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by apache.

the class BufferWritingResultPartition method requestNewUnicastBufferBuilder.

private BufferBuilder requestNewUnicastBufferBuilder(int targetSubpartition) throws IOException {
    checkInProduceState();
    ensureUnicastMode();
    final BufferBuilder bufferBuilder = requestNewBufferBuilderFromPool(targetSubpartition);
    unicastBufferBuilders[targetSubpartition] = bufferBuilder;
    return bufferBuilder;
}

Example 14 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by apache.

the class StreamTaskNetworkInputTest method createDataBuffer.

private BufferOrEvent createDataBuffer() throws IOException {
    try (BufferBuilder bufferBuilder = BufferBuilderTestUtils.createEmptyBufferBuilder(PAGE_SIZE)) {
        BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
        serializeRecord(42L, bufferBuilder);
        serializeRecord(44L, bufferBuilder);
        return new BufferOrEvent(bufferConsumer.build(), new InputChannelInfo(0, 0));
    }
}

Example 15 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by apache.

the class RemoteInputChannelTest method testConcurrentRecycleAndRelease2.

/**
 * Tests to verify that there is no race condition with two things running in parallel:
 * recycling exclusive buffers and recycling external buffers to the buffer pool while the
 * recycling of the exclusive buffer triggers recycling a floating buffer (FLINK-9676).
 */
@Test
public void testConcurrentRecycleAndRelease2() throws Exception {
    // Setup
    final int retries = 1_000;
    final int numExclusiveBuffers = 2;
    final int numFloatingBuffers = 2;
    final int numTotalBuffers = numExclusiveBuffers + numFloatingBuffers;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(numTotalBuffers, 32);
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
    inputGate.setInputChannels(inputChannel);
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        final Callable<Void> bufferPoolInteractionsTask = () -> {
            for (int i = 0; i < retries; ++i) {
                try (BufferBuilder bufferBuilder = bufferPool.requestBufferBuilderBlocking()) {
                    Buffer buffer = buildSingleBuffer(bufferBuilder);
                    buffer.recycleBuffer();
                }
            }
            return null;
        };
        final Callable<Void> channelInteractionsTask = () -> {
            ArrayList<Buffer> exclusiveBuffers = new ArrayList<>(numExclusiveBuffers);
            ArrayList<Buffer> floatingBuffers = new ArrayList<>(numExclusiveBuffers);
            try {
                for (int i = 0; i < retries; ++i) {
                    // floating buffers as soon as we take exclusive ones
                    for (int j = 0; j < numTotalBuffers; ++j) {
                        Buffer buffer = inputChannel.requestBuffer();
                        if (buffer == null) {
                            break;
                        } else {
                            // noinspection ObjectEquality
                            if (buffer.getRecycler() == inputChannel.getBufferManager()) {
                                exclusiveBuffers.add(buffer);
                            } else {
                                floatingBuffers.add(buffer);
                            }
                        }
                    }
                    // recycle excess floating buffers (will go back into the channel)
                    floatingBuffers.forEach(Buffer::recycleBuffer);
                    floatingBuffers.clear();
                    assertEquals(numExclusiveBuffers, exclusiveBuffers.size());
                    inputChannel.onSenderBacklog(// trigger subscription to buffer pool
                    0);
                    // note: if we got a floating buffer by increasing the backlog, it
                    // will be released again when recycling the exclusive buffer, if
                    // not, we should release it once we get it
                    exclusiveBuffers.forEach(Buffer::recycleBuffer);
                    exclusiveBuffers.clear();
                }
            } finally {
                inputChannel.releaseAllResources();
            }
            return null;
        };
        // Submit tasks and wait to finish
        submitTasksAndWaitForResults(executor, new Callable[] { bufferPoolInteractionsTask, channelInteractionsTask });
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, executor, null, thrown, inputChannel);
    }
}