Examples with BufferConsumer org.apache.flink.runtime.io.network.buffer.BufferConsumer used on opensource projects

Example 6 with BufferConsumer

use of org.apache.flink.runtime.io.network.buffer.BufferConsumer in project flink by apache.

the class BufferWritingResultPartition method createBroadcastBufferConsumers.

private void createBroadcastBufferConsumers(BufferBuilder buffer, int partialRecordBytes, int minDesirableBufferSize) throws IOException {
    try (final BufferConsumer consumer = buffer.createBufferConsumerFromBeginning()) {
        int desirableBufferSize = Integer.MAX_VALUE;
        for (ResultSubpartition subpartition : subpartitions) {
            int subPartitionBufferSize = subpartition.add(consumer.copy(), partialRecordBytes);
            desirableBufferSize = subPartitionBufferSize > 0 ? Math.min(desirableBufferSize, subPartitionBufferSize) : desirableBufferSize;
        }
        resizeBuffer(buffer, desirableBufferSize, minDesirableBufferSize);
    }
}

Example 7 with BufferConsumer

use of org.apache.flink.runtime.io.network.buffer.BufferConsumer in project flink by apache.

the class BufferWritingResultPartition method broadcastEvent.

@Override
public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
    checkInProduceState();
    finishBroadcastBufferBuilder();
    finishUnicastBufferBuilders();
    try (BufferConsumer eventBufferConsumer = EventSerializer.toBufferConsumer(event, isPriorityEvent)) {
        totalWrittenBytes += ((long) eventBufferConsumer.getWrittenBytes() * numSubpartitions);
        for (ResultSubpartition subpartition : subpartitions) {
            // Retain the buffer so that it can be recycled by each channel of targetPartition
            subpartition.add(eventBufferConsumer.copy(), 0);
        }
    }
}

Example 8 with BufferConsumer

use of org.apache.flink.runtime.io.network.buffer.BufferConsumer in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionLocalChannelsPreFilled.

@Test
public void testFairConsumptionLocalChannelsPreFilled() throws Exception {
    final int numberOfChannels = 37;
    final int buffersPerChannel = 27;
    PipelinedResultPartition[] resultPartitions = IntStream.range(0, numberOfChannels).mapToObj(i -> (PipelinedResultPartition) new ResultPartitionBuilder().build()).toArray(PipelinedResultPartition[]::new);
    final BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(42);
    // ----- create some source channels and fill them with buffers -----
    final PipelinedSubpartition[] sources = Arrays.stream(resultPartitions).map(resultPartition -> resultPartition.getAllPartitions()[0]).toArray(PipelinedSubpartition[]::new);
    for (final PipelinedSubpartition subpartition : sources) {
        for (int p = 0; p < buffersPerChannel; p++) {
            subpartition.add(bufferConsumer.copy());
        }
        subpartition.finish();
    }
    for (ResultPartition rp : resultPartitions) {
        rp.setup();
    }
    // ----- create reading side -----
    final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
    final InputChannel[] inputChannels = IntStream.range(0, numberOfChannels).mapToObj(i -> InputChannelBuilder.newBuilder().setChannelIndex(i).setPartitionManager(resultPartitions[i].partitionManager).setPartitionId(resultPartitions[i].getPartitionId()).buildLocalChannel(gate)).toArray(InputChannel[]::new);
    setupInputGate(gate, inputChannels);
    // read all the buffers and the EOF event
    for (int i = numberOfChannels * (buffersPerChannel + 1); i > 0; --i) {
        assertNotNull(gate.getNext());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (PipelinedSubpartition source : sources) {
            int size = source.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == (min + 1));
    }
    assertFalse(gate.getNext().isPresent());
}

Example 9 with BufferConsumer

use of org.apache.flink.runtime.io.network.buffer.BufferConsumer in project flink by apache.

the class StreamTaskNetworkInputTest method createDataBuffer.

private BufferOrEvent createDataBuffer() throws IOException {
    try (BufferBuilder bufferBuilder = BufferBuilderTestUtils.createEmptyBufferBuilder(PAGE_SIZE)) {
        BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
        serializeRecord(42L, bufferBuilder);
        serializeRecord(44L, bufferBuilder);
        return new BufferOrEvent(bufferConsumer.build(), new InputChannelInfo(0, 0));
    }
}

Example 10 with BufferConsumer

use of org.apache.flink.runtime.io.network.buffer.BufferConsumer in project flink by apache.

the class SubpartitionTestBase method testAddAfterFinish.

@Test
public void testAddAfterFinish() throws Exception {
    final ResultSubpartition subpartition = createSubpartition();
    try {
        subpartition.finish();
        assertEquals(1, subpartition.getTotalNumberOfBuffersUnsafe());
        assertEquals(0, subpartition.getBuffersInBacklogUnsafe());
        BufferConsumer bufferConsumer = createFilledFinishedBufferConsumer(4096);
        assertEquals(-1, subpartition.add(bufferConsumer));
        assertTrue(bufferConsumer.isRecycled());
        assertEquals(1, subpartition.getTotalNumberOfBuffersUnsafe());
        assertEquals(0, subpartition.getBuffersInBacklogUnsafe());
    } finally {
        if (subpartition != null) {
            subpartition.release();
        }
    }
}