Examples with BufferBuilder org.apache.flink.runtime.io.network.buffer.BufferBuilder used on opensource projects

Example 41 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class HashBasedDataBuffer method writeRecord.

private void writeRecord(ByteBuffer source, int targetChannel) throws IOException {
    do {
        BufferBuilder builder = builders[targetChannel];
        if (builder == null) {
            builder = requestBufferFromPool();
            if (builder == null) {
                break;
            }
            ++numBuffersOccupied;
            builders[targetChannel] = builder;
        }
        builder.append(source);
        if (builder.isFull()) {
            builder.finish();
            buffers[targetChannel].add(builder.createBufferConsumerFromBeginning());
            builder.close();
            builders[targetChannel] = null;
        }
    } while (source.hasRemaining());
}

Example 42 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class HashBasedDataBuffer method writeEvent.

private void writeEvent(ByteBuffer source, int targetChannel, Buffer.DataType dataType) {
    BufferBuilder builder = builders[targetChannel];
    if (builder != null) {
        builder.finish();
        buffers[targetChannel].add(builder.createBufferConsumerFromBeginning());
        builder.close();
        builders[targetChannel] = null;
    }
    MemorySegment segment = MemorySegmentFactory.allocateUnpooledOffHeapMemory(source.remaining());
    segment.put(0, source, segment.size());
    BufferConsumer consumer = new BufferConsumer(new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE, dataType), segment.size());
    buffers[targetChannel].add(consumer);
}

Example 43 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class DemultiplexingRecordDeserializerTest method testAmbiguousChannels.

/**
 * Tests that {@link RecordFilter} are used correctly.
 */
@Test
public void testAmbiguousChannels() throws IOException {
    DemultiplexingRecordDeserializer<Long> deserializer = DemultiplexingRecordDeserializer.create(new InputChannelInfo(1, 0), rescalingDescriptor(to(41, 42), array(mappings(), mappings(to(2, 3), to(4, 5))), set(42)), unused -> new SpillingAdaptiveSpanningRecordDeserializer<>(ioManager.getSpillingDirectoriesPaths()), unused -> new RecordFilter(new ModSelector(2), LongSerializer.INSTANCE, 1));
    assertEquals(Sets.newSet(new SubtaskConnectionDescriptor(41, 2), new SubtaskConnectionDescriptor(41, 3), new SubtaskConnectionDescriptor(42, 2), new SubtaskConnectionDescriptor(42, 3)), deserializer.getVirtualChannelSelectors());
    for (int i = 0; i < 100; i++) {
        MemorySegment memorySegment = allocateUnpooledSegment(128);
        try (BufferBuilder bufferBuilder = createBufferBuilder(memorySegment)) {
            // add one even and one odd number
            Buffer buffer = writeLongs(bufferBuilder, i, i + 1L);
            SubtaskConnectionDescriptor selector = Iterables.get(deserializer.getVirtualChannelSelectors(), i / 10 % 2);
            deserializer.select(selector);
            deserializer.setNextBuffer(buffer);
            if (selector.getInputSubtaskIndex() == 41) {
                assertEquals(Arrays.asList((long) i, i + 1L), readLongs(deserializer));
            } else {
                // only odd should occur in output
                assertEquals(Arrays.asList(i / 2 * 2 + 1L), readLongs(deserializer));
            }
        }
        assertTrue(memorySegment.isFreed());
    }
}

Example 44 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class ChannelStateSerializerImplTest method testReadToBufferBuilder.

@Test
public void testReadToBufferBuilder() throws IOException {
    byte[] data = generateData(100);
    BufferBuilder bufferBuilder = new BufferBuilder(MemorySegmentFactory.allocateUnpooledSegment(data.length, null), FreeingBufferRecycler.INSTANCE);
    BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
    new ChannelStateSerializerImpl().readData(new ByteArrayInputStream(data), wrap(bufferBuilder), Integer.MAX_VALUE);
    assertFalse(bufferBuilder.isFinished());
    bufferBuilder.finish();
    Buffer buffer = bufferConsumer.build();
    assertEquals(data.length, buffer.readableBytes());
    byte[] actual = new byte[buffer.readableBytes()];
    buffer.asByteBuf().readBytes(actual);
    assertArrayEquals(data, actual);
}

Example 45 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink by splunk.

the class SpanningRecordSerializationTest method appendLeftOverBytes.

private static Buffer appendLeftOverBytes(Buffer buffer, byte[] leftOverBytes) {
    try (BufferBuilder bufferBuilder = new BufferBuilder(MemorySegmentFactory.allocateUnpooledSegment(buffer.readableBytes() + leftOverBytes.length), FreeingBufferRecycler.INSTANCE)) {
        try (BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer()) {
            bufferBuilder.append(buffer.getNioBufferReadable());
            bufferBuilder.appendAndCommit(ByteBuffer.wrap(leftOverBytes));
            return bufferConsumer.build();
        }
    }
}