Examples with BufferBuilder org.apache.flink.runtime.io.network.buffer.BufferBuilder used on opensource projects

Example 71 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink-mirror by flink-ci.

the class StreamTestSingleInputGate method setupInputChannels.

private TestInputChannel[] setupInputChannels() {
    TestInputChannel[] inputChannels = new TestInputChannel[numInputChannels];
    for (int i = 0; i < numInputChannels; i++) {
        final int channelIndex = i;
        final DataOutputSerializer dataOutputSerializer = new DataOutputSerializer(128);
        final SerializationDelegate<StreamElement> delegate = new SerializationDelegate<>(new StreamElementSerializer<T>(serializer));
        inputQueues[channelIndex] = new ConcurrentLinkedQueue<>();
        inputChannels[channelIndex] = new TestInputChannel(inputGate, i);
        final BufferAndAvailabilityProvider answer = () -> {
            ConcurrentLinkedQueue<InputValue<Object>> inputQueue = inputQueues[channelIndex];
            InputValue<Object> input;
            Buffer.DataType nextType;
            synchronized (inputQueue) {
                input = inputQueue.poll();
                nextType = !inputQueue.isEmpty() ? Buffer.DataType.DATA_BUFFER : Buffer.DataType.NONE;
            }
            if (input != null && input.isStreamEnd()) {
                inputChannels[channelIndex].setReleased();
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE, false), nextType, 0, 0));
            } else if (input != null && input.isDataEnd()) {
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(new EndOfData(StopMode.DRAIN), false), nextType, 0, 0));
            } else if (input != null && input.isStreamRecord()) {
                StreamElement inputElement = input.getStreamRecord();
                delegate.setInstance(inputElement);
                ByteBuffer serializedRecord = RecordWriter.serializeRecord(dataOutputSerializer, delegate);
                BufferBuilder bufferBuilder = createBufferBuilder(bufferSize);
                BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
                bufferBuilder.appendAndCommit(serializedRecord);
                bufferBuilder.finish();
                bufferBuilder.close();
                // Call getCurrentBuffer to ensure size is set
                return Optional.of(new BufferAndAvailability(bufferConsumer.build(), nextType, 0, 0));
            } else if (input != null && input.isEvent()) {
                AbstractEvent event = input.getEvent();
                if (event instanceof EndOfPartitionEvent) {
                    inputChannels[channelIndex].setReleased();
                }
                return Optional.of(new BufferAndAvailability(EventSerializer.toBuffer(event, false), nextType, 0, 0));
            } else {
                return Optional.empty();
            }
        };
        inputChannels[channelIndex].addBufferAndAvailability(answer);
    }
    return inputChannels;
}

Example 72 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink-mirror by flink-ci.

the class DemultiplexingRecordDeserializerTest method testWatermarks.

/**
 * Tests that Watermarks are only forwarded when all watermarks are received.
 */
@Test
public void testWatermarks() throws IOException {
    DemultiplexingRecordDeserializer<Long> deserializer = DemultiplexingRecordDeserializer.create(new InputChannelInfo(0, 0), rescalingDescriptor(to(0, 1), array(mappings(to(0, 1), to(4, 5))), emptySet()), unused -> new SpillingAdaptiveSpanningRecordDeserializer<>(ioManager.getSpillingDirectoriesPaths()), unused -> RecordFilter.all());
    assertEquals(4, deserializer.getVirtualChannelSelectors().size());
    for (Iterator<SubtaskConnectionDescriptor> iterator = deserializer.getVirtualChannelSelectors().iterator(); iterator.hasNext(); ) {
        SubtaskConnectionDescriptor selector = iterator.next();
        MemorySegment memorySegment = allocateUnpooledSegment(128);
        try (BufferBuilder bufferBuilder = createBufferBuilder(memorySegment)) {
            final long ts = 42L + selector.getInputSubtaskIndex() + selector.getOutputSubtaskIndex();
            Buffer buffer = write(bufferBuilder, new Watermark(ts));
            deserializer.select(selector);
            deserializer.setNextBuffer(buffer);
        }
        if (iterator.hasNext()) {
            assertEquals(Collections.emptyList(), read(deserializer));
        } else {
            // last channel, min should be 42 + 0 + 0
            assertEquals(Arrays.asList(new Watermark(42)), read(deserializer));
        }
        assertTrue(memorySegment.isFreed());
    }
}

Example 73 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink-mirror by flink-ci.

the class DemultiplexingRecordDeserializerTest method testUpscale.

/**
 * Tests {@link SubtaskConnectionDescriptor} by mixing buffers from 4 different virtual
 * channels.
 */
@Test
public void testUpscale() throws IOException {
    DemultiplexingRecordDeserializer<Long> deserializer = DemultiplexingRecordDeserializer.create(new InputChannelInfo(2, 0), rescalingDescriptor(to(0, 1), array(mappings(), mappings(), mappings(to(2, 3), to(4, 5))), emptySet()), unused -> new SpillingAdaptiveSpanningRecordDeserializer<>(ioManager.getSpillingDirectoriesPaths()), unused -> RecordFilter.all());
    assertEquals(Sets.newSet(new SubtaskConnectionDescriptor(0, 2), new SubtaskConnectionDescriptor(0, 3), new SubtaskConnectionDescriptor(1, 2), new SubtaskConnectionDescriptor(1, 3)), deserializer.getVirtualChannelSelectors());
    for (int i = 0; i < 100; i++) {
        SubtaskConnectionDescriptor selector = Iterables.get(deserializer.getVirtualChannelSelectors(), random.nextInt(4));
        long start = selector.getInputSubtaskIndex() << 4 | selector.getOutputSubtaskIndex();
        MemorySegment memorySegment = allocateUnpooledSegment(128);
        try (BufferBuilder bufferBuilder = createBufferBuilder(memorySegment)) {
            Buffer buffer = writeLongs(bufferBuilder, start + 1L, start + 2L, start + 3L);
            deserializer.select(selector);
            deserializer.setNextBuffer(buffer);
        }
        assertEquals(Arrays.asList(start + 1L, start + 2L, start + 3L), readLongs(deserializer));
        assertTrue(memorySegment.isFreed());
    }
}

Example 74 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink-mirror by flink-ci.

the class BufferWritingResultPartition method broadcastRecord.

@Override
public void broadcastRecord(ByteBuffer record) throws IOException {
    totalWrittenBytes += ((long) record.remaining() * numSubpartitions);
    BufferBuilder buffer = appendBroadcastDataForNewRecord(record);
    while (record.hasRemaining()) {
        // full buffer, partial record
        finishBroadcastBufferBuilder();
        buffer = appendBroadcastDataForRecordContinuation(record);
    }
    if (buffer.isFull()) {
        // full buffer, full record
        finishBroadcastBufferBuilder();
    }
// partial buffer, full record
}

Example 75 with BufferBuilder

use of org.apache.flink.runtime.io.network.buffer.BufferBuilder in project flink-mirror by flink-ci.

the class BufferWritingResultPartition method requestNewBufferBuilderFromPool.

private BufferBuilder requestNewBufferBuilderFromPool(int targetSubpartition) throws IOException {
    BufferBuilder bufferBuilder = bufferPool.requestBufferBuilder(targetSubpartition);
    if (bufferBuilder != null) {
        return bufferBuilder;
    }
    hardBackPressuredTimeMsPerSecond.markStart();
    try {
        bufferBuilder = bufferPool.requestBufferBuilderBlocking(targetSubpartition);
        hardBackPressuredTimeMsPerSecond.markEnd();
        return bufferBuilder;
    } catch (InterruptedException e) {
        throw new IOException("Interrupted while waiting for buffer");
    }
}