Examples with BufferOrEvent org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent used on opensource projects

Example 71 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class RecordWriterTest method parseBuffer.

// ---------------------------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------------------------
static BufferOrEvent parseBuffer(Buffer buffer, int targetChannel) throws IOException {
    if (buffer.isBuffer()) {
        return new BufferOrEvent(buffer, new InputChannelInfo(0, targetChannel));
    } else {
        // is event:
        AbstractEvent event = EventSerializer.fromBuffer(buffer, RecordWriterTest.class.getClassLoader());
        // the buffer is not needed anymore
        buffer.recycleBuffer();
        return new BufferOrEvent(event, new InputChannelInfo(0, targetChannel));
    }
}

Example 72 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class RecordWriterTest method testBroadcastEventNoRecords.

// ---------------------------------------------------------------------------------------------
// Resource release tests
// ---------------------------------------------------------------------------------------------
/**
 * Tests broadcasting events when no records have been emitted yet.
 */
@Test
public void testBroadcastEventNoRecords() throws Exception {
    int numberOfChannels = 4;
    int bufferSize = 32;
    ResultPartition partition = createResultPartition(bufferSize, numberOfChannels);
    RecordWriter<ByteArrayIO> writer = createRecordWriter(partition);
    CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 919192L, Integer.MAX_VALUE + 18828228L, CheckpointOptions.forCheckpointWithDefaultLocation());
    // No records emitted yet, broadcast should not request a buffer
    writer.broadcastEvent(barrier);
    assertEquals(0, partition.getBufferPool().bestEffortGetNumOfUsedBuffers());
    for (int i = 0; i < numberOfChannels; i++) {
        assertEquals(1, partition.getNumberOfQueuedBuffers(i));
        ResultSubpartitionView view = partition.createSubpartitionView(i, new NoOpBufferAvailablityListener());
        BufferOrEvent boe = parseBuffer(view.getNextBuffer().buffer(), i);
        assertTrue(boe.isEvent());
        assertEquals(barrier, boe.getEvent());
        assertFalse(view.getAvailabilityAndBacklog(Integer.MAX_VALUE).isAvailable());
    }
}

Example 73 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class AbstractRecordReader method getNextRecord.

protected boolean getNextRecord(T target) throws IOException, InterruptedException {
    // judgement future.
    if (!finishedStateReading) {
        inputGate.finishReadRecoveredState();
        finishedStateReading = true;
    }
    if (!requestedPartitions) {
        CompletableFuture<Void> stateConsumedFuture = inputGate.getStateConsumedFuture();
        while (!stateConsumedFuture.isDone()) {
            Optional<BufferOrEvent> polled = inputGate.pollNext();
            Preconditions.checkState(!polled.isPresent());
        }
        inputGate.setChannelStateWriter(ChannelStateWriter.NO_OP);
        inputGate.requestPartitions();
        requestedPartitions = true;
    }
    if (isFinished) {
        return false;
    }
    while (true) {
        if (currentRecordDeserializer != null) {
            DeserializationResult result = currentRecordDeserializer.getNextRecord(target);
            if (result.isBufferConsumed()) {
                partialData.put(currentRecordDeserializer, Boolean.FALSE);
                currentRecordDeserializer = null;
            }
            if (result.isFullRecord()) {
                return true;
            }
        }
        final BufferOrEvent bufferOrEvent = inputGate.getNext().orElseThrow(IllegalStateException::new);
        if (bufferOrEvent.isBuffer()) {
            currentRecordDeserializer = recordDeserializers.get(bufferOrEvent.getChannelInfo());
            currentRecordDeserializer.setNextBuffer(bufferOrEvent.getBuffer());
            partialData.put(currentRecordDeserializer, Boolean.TRUE);
        } else {
            // records, not in the middle of a fragment
            if (partialData.get(recordDeserializers.get(bufferOrEvent.getChannelInfo()))) {
                throw new IOException("Received an event in channel " + bufferOrEvent.getChannelInfo() + " while still having " + "data from a record. This indicates broken serialization logic. " + "If you are using custom serialization code (Writable or Value types), check their " + "serialization routines. In the case of Kryo, check the respective Kryo serializer.");
            }
            if (handleEvent(bufferOrEvent.getEvent())) {
                if (inputGate.isFinished()) {
                    isFinished = true;
                    return false;
                } else if (hasReachedEndOfSuperstep()) {
                    return false;
                }
            // else: More data is coming...
            }
        }
    }
}