Examples with BufferOrEvent org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent used on opensource projects

Example 31 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBuffer method getNextNonBlocked.

// ------------------------------------------------------------------------
//  Buffer and barrier handling
// ------------------------------------------------------------------------
@Override
public BufferOrEvent getNextNonBlocked() throws Exception {
    while (true) {
        // process buffered BufferOrEvents before grabbing new ones
        BufferOrEvent next;
        if (currentBuffered == null) {
            next = inputGate.getNextBufferOrEvent();
        } else {
            next = currentBuffered.getNext();
            if (next == null) {
                completeBufferedSequence();
                return getNextNonBlocked();
            }
        }
        if (next != null) {
            if (isBlocked(next.getChannelIndex())) {
                // if the channel is blocked we, we just store the BufferOrEvent
                bufferSpiller.add(next);
                checkSizeLimit();
            } else if (next.isBuffer()) {
                return next;
            } else if (next.getEvent().getClass() == CheckpointBarrier.class) {
                if (!endOfStream) {
                    // process barriers only if there is a chance of the checkpoint completing
                    processBarrier((CheckpointBarrier) next.getEvent(), next.getChannelIndex());
                }
            } else if (next.getEvent().getClass() == CancelCheckpointMarker.class) {
                processCancellationBarrier((CancelCheckpointMarker) next.getEvent());
            } else {
                if (next.getEvent().getClass() == EndOfPartitionEvent.class) {
                    processEndOfPartition();
                }
                return next;
            }
        } else if (!endOfStream) {
            // end of input stream. stream continues with the buffered data
            endOfStream = true;
            releaseBlocksAndResetBarriers();
            return getNextNonBlocked();
        } else {
            // final end of both input and buffered data
            return null;
        }
    }
}

Example 32 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class MockInputGate method getNextBufferOrEvent.

@Override
public BufferOrEvent getNextBufferOrEvent() {
    BufferOrEvent next = boes.poll();
    if (next == null) {
        return null;
    }
    int channelIdx = next.getChannelIndex();
    if (closed[channelIdx]) {
        throw new RuntimeException("Inconsistent: Channel " + channelIdx + " has data even though it is already closed.");
    }
    if (next.isEvent() && next.getEvent() instanceof EndOfPartitionEvent) {
        closed[channelIdx] = true;
        closedChannels++;
    }
    return next;
}

Example 33 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class SpilledBufferOrEventSequenceTest method testMixedSequence.

@Test
public void testMixedSequence() {
    try {
        final Random rnd = new Random();
        final Random bufferRnd = new Random();
        final long bufferSeed = rnd.nextLong();
        bufferRnd.setSeed(bufferSeed);
        final int numEventsAndBuffers = 3000;
        final int numChannels = 1656;
        final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(128);
        for (int i = 0; i < numEventsAndBuffers; i++) {
            boolean isEvent = rnd.nextDouble() < 0.05d;
            if (isEvent) {
                events.add(generateAndWriteEvent(fileChannel, rnd, numChannels));
            } else {
                writeBuffer(fileChannel, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        // reset and create reader
        fileChannel.position(0L);
        bufferRnd.setSeed(bufferSeed);
        SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
        seq.open();
        // read and validate the sequence
        int numEvent = 0;
        for (int i = 0; i < numEventsAndBuffers; i++) {
            BufferOrEvent next = seq.getNext();
            if (next.isEvent()) {
                BufferOrEvent expected = events.get(numEvent++);
                assertEquals(expected.getEvent(), next.getEvent());
                assertEquals(expected.getChannelIndex(), next.getChannelIndex());
            } else {
                validateBuffer(next, bufferRnd.nextInt(pageSize) + 1, bufferRnd.nextInt(numChannels));
            }
        }
        // no further data
        assertNull(seq.getNext());
        // all events need to be consumed
        assertEquals(events.size(), numEvent);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 34 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class SpilledBufferOrEventSequenceTest method testEventSequence.

@Test
public void testEventSequence() {
    try {
        final Random rnd = new Random();
        final int numEvents = 3000;
        final int numChannels = 1656;
        final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(numEvents);
        for (int i = 0; i < numEvents; i++) {
            events.add(generateAndWriteEvent(fileChannel, rnd, numChannels));
        }
        fileChannel.position(0L);
        SpilledBufferOrEventSequence seq = new SpilledBufferOrEventSequence(tempFile, fileChannel, buffer, pageSize);
        seq.open();
        int i = 0;
        BufferOrEvent boe;
        while ((boe = seq.getNext()) != null) {
            BufferOrEvent expected = events.get(i);
            assertTrue(boe.isEvent());
            assertEquals(expected.getEvent(), boe.getEvent());
            assertEquals(expected.getChannelIndex(), boe.getChannelIndex());
            i++;
        }
        assertEquals(numEvents, i);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 35 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class SpilledBufferOrEventSequenceTest method generateAndWriteEvent.

// ------------------------------------------------------------------------
//  Utils
// ------------------------------------------------------------------------
private static BufferOrEvent generateAndWriteEvent(FileChannel fileChannel, Random rnd, int numChannels) throws IOException {
    long magicNumber = rnd.nextLong();
    byte[] data = new byte[rnd.nextInt(1000)];
    rnd.nextBytes(data);
    TestEvent evt = new TestEvent(magicNumber, data);
    int channelIndex = rnd.nextInt(numChannels);
    ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(evt);
    ByteBuffer header = ByteBuffer.allocate(9);
    header.order(ByteOrder.LITTLE_ENDIAN);
    header.putInt(channelIndex);
    header.putInt(serializedEvent.remaining());
    header.put((byte) 1);
    header.flip();
    fileChannel.write(header);
    fileChannel.write(serializedEvent);
    return new BufferOrEvent(evt, channelIndex);
}