Examples with BufferOrEvent org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent used on opensource projects

Example 51 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierTrackerTest method testMultiChannelNoBarriers.

@Test
public void testMultiChannelNoBarriers() {
    try {
        BufferOrEvent[] sequence = { createBuffer(2), createBuffer(2), createBuffer(0), createBuffer(1), createBuffer(0), createBuffer(3), createBuffer(1), createBuffer(1), createBuffer(2) };
        MockInputGate gate = new MockInputGate(PAGE_SIZE, 4, Arrays.asList(sequence));
        BarrierTracker tracker = new BarrierTracker(gate);
        for (BufferOrEvent boe : sequence) {
            assertEquals(boe, tracker.getNextNonBlocked());
        }
        assertNull(tracker.getNextNonBlocked());
        assertNull(tracker.getNextNonBlocked());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 52 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierTrackerTest method testMultiChannelWithBarriers.

@Test
public void testMultiChannelWithBarriers() {
    try {
        BufferOrEvent[] sequence = { createBuffer(0), createBuffer(2), createBuffer(0), createBarrier(1, 1), createBarrier(1, 2), createBuffer(2), createBuffer(1), createBarrier(1, 0), createBuffer(0), createBuffer(0), createBuffer(1), createBuffer(1), createBuffer(2), createBarrier(2, 0), createBarrier(2, 1), createBarrier(2, 2), createBuffer(2), createBuffer(2), createBarrier(3, 2), createBuffer(2), createBuffer(2), createBarrier(3, 0), createBarrier(3, 1), createBarrier(4, 1), createBarrier(4, 2), createBarrier(4, 0), createBuffer(0) };
        MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
        BarrierTracker tracker = new BarrierTracker(gate);
        CheckpointSequenceValidator validator = new CheckpointSequenceValidator(1, 2, 3, 4);
        tracker.registerCheckpointEventHandler(validator);
        for (BufferOrEvent boe : sequence) {
            if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) {
                assertEquals(boe, tracker.getNextNonBlocked());
            }
        }
        assertNull(tracker.getNextNonBlocked());
        assertNull(tracker.getNextNonBlocked());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 53 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BufferSpillerTest method generateRandomBuffer.

private static BufferOrEvent generateRandomBuffer(int size, int channelIndex) {
    MemorySegment seg = MemorySegmentFactory.allocateUnpooledSegment(PAGE_SIZE);
    for (int i = 0; i < size; i++) {
        seg.put(i, (byte) i);
    }
    Buffer buf = new Buffer(seg, FreeingBufferRecycler.INSTANCE);
    buf.setSize(size);
    return new BufferOrEvent(buf, channelIndex);
}

Example 54 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BufferSpillerTest method testHeaderSizeStaticField.

/**
	 * Tests that the static HEADER_SIZE field has valid header size.
	 */
@Test
public void testHeaderSizeStaticField() throws Exception {
    int size = 13;
    BufferOrEvent boe = generateRandomBuffer(size, 0);
    spiller.add(boe);
    assertEquals("Changed the header format, but did not adjust the HEADER_SIZE field", BufferSpiller.HEADER_SIZE + size, spiller.getBytesWritten());
}

Example 55 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BufferSpillerTest method testSpillAndRollOverSimple.

@Test
public void testSpillAndRollOverSimple() {
    try {
        final Random rnd = new Random();
        final Random bufferRnd = new Random();
        final int maxNumEventsAndBuffers = 3000;
        final int maxNumChannels = 1656;
        // do multiple spilling / rolling over rounds
        for (int round = 0; round < 5; round++) {
            final long bufferSeed = rnd.nextLong();
            bufferRnd.setSeed(bufferSeed);
            final int numEventsAndBuffers = rnd.nextInt(maxNumEventsAndBuffers) + 1;
            final int numChannels = rnd.nextInt(maxNumChannels) + 1;
            final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(128);
            // generate sequence
            for (int i = 0; i < numEventsAndBuffers; i++) {
                boolean isEvent = rnd.nextDouble() < 0.05d;
                if (isEvent) {
                    BufferOrEvent evt = generateRandomEvent(rnd, numChannels);
                    events.add(evt);
                    spiller.add(evt);
                } else {
                    BufferOrEvent evt = generateRandomBuffer(bufferRnd.nextInt(PAGE_SIZE) + 1, bufferRnd.nextInt(numChannels));
                    spiller.add(evt);
                }
            }
            // reset and create reader
            bufferRnd.setSeed(bufferSeed);
            BufferSpiller.SpilledBufferOrEventSequence seq = spiller.rollOver();
            seq.open();
            // read and validate the sequence
            int numEvent = 0;
            for (int i = 0; i < numEventsAndBuffers; i++) {
                BufferOrEvent next = seq.getNext();
                assertNotNull(next);
                if (next.isEvent()) {
                    BufferOrEvent expected = events.get(numEvent++);
                    assertEquals(expected.getEvent(), next.getEvent());
                    assertEquals(expected.getChannelIndex(), next.getChannelIndex());
                } else {
                    validateBuffer(next, bufferRnd.nextInt(PAGE_SIZE) + 1, bufferRnd.nextInt(numChannels));
                }
            }
            // no further data
            assertNull(seq.getNext());
            // all events need to be consumed
            assertEquals(events.size(), numEvent);
            seq.cleanup();
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}