Examples with BufferOrEvent org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent used on opensource projects

Example 56 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BufferSpillerTest method testSpillWhileReading.

@Test
public void testSpillWhileReading() {
    LOG.info("Starting SpillWhileReading test");
    try {
        final int sequences = 10;
        final Random rnd = new Random();
        final int maxNumEventsAndBuffers = 30000;
        final int maxNumChannels = 1656;
        int sequencesConsumed = 0;
        ArrayDeque<SequenceToConsume> pendingSequences = new ArrayDeque<SequenceToConsume>();
        SequenceToConsume currentSequence = null;
        int currentNumEvents = 0;
        int currentNumRecordAndEvents = 0;
        // do multiple spilling / rolling over rounds
        for (int round = 0; round < 2 * sequences; round++) {
            if (round % 2 == 1) {
                // make this an empty sequence
                assertNull(spiller.rollOver());
            } else {
                // proper spilled sequence
                final long bufferSeed = rnd.nextLong();
                final Random bufferRnd = new Random(bufferSeed);
                final int numEventsAndBuffers = rnd.nextInt(maxNumEventsAndBuffers) + 1;
                final int numChannels = rnd.nextInt(maxNumChannels) + 1;
                final ArrayList<BufferOrEvent> events = new ArrayList<BufferOrEvent>(128);
                int generated = 0;
                while (generated < numEventsAndBuffers) {
                    if (currentSequence == null || rnd.nextDouble() < 0.5) {
                        // add a new record
                        boolean isEvent = rnd.nextDouble() < 0.05;
                        if (isEvent) {
                            BufferOrEvent evt = generateRandomEvent(rnd, numChannels);
                            events.add(evt);
                            spiller.add(evt);
                        } else {
                            BufferOrEvent evt = generateRandomBuffer(bufferRnd.nextInt(PAGE_SIZE) + 1, bufferRnd.nextInt(numChannels));
                            spiller.add(evt);
                        }
                        generated++;
                    } else {
                        // consume a record
                        BufferOrEvent next = currentSequence.sequence.getNext();
                        assertNotNull(next);
                        if (next.isEvent()) {
                            BufferOrEvent expected = currentSequence.events.get(currentNumEvents++);
                            assertEquals(expected.getEvent(), next.getEvent());
                            assertEquals(expected.getChannelIndex(), next.getChannelIndex());
                        } else {
                            Random validationRnd = currentSequence.bufferRnd;
                            validateBuffer(next, validationRnd.nextInt(PAGE_SIZE) + 1, validationRnd.nextInt(currentSequence.numChannels));
                        }
                        currentNumRecordAndEvents++;
                        if (currentNumRecordAndEvents == currentSequence.numBuffersAndEvents) {
                            // done with the sequence
                            currentSequence.sequence.cleanup();
                            sequencesConsumed++;
                            // validate we had all events
                            assertEquals(currentSequence.events.size(), currentNumEvents);
                            // reset
                            currentSequence = pendingSequences.pollFirst();
                            if (currentSequence != null) {
                                currentSequence.sequence.open();
                            }
                            currentNumRecordAndEvents = 0;
                            currentNumEvents = 0;
                        }
                    }
                }
                // done generating a sequence. queue it for consumption
                bufferRnd.setSeed(bufferSeed);
                BufferSpiller.SpilledBufferOrEventSequence seq = spiller.rollOver();
                SequenceToConsume stc = new SequenceToConsume(bufferRnd, events, seq, numEventsAndBuffers, numChannels);
                if (currentSequence == null) {
                    currentSequence = stc;
                    stc.sequence.open();
                } else {
                    pendingSequences.addLast(stc);
                }
            }
        }
        // consume all the remainder
        while (currentSequence != null) {
            // consume a record
            BufferOrEvent next = currentSequence.sequence.getNext();
            assertNotNull(next);
            if (next.isEvent()) {
                BufferOrEvent expected = currentSequence.events.get(currentNumEvents++);
                assertEquals(expected.getEvent(), next.getEvent());
                assertEquals(expected.getChannelIndex(), next.getChannelIndex());
            } else {
                Random validationRnd = currentSequence.bufferRnd;
                validateBuffer(next, validationRnd.nextInt(PAGE_SIZE) + 1, validationRnd.nextInt(currentSequence.numChannels));
            }
            currentNumRecordAndEvents++;
            if (currentNumRecordAndEvents == currentSequence.numBuffersAndEvents) {
                // done with the sequence
                currentSequence.sequence.cleanup();
                sequencesConsumed++;
                // validate we had all events
                assertEquals(currentSequence.events.size(), currentNumEvents);
                // reset
                currentSequence = pendingSequences.pollFirst();
                if (currentSequence != null) {
                    currentSequence.sequence.open();
                }
                currentNumRecordAndEvents = 0;
                currentNumEvents = 0;
            }
        }
        assertEquals(sequences, sequencesConsumed);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 57 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class MockInputGate method getNext.

@Override
public Optional<BufferOrEvent> getNext() {
    BufferOrEvent next = bufferOrEvents.poll();
    if (!finishAfterLastBuffer && bufferOrEvents.isEmpty()) {
        availabilityHelper.resetUnavailable();
    }
    if (next == null) {
        return Optional.empty();
    }
    int channelIdx = next.getChannelInfo().getInputChannelIdx();
    if (closed[channelIdx]) {
        throw new RuntimeException("Inconsistent: Channel " + channelIdx + " has data even though it is already closed.");
    }
    if (next.isEvent() && next.getEvent() instanceof EndOfPartitionEvent) {
        closed[channelIdx] = true;
    }
    return Optional.of(next);
}

Example 58 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class AlignedCheckpointsMassiveRandomTest method testWithTwoChannelsAndRandomBarriers.

@Test
public void testWithTwoChannelsAndRandomBarriers() throws Exception {
    NetworkBufferPool networkBufferPool1 = null;
    NetworkBufferPool networkBufferPool2 = null;
    try {
        networkBufferPool1 = new NetworkBufferPool(100, PAGE_SIZE);
        networkBufferPool2 = new NetworkBufferPool(100, PAGE_SIZE);
        BufferPool pool1 = networkBufferPool1.createBufferPool(100, 100);
        BufferPool pool2 = networkBufferPool2.createBufferPool(100, 100);
        RandomGeneratingInputGate myIG = new RandomGeneratingInputGate(new BufferPool[] { pool1, pool2 }, new BarrierGenerator[] { new CountBarrier(100000), new RandomBarrier(100000) });
        CheckpointedInputGate checkpointedInputGate = new CheckpointedInputGate(myIG, SingleCheckpointBarrierHandler.aligned("Testing: No task associated", new DummyCheckpointInvokable(), SystemClock.getInstance(), myIG.getNumberOfInputChannels(), (callable, duration) -> () -> {
        }, true, myIG), new SyncMailboxExecutor());
        for (int i = 0; i < 2000000; i++) {
            BufferOrEvent boe = checkpointedInputGate.pollNext().get();
            if (boe.isBuffer()) {
                boe.getBuffer().recycleBuffer();
            }
        }
    } finally {
        if (networkBufferPool1 != null) {
            networkBufferPool1.destroyAllBufferPools();
            networkBufferPool1.destroy();
        }
        if (networkBufferPool2 != null) {
            networkBufferPool2.destroyAllBufferPools();
            networkBufferPool2.destroy();
        }
    }
}

Example 59 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class CheckpointBarrierTrackerTest method testMultiChannelSkippingCheckpoints.

@Test
public void testMultiChannelSkippingCheckpoints() throws Exception {
    BufferOrEvent[] sequence = { createBuffer(0), createBuffer(2), createBuffer(0), createBarrier(1, 1), createBarrier(1, 2), createBuffer(2), createBuffer(1), createBarrier(1, 0), createBuffer(0), createBuffer(0), createBuffer(1), createBuffer(1), createBuffer(2), createBarrier(2, 0), createBarrier(2, 1), createBarrier(2, 2), createBuffer(2), createBuffer(2), createBarrier(3, 2), createBuffer(2), createBuffer(2), // jump to checkpoint 4
    createBarrier(4, 0), createBuffer(0), createBuffer(1), createBuffer(2), createBarrier(4, 1), createBuffer(1), createBarrier(4, 2), createBuffer(0) };
    CheckpointSequenceValidator validator = new CheckpointSequenceValidator(1, 2, 4);
    inputGate = createCheckpointedInputGate(3, sequence, validator);
    for (BufferOrEvent boe : sequence) {
        assertEquals(boe, inputGate.pollNext().get());
    }
}

Example 60 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class CheckpointBarrierTrackerTest method testCompleteAndRemoveAbortedCheckpointWithEndOfPartition.

@Test
public void testCompleteAndRemoveAbortedCheckpointWithEndOfPartition() throws Exception {
    BufferOrEvent[] sequence = { createCancellationBarrier(4, 0), createBarrier(4, 1), createBarrier(5, 1), createEndOfPartition(2) };
    ValidatingCheckpointHandler validator = new ValidatingCheckpointHandler(-1);
    inputGate = createCheckpointedInputGate(3, sequence, validator);
    CheckpointBarrierTracker checkpointBarrierTracker = (CheckpointBarrierTracker) inputGate.getCheckpointBarrierHandler();
    for (BufferOrEvent boe : sequence) {
        assertEquals(boe, inputGate.pollNext().get());
    }
    assertEquals(1, validator.getAbortedCheckpointCounter());
    assertEquals(4L, validator.getLastCanceledCheckpointId());
    assertEquals(0, validator.getTriggeredCheckpointCounter());
    assertThat(checkpointBarrierTracker.getPendingCheckpointIds(), contains(5L));
    assertEquals(2, checkpointBarrierTracker.getNumOpenChannels());
}