Examples with BufferOrEvent org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent used on opensource projects

Example 16 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class RecordWriterDelegateTest method verifyBroadcastEvent.

private void verifyBroadcastEvent(RecordWriterDelegate writerDelegate, List<ResultPartition> partitions) throws Exception {
    final CancelCheckpointMarker message = new CancelCheckpointMarker(1);
    writerDelegate.broadcastEvent(message);
    // verify the added messages in all the queues
    for (ResultPartition partition : partitions) {
        for (int i = 0; i < partition.getNumberOfSubpartitions(); i++) {
            assertEquals(1, partition.getNumberOfQueuedBuffers(i));
            ResultSubpartitionView view = partition.createSubpartitionView(i, new NoOpBufferAvailablityListener());
            BufferOrEvent boe = RecordWriterTest.parseBuffer(view.getNextBuffer().buffer(), i);
            assertTrue(boe.isEvent());
            assertEquals(message, boe.getEvent());
        }
    }
}

Example 17 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class RecordWriterTest method testBroadcastEventMixedRecords.

/**
 * Tests broadcasting events when records have been emitted.
 */
@Test
public void testBroadcastEventMixedRecords() throws Exception {
    Random rand = new XORShiftRandom();
    int numberOfChannels = 4;
    int bufferSize = 32;
    // serialized length
    int lenBytes = 4;
    ResultPartition partition = createResultPartition(bufferSize, numberOfChannels);
    RecordWriter<ByteArrayIO> writer = createRecordWriter(partition);
    CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 1292L, Integer.MAX_VALUE + 199L, CheckpointOptions.forCheckpointWithDefaultLocation());
    // Emit records on some channels first (requesting buffers), then
    // broadcast the event. The record buffers should be emitted first, then
    // the event. After the event, no new buffer should be requested.
    // (i) Smaller than the buffer size
    byte[] bytes = new byte[bufferSize / 2];
    rand.nextBytes(bytes);
    writer.emit(new ByteArrayIO(bytes));
    // (ii) Larger than the buffer size
    bytes = new byte[bufferSize + 1];
    rand.nextBytes(bytes);
    writer.emit(new ByteArrayIO(bytes));
    // (iii) Exactly the buffer size
    bytes = new byte[bufferSize - lenBytes];
    rand.nextBytes(bytes);
    writer.emit(new ByteArrayIO(bytes));
    // (iv) Broadcast the event
    writer.broadcastEvent(barrier);
    if (isBroadcastWriter) {
        assertEquals(3, partition.getBufferPool().bestEffortGetNumOfUsedBuffers());
        for (int i = 0; i < numberOfChannels; i++) {
            // 3 buffer + 1 event
            assertEquals(4, partition.getNumberOfQueuedBuffers(i));
            ResultSubpartitionView view = partition.createSubpartitionView(i, new NoOpBufferAvailablityListener());
            for (int j = 0; j < 3; j++) {
                assertTrue(parseBuffer(view.getNextBuffer().buffer(), 0).isBuffer());
            }
            BufferOrEvent boe = parseBuffer(view.getNextBuffer().buffer(), i);
            assertTrue(boe.isEvent());
            assertEquals(barrier, boe.getEvent());
        }
    } else {
        assertEquals(4, partition.getBufferPool().bestEffortGetNumOfUsedBuffers());
        ResultSubpartitionView[] views = new ResultSubpartitionView[4];
        // 1 buffer + 1 event
        assertEquals(2, partition.getNumberOfQueuedBuffers(0));
        views[0] = partition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
        assertTrue(parseBuffer(views[0].getNextBuffer().buffer(), 0).isBuffer());
        // 2 buffers + 1 event
        assertEquals(3, partition.getNumberOfQueuedBuffers(1));
        views[1] = partition.createSubpartitionView(1, new NoOpBufferAvailablityListener());
        assertTrue(parseBuffer(views[1].getNextBuffer().buffer(), 1).isBuffer());
        assertTrue(parseBuffer(views[1].getNextBuffer().buffer(), 1).isBuffer());
        // 1 buffer + 1 event
        assertEquals(2, partition.getNumberOfQueuedBuffers(2));
        views[2] = partition.createSubpartitionView(2, new NoOpBufferAvailablityListener());
        assertTrue(parseBuffer(views[2].getNextBuffer().buffer(), 2).isBuffer());
        views[3] = partition.createSubpartitionView(3, new NoOpBufferAvailablityListener());
        // 0 buffers + 1 event
        assertEquals(1, partition.getNumberOfQueuedBuffers(3));
        // every queue's last element should be the event
        for (int i = 0; i < numberOfChannels; i++) {
            BufferOrEvent boe = parseBuffer(views[i].getNextBuffer().buffer(), i);
            assertTrue(boe.isEvent());
            assertEquals(barrier, boe.getEvent());
        }
    }
}

Example 18 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class AlignedCheckpointsTest method testSingleChannelNoBarriers.

// ------------------------------------------------------------------------
// Tests
// ------------------------------------------------------------------------
/**
 * Validates that the buffer behaves correctly if no checkpoint barriers come, for a single
 * input channel.
 */
@Test
public void testSingleChannelNoBarriers() throws Exception {
    BufferOrEvent[] sequence = { createBuffer(0), createBuffer(0), createBuffer(0), createEndOfPartition(0) };
    inputGate = createCheckpointedInputGate(1, sequence);
    for (BufferOrEvent boe : sequence) {
        assertEquals(boe, inputGate.pollNext().get());
    }
    assertEquals(0L, inputGate.getAlignmentDurationNanos());
}

Example 19 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class CheckpointBarrierTrackerTest method testMultiChannelWithBarriers.

@Test
public void testMultiChannelWithBarriers() throws Exception {
    BufferOrEvent[] sequence = { createBuffer(0), createBuffer(2), createBuffer(0), createBarrier(1, 1), createBarrier(1, 2), createBuffer(2), createBuffer(1), createBarrier(1, 0), createBuffer(0), createBuffer(0), createBuffer(1), createBuffer(1), createBuffer(2), createBarrier(2, 0), createBarrier(2, 1), createBarrier(2, 2), createBuffer(2), createBuffer(2), createBarrier(3, 2), createBuffer(2), createBuffer(2), createBarrier(3, 0), createBarrier(3, 1), createBarrier(4, 1), createBarrier(4, 2), createBarrier(4, 0), createBuffer(0) };
    CheckpointSequenceValidator validator = new CheckpointSequenceValidator(1, 2, 3, 4);
    inputGate = createCheckpointedInputGate(3, sequence, validator);
    for (BufferOrEvent boe : sequence) {
        assertEquals(boe, inputGate.pollNext().get());
    }
}

Example 20 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class CheckpointBarrierTrackerTest method testTwoLastBarriersOneByOne.

@Test
public void testTwoLastBarriersOneByOne() throws Exception {
    BufferOrEvent[] sequence = { // start checkpoint 1
    createBarrier(1, 1), // start checkpoint 2
    createBarrier(2, 1), // finish the checkpoint 1
    createBarrier(1, 0), // finish the checkpoint 2
    createBarrier(2, 0) };
    ValidatingCheckpointHandler validator = new ValidatingCheckpointHandler();
    ManualClock manualClock = new ManualClock();
    inputGate = createCheckpointedInputGate(2, sequence, validator, manualClock);
    for (BufferOrEvent boe : sequence) {
        assertEquals(boe, inputGate.pollNext().get());
        manualClock.advanceTime(Duration.ofSeconds(1));
    }
    assertEquals(Duration.ofSeconds(2).toNanos(), validator.lastAlignmentDurationNanos.get().longValue());
}