Examples with BufferOrEvent org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent used on opensource projects

Example 46 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferTest method testAbortWhileHavingQueuedBarriers.

/**
	 * This tests the where a replay of queued checkpoint barriers meets
	 * a canceled checkpoint.
	 *
	 * The replayed newer checkpoint barrier must not try to cancel the
	 * already canceled checkpoint.
	 */
@Test
public void testAbortWhileHavingQueuedBarriers() throws Exception {
    BufferOrEvent[] sequence = { /*  0 */
    createBuffer(1), /*  1 */
    createBarrier(1, 1), /*  2 */
    createBuffer(2), createBuffer(0), createBuffer(1), /*  5 */
    createBarrier(2, 1), /*  6 */
    createBuffer(2), createBuffer(1), /*  8 */
    createCancellationBarrier(1, 0), /*  9 */
    createBuffer(2), createBuffer(1), createBuffer(0), /* 12 */
    createBarrier(1, 2), /* 13 */
    createBuffer(0), createBuffer(1), createBuffer(2), /* 16 */
    createBarrier(2, 0), createBarrier(2, 2), /* 18 */
    createBuffer(0), createBuffer(1), createBuffer(2) };
    MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
    BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER);
    StatefulTask toNotify = mock(StatefulTask.class);
    buffer.registerCheckpointEventHandler(toNotify);
    long startTs;
    check(sequence[0], buffer.getNextNonBlocked());
    // starting first checkpoint
    startTs = System.nanoTime();
    check(sequence[2], buffer.getNextNonBlocked());
    check(sequence[3], buffer.getNextNonBlocked());
    check(sequence[6], buffer.getNextNonBlocked());
    // cancelled by cancellation barrier
    check(sequence[4], buffer.getNextNonBlocked());
    validateAlignmentTime(startTs, buffer);
    verify(toNotify).abortCheckpointOnBarrier(eq(1L), any(CheckpointDeclineOnCancellationBarrierException.class));
    // the next checkpoint alignment starts now
    startTs = System.nanoTime();
    check(sequence[9], buffer.getNextNonBlocked());
    check(sequence[11], buffer.getNextNonBlocked());
    check(sequence[13], buffer.getNextNonBlocked());
    check(sequence[15], buffer.getNextNonBlocked());
    // checkpoint done
    check(sequence[7], buffer.getNextNonBlocked());
    validateAlignmentTime(startTs, buffer);
    verify(toNotify).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(2L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    // queued data
    check(sequence[10], buffer.getNextNonBlocked());
    check(sequence[14], buffer.getNextNonBlocked());
    // trailing data
    check(sequence[18], buffer.getNextNonBlocked());
    check(sequence[19], buffer.getNextNonBlocked());
    check(sequence[20], buffer.getNextNonBlocked());
    // all done
    assertNull(buffer.getNextNonBlocked());
    assertNull(buffer.getNextNonBlocked());
    buffer.cleanup();
    checkNoTempFilesRemain();
    // check overall notifications
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(any(CheckpointMetaData.class), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    verify(toNotify, times(1)).abortCheckpointOnBarrier(anyLong(), any(Throwable.class));
}

Example 47 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierTrackerTest method testCompleteCheckpointsOnLateBarriers.

/**
	 * This test validates that the barrier tracker does not immediately
	 * discard a pending checkpoint as soon as it sees a barrier from a
	 * later checkpoint from some channel.
	 * 
	 * This behavior is crucial, otherwise topologies where different inputs
	 * have different latency (and that latency is close to or higher than the
	 * checkpoint interval) may skip many checkpoints, or fail to complete a
	 * checkpoint all together.
	 */
@Test
public void testCompleteCheckpointsOnLateBarriers() {
    try {
        BufferOrEvent[] sequence = { // checkpoint 2
        createBuffer(1), createBuffer(1), createBuffer(0), createBuffer(2), createBarrier(2, 1), createBarrier(2, 0), createBarrier(2, 2), // incomplete checkpoint 3
        createBuffer(1), createBuffer(0), createBarrier(3, 1), createBarrier(3, 2), // some barriers from checkpoint 4
        createBuffer(1), createBuffer(0), createBarrier(4, 2), createBarrier(4, 1), createBuffer(1), createBuffer(2), // last barrier from checkpoint 3
        createBarrier(3, 0), // complete checkpoint 4
        createBuffer(0), createBarrier(4, 0), // regular checkpoint 5
        createBuffer(1), createBuffer(2), createBarrier(5, 1), createBuffer(0), createBarrier(5, 0), createBuffer(1), createBarrier(5, 2), // checkpoint 6 (incomplete),
        createBuffer(1), createBarrier(6, 1), createBuffer(0), createBarrier(6, 0), // checkpoint 7, with early barriers for checkpoints 8 and 9
        createBuffer(1), createBarrier(7, 1), createBuffer(0), createBarrier(7, 2), createBuffer(2), createBarrier(8, 2), createBuffer(0), createBarrier(8, 1), createBuffer(1), createBarrier(9, 1), // complete checkpoint 7, first barriers from checkpoint 10
        createBarrier(7, 0), createBuffer(0), createBarrier(9, 2), createBuffer(2), createBarrier(10, 2), // complete checkpoint 8 and 9
        createBarrier(8, 0), createBuffer(1), createBuffer(2), createBarrier(9, 0), // trailing data
        createBuffer(1), createBuffer(0), createBuffer(2) };
        MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
        BarrierTracker tracker = new BarrierTracker(gate);
        CheckpointSequenceValidator validator = new CheckpointSequenceValidator(2, 3, 4, 5, 7, 8, 9);
        tracker.registerCheckpointEventHandler(validator);
        for (BufferOrEvent boe : sequence) {
            if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) {
                assertEquals(boe, tracker.getNextNonBlocked());
            }
        }
        assertNull(tracker.getNextNonBlocked());
        assertNull(tracker.getNextNonBlocked());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 48 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierTrackerTest method testSingleChannelWithBarriers.

@Test
public void testSingleChannelWithBarriers() {
    try {
        BufferOrEvent[] sequence = { createBuffer(0), createBuffer(0), createBuffer(0), createBarrier(1, 0), createBuffer(0), createBuffer(0), createBuffer(0), createBuffer(0), createBarrier(2, 0), createBarrier(3, 0), createBuffer(0), createBuffer(0), createBarrier(4, 0), createBarrier(5, 0), createBarrier(6, 0), createBuffer(0) };
        MockInputGate gate = new MockInputGate(PAGE_SIZE, 1, Arrays.asList(sequence));
        BarrierTracker tracker = new BarrierTracker(gate);
        CheckpointSequenceValidator validator = new CheckpointSequenceValidator(1, 2, 3, 4, 5, 6);
        tracker.registerCheckpointEventHandler(validator);
        for (BufferOrEvent boe : sequence) {
            if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) {
                assertEquals(boe, tracker.getNextNonBlocked());
            }
        }
        assertNull(tracker.getNextNonBlocked());
        assertNull(tracker.getNextNonBlocked());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 49 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierTrackerTest method testSingleChannelAbortCheckpoint.

@Test
public void testSingleChannelAbortCheckpoint() throws Exception {
    BufferOrEvent[] sequence = { createBuffer(0), createBarrier(1, 0), createBuffer(0), createBarrier(2, 0), createCancellationBarrier(4, 0), createBarrier(5, 0), createBuffer(0), createCancellationBarrier(6, 0), createBuffer(0) };
    MockInputGate gate = new MockInputGate(PAGE_SIZE, 1, Arrays.asList(sequence));
    BarrierTracker tracker = new BarrierTracker(gate);
    // negative values mean an expected cancellation call!
    CheckpointSequenceValidator validator = new CheckpointSequenceValidator(1, 2, -4, 5, -6);
    tracker.registerCheckpointEventHandler(validator);
    for (BufferOrEvent boe : sequence) {
        if (boe.isBuffer()) {
            assertEquals(boe, tracker.getNextNonBlocked());
        }
        assertTrue(tracker.isEmpty());
    }
    assertNull(tracker.getNextNonBlocked());
    assertNull(tracker.getNextNonBlocked());
}

Example 50 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierTrackerTest method testSingleChannelNoBarriers.

@Test
public void testSingleChannelNoBarriers() {
    try {
        BufferOrEvent[] sequence = { createBuffer(0), createBuffer(0), createBuffer(0) };
        MockInputGate gate = new MockInputGate(PAGE_SIZE, 1, Arrays.asList(sequence));
        BarrierTracker tracker = new BarrierTracker(gate);
        for (BufferOrEvent boe : sequence) {
            assertEquals(boe, tracker.getNextNonBlocked());
        }
        assertNull(tracker.getNextNonBlocked());
        assertNull(tracker.getNextNonBlocked());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}