Examples with BufferOrEvent org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent used on opensource projects

Example 6 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierBufferTest method testSingleChannelAbortCheckpoint.

@Test
public void testSingleChannelAbortCheckpoint() throws Exception {
    BufferOrEvent[] sequence = { createBuffer(0), createBarrier(1, 0), createBuffer(0), createBarrier(2, 0), createCancellationBarrier(4, 0), createBarrier(5, 0), createBuffer(0), createCancellationBarrier(6, 0), createBuffer(0) };
    MockInputGate gate = new MockInputGate(PAGE_SIZE, 1, Arrays.asList(sequence));
    BarrierBuffer buffer = new BarrierBuffer(gate, IO_MANAGER);
    StatefulTask toNotify = mock(StatefulTask.class);
    buffer.registerCheckpointEventHandler(toNotify);
    check(sequence[0], buffer.getNextNonBlocked());
    check(sequence[2], buffer.getNextNonBlocked());
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(1L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    assertEquals(0L, buffer.getAlignmentDurationNanos());
    check(sequence[6], buffer.getNextNonBlocked());
    assertEquals(5L, buffer.getCurrentCheckpointId());
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(2L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(4L), any(CheckpointDeclineOnCancellationBarrierException.class));
    verify(toNotify, times(1)).triggerCheckpointOnBarrier(argThat(new CheckpointMatcher(5L)), any(CheckpointOptions.class), any(CheckpointMetrics.class));
    assertEquals(0L, buffer.getAlignmentDurationNanos());
    check(sequence[8], buffer.getNextNonBlocked());
    assertEquals(6L, buffer.getCurrentCheckpointId());
    verify(toNotify, times(1)).abortCheckpointOnBarrier(eq(6L), any(CheckpointDeclineOnCancellationBarrierException.class));
    assertEquals(0L, buffer.getAlignmentDurationNanos());
    buffer.cleanup();
    checkNoTempFilesRemain();
}

Example 7 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierTrackerTest method testMultiChannelAbortCheckpoint.

@Test
public void testMultiChannelAbortCheckpoint() throws Exception {
    BufferOrEvent[] sequence = { // some buffers and a successful checkpoint
    createBuffer(0), createBuffer(2), createBuffer(0), createBarrier(1, 1), createBarrier(1, 2), createBuffer(2), createBuffer(1), createBarrier(1, 0), // aborted on last barrier
    createBuffer(0), createBuffer(2), createBarrier(2, 0), createBarrier(2, 2), createBuffer(0), createBuffer(2), createCancellationBarrier(2, 1), // successful checkpoint
    createBuffer(2), createBuffer(1), createBarrier(3, 1), createBarrier(3, 2), createBarrier(3, 0), // abort on first barrier
    createBuffer(0), createBuffer(1), createCancellationBarrier(4, 1), createBarrier(4, 2), createBuffer(0), createBarrier(4, 0), // another successful checkpoint
    createBuffer(0), createBuffer(1), createBuffer(2), createBarrier(5, 2), createBarrier(5, 1), createBarrier(5, 0), // abort multiple cancellations and a barrier after the cancellations 
    createBuffer(0), createBuffer(1), createCancellationBarrier(6, 1), createCancellationBarrier(6, 2), createBarrier(6, 0), createBuffer(0) };
    MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
    BarrierTracker tracker = new BarrierTracker(gate);
    // negative values mean an expected cancellation call!
    CheckpointSequenceValidator validator = new CheckpointSequenceValidator(1, -2, 3, -4, 5, -6);
    tracker.registerCheckpointEventHandler(validator);
    for (BufferOrEvent boe : sequence) {
        if (boe.isBuffer()) {
            assertEquals(boe, tracker.getNextNonBlocked());
        }
    }
    assertTrue(tracker.isEmpty());
    assertNull(tracker.getNextNonBlocked());
    assertNull(tracker.getNextNonBlocked());
    assertTrue(tracker.isEmpty());
}

Example 8 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierTrackerTest method testInterleavedCancellationBarriers.

/**
	 * Tests that each checkpoint is only aborted once in case of an interleaved cancellation
	 * barrier arrival of two consecutive checkpoints.
	 */
@Test
public void testInterleavedCancellationBarriers() throws Exception {
    BufferOrEvent[] sequence = { createBarrier(1L, 0), createCancellationBarrier(2L, 0), createCancellationBarrier(1L, 1), createCancellationBarrier(2L, 1), createCancellationBarrier(1L, 2), createCancellationBarrier(2L, 2), createBuffer(0) };
    MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
    BarrierTracker tracker = new BarrierTracker(gate);
    StatefulTask statefulTask = mock(StatefulTask.class);
    tracker.registerCheckpointEventHandler(statefulTask);
    for (BufferOrEvent boe : sequence) {
        if (boe.isBuffer() || (boe.getEvent().getClass() != CheckpointBarrier.class && boe.getEvent().getClass() != CancelCheckpointMarker.class)) {
            assertEquals(boe, tracker.getNextNonBlocked());
        }
    }
    verify(statefulTask, times(1)).abortCheckpointOnBarrier(eq(1L), any(Throwable.class));
    verify(statefulTask, times(1)).abortCheckpointOnBarrier(eq(2L), any(Throwable.class));
}

Example 9 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierTrackerTest method testMultiChannelSkippingCheckpoints.

@Test
public void testMultiChannelSkippingCheckpoints() {
    try {
        BufferOrEvent[] sequence = { createBuffer(0), createBuffer(2), createBuffer(0), createBarrier(1, 1), createBarrier(1, 2), createBuffer(2), createBuffer(1), createBarrier(1, 0), createBuffer(0), createBuffer(0), createBuffer(1), createBuffer(1), createBuffer(2), createBarrier(2, 0), createBarrier(2, 1), createBarrier(2, 2), createBuffer(2), createBuffer(2), createBarrier(3, 2), createBuffer(2), createBuffer(2), // jump to checkpoint 4
        createBarrier(4, 0), createBuffer(0), createBuffer(1), createBuffer(2), createBarrier(4, 1), createBuffer(1), createBarrier(4, 2), createBuffer(0) };
        MockInputGate gate = new MockInputGate(PAGE_SIZE, 3, Arrays.asList(sequence));
        BarrierTracker tracker = new BarrierTracker(gate);
        CheckpointSequenceValidator validator = new CheckpointSequenceValidator(1, 2, 4);
        tracker.registerCheckpointEventHandler(validator);
        for (BufferOrEvent boe : sequence) {
            if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) {
                assertEquals(boe, tracker.getNextNonBlocked());
            }
        }
        assertNull(tracker.getNextNonBlocked());
        assertNull(tracker.getNextNonBlocked());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 10 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class BarrierTrackerTest method testSingleChannelWithSkippedBarriers.

@Test
public void testSingleChannelWithSkippedBarriers() {
    try {
        BufferOrEvent[] sequence = { createBuffer(0), createBarrier(1, 0), createBuffer(0), createBuffer(0), createBarrier(3, 0), createBuffer(0), createBarrier(4, 0), createBarrier(6, 0), createBuffer(0), createBarrier(7, 0), createBuffer(0), createBarrier(10, 0), createBuffer(0) };
        MockInputGate gate = new MockInputGate(PAGE_SIZE, 1, Arrays.asList(sequence));
        BarrierTracker tracker = new BarrierTracker(gate);
        CheckpointSequenceValidator validator = new CheckpointSequenceValidator(1, 3, 4, 6, 7, 10);
        tracker.registerCheckpointEventHandler(validator);
        for (BufferOrEvent boe : sequence) {
            if (boe.isBuffer() || boe.getEvent().getClass() != CheckpointBarrier.class) {
                assertEquals(boe, tracker.getNextNonBlocked());
            }
        }
        assertNull(tracker.getNextNonBlocked());
        assertNull(tracker.getNextNonBlocked());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}