Examples with BufferOrEvent org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent used on opensource projects

Example 26 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class CheckpointBarrierTrackerTest method testNoFastPathWithChannelFinishedDuringCheckpoints.

@Test
public void testNoFastPathWithChannelFinishedDuringCheckpoints() throws Exception {
    BufferOrEvent[] sequence = { createBarrier(1, 0), createEndOfPartition(0), createBarrier(1, 1) };
    ValidatingCheckpointHandler validator = new ValidatingCheckpointHandler();
    inputGate = createCheckpointedInputGate(2, sequence, validator);
    for (BufferOrEvent boe : sequence) {
        assertEquals(boe, inputGate.pollNext().get());
    }
    // The last barrier should finish the pending checkpoint instead of trigger a "new" one.
    assertEquals(1, validator.getTriggeredCheckpointCounter());
    assertFalse(inputGate.getCheckpointBarrierHandler().isCheckpointPending());
}

Example 27 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class CheckpointBarrierTrackerTest method testCompleteCheckpointsOnLateBarriers.

/**
 * This test validates that the barrier tracker does not immediately discard a pending
 * checkpoint as soon as it sees a barrier from a later checkpoint from some channel.
 *
 * <p>This behavior is crucial, otherwise topologies where different inputs have different
 * latency (and that latency is close to or higher than the checkpoint interval) may skip many
 * checkpoints, or fail to complete a checkpoint all together.
 */
@Test
public void testCompleteCheckpointsOnLateBarriers() throws Exception {
    BufferOrEvent[] sequence = { // checkpoint 2
    createBuffer(1), createBuffer(1), createBuffer(0), createBuffer(2), createBarrier(2, 1), createBarrier(2, 0), createBarrier(2, 2), // incomplete checkpoint 3
    createBuffer(1), createBuffer(0), createBarrier(3, 1), createBarrier(3, 2), // some barriers from checkpoint 4
    createBuffer(1), createBuffer(0), createBarrier(4, 2), createBarrier(4, 1), createBuffer(1), createBuffer(2), // last barrier from checkpoint 3
    createBarrier(3, 0), // complete checkpoint 4
    createBuffer(0), createBarrier(4, 0), // regular checkpoint 5
    createBuffer(1), createBuffer(2), createBarrier(5, 1), createBuffer(0), createBarrier(5, 0), createBuffer(1), createBarrier(5, 2), // checkpoint 6 (incomplete),
    createBuffer(1), createBarrier(6, 1), createBuffer(0), createBarrier(6, 0), // checkpoint 7, with early barriers for checkpoints 8 and 9
    createBuffer(1), createBarrier(7, 1), createBuffer(0), createBarrier(7, 2), createBuffer(2), createBarrier(8, 2), createBuffer(0), createBarrier(8, 1), createBuffer(1), createBarrier(9, 1), // complete checkpoint 7, first barriers from checkpoint 10
    createBarrier(7, 0), createBuffer(0), createBarrier(9, 2), createBuffer(2), createBarrier(10, 2), // complete checkpoint 8 and 9
    createBarrier(8, 0), createBuffer(1), createBuffer(2), createBarrier(9, 0), // trailing data
    createBuffer(1), createBuffer(0), createBuffer(2), // complete checkpoint 10
    createBarrier(10, 0), createBarrier(10, 1) };
    CheckpointSequenceValidator validator = new CheckpointSequenceValidator(2, 3, 4, 5, 7, 8, 9, 10);
    inputGate = createCheckpointedInputGate(3, sequence, validator);
    for (BufferOrEvent boe : sequence) {
        assertEquals(boe, inputGate.pollNext().get());
    }
}

Example 28 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class CheckpointBarrierTrackerTest method testNoFastPathWithChannelFinishedDuringCheckpointsCancel.

@Test
public void testNoFastPathWithChannelFinishedDuringCheckpointsCancel() throws Exception {
    BufferOrEvent[] sequence = { createBarrier(1, 0, 0), createEndOfPartition(0), createCancellationBarrier(1, 1) };
    ValidatingCheckpointHandler checkpointHandler = new ValidatingCheckpointHandler();
    inputGate = createCheckpointedInputGate(2, sequence, checkpointHandler);
    for (BufferOrEvent boe : sequence) {
        assertEquals(boe, inputGate.pollNext().get());
    }
    assertEquals(1, checkpointHandler.getLastCanceledCheckpointId());
    // If go with the fast path, the pending checkpoint would not be removed normally.
    assertFalse(inputGate.getCheckpointBarrierHandler().isCheckpointPending());
}

Example 29 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class AlignedCheckpointsTest method testMultiChannelNoBarriers.

/**
 * Validates that the buffer behaves correctly if no checkpoint barriers come, for an input with
 * multiple input channels.
 */
@Test
public void testMultiChannelNoBarriers() throws Exception {
    BufferOrEvent[] sequence = { createBuffer(2), createBuffer(2), createBuffer(0), createBuffer(1), createBuffer(0), createEndOfPartition(0), createBuffer(3), createBuffer(1), createEndOfPartition(3), createBuffer(1), createEndOfPartition(1), createBuffer(2), createEndOfPartition(2) };
    inputGate = createCheckpointedInputGate(4, sequence);
    for (BufferOrEvent boe : sequence) {
        assertEquals(boe, inputGate.pollNext().get());
    }
    assertEquals(0L, inputGate.getAlignmentDurationNanos());
}

Example 30 with BufferOrEvent

use of org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent in project flink by apache.

the class AlignedCheckpointsTest method testMissingCancellationBarriers.

@Test
public void testMissingCancellationBarriers() throws Exception {
    BufferOrEvent[] sequence = { createBarrier(1L, 0), createCancellationBarrier(3L, 1), createCancellationBarrier(2L, 0), createCancellationBarrier(3L, 0), createBuffer(0) };
    AbstractInvokable validator = new ValidatingCheckpointHandler();
    inputGate = createCheckpointedInputGate(2, sequence, validator);
    for (BufferOrEvent boe : sequence) {
        assertEquals(boe, inputGate.pollNext().get());
    }
    assertThat(mockInputGate.getBlockedChannels(), empty());
}