Examples with TestCheckpointedInputGateBuilder org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder used on opensource projects

Example 1 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentOnFirstBarrier.

/**
 * This test tries to make sure that the first time out happens after processing {@link
 * EventAnnouncement} but before/during processing the first {@link CheckpointBarrier}.
 */
@Test
public void testTimeoutAlignmentOnFirstBarrier() throws Exception {
    int numChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build();
    long alignmentTimeout = 100;
    Buffer checkpointBarrier = withTimeout(alignmentTimeout);
    for (int i = 0; i < numChannels; i++) {
        (getChannel(gate, i)).onBuffer(checkpointBarrier.retainBuffer(), 0, 0);
    }
    assertEquals(0, target.getTriggeredCheckpointCounter());
    for (int i = 0; i < numChannels; i++) {
        assertAnnouncement(gate);
    }
    assertEquals(0, target.getTriggeredCheckpointCounter());
    clock.advanceTime(alignmentTimeout * 4, TimeUnit.MILLISECONDS);
    assertBarrier(gate);
    assertEquals(1, target.getTriggeredCheckpointCounter());
}

Example 2 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testAlignedAfterTimedOut.

@Test
public void testAlignedAfterTimedOut() throws Exception {
    int numChannels = 1;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    long alignmentTimeOut = 100L;
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build()) {
        Buffer barrier1 = barrier(1, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), alignmentTimeOut));
        ((RemoteInputChannel) gate.getChannel(0)).onBuffer(barrier1.retainBuffer(), 0, 0);
        assertAnnouncement(gate);
        clock.advanceTime(alignmentTimeOut + 1, TimeUnit.MILLISECONDS);
        assertBarrier(gate);
        assertEquals(1, target.getTriggeredCheckpointCounter());
        Buffer barrier2 = barrier(2, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), alignmentTimeOut));
        ((RemoteInputChannel) gate.getChannel(0)).onBuffer(barrier2.retainBuffer(), 1, 0);
        assertAnnouncement(gate);
        assertBarrier(gate);
        assertEquals(2, target.getTriggeredCheckpointCounter());
        assertThat(target.getTriggeredCheckpointOptions(), contains(unaligned(CheckpointType.CHECKPOINT, getDefault()), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), alignmentTimeOut)));
    }
}

Example 3 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testStartNewCheckpointViaAnnouncement.

/**
 * This test verifies a special case that the checkpoint handler starts the new checkpoint via
 * received barrier announcement from the first channel, then {@link EndOfPartitionEvent} from
 * the second channel and then the barrier from the first channel. In this case we should ensure
 * the {@link SingleCheckpointBarrierHandler#markAlignmentStart(long, long)} should be called.
 * More information is available in https://issues.apache.org/jira/browse/FLINK-24068.
 */
@Test
public void testStartNewCheckpointViaAnnouncement() throws Exception {
    int numChannels = 3;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    long alignmentTimeOut = 10000L;
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build()) {
        getChannel(gate, 0).onBuffer(barrier(1, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), alignmentTimeOut)), 0, 0);
        getChannel(gate, 1).onBuffer(endOfPartition(), 0, 0);
        // The barrier announcement would start the checkpoint.
        assertAnnouncement(gate);
        // When received the EndOfPartition from channel 1 markAlignmentStart should be called.
        assertEvent(gate, EndOfPartitionEvent.class);
        assertTrue(gate.getCheckpointBarrierHandler().isDuringAlignment());
        // Received barrier from channel 0.
        assertBarrier(gate);
        // The last barrier from channel 2 finalize the checkpoint.
        getChannel(gate, 2).onBuffer(barrier(1, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), alignmentTimeOut)), 0, 0);
        assertAnnouncement(gate);
        assertBarrier(gate);
        assertThat(target.triggeredCheckpoints, contains(1L));
    }
}

Example 4 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentAfterReceivedEndOfPartition.

@Test
public void testTimeoutAlignmentAfterReceivedEndOfPartition() throws Exception {
    int numChannels = 3;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    long alignmentTimeOut = 100L;
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build()) {
        getChannel(gate, 0).onBuffer(barrier(1, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), alignmentTimeOut)), 0, 0);
        assertAnnouncement(gate);
        assertBarrier(gate);
        // Advances time but do not execute the registered callable which would turns into
        // unaligned checkpoint.
        clock.advanceTimeWithoutRunningCallables(alignmentTimeOut + 1, TimeUnit.MILLISECONDS);
        // The EndOfPartition should convert the checkpoint into unaligned.
        getChannel(gate, 1).onBuffer(dataBuffer(), 0, 0);
        getChannel(gate, 1).onBuffer(endOfPartition(), 1, 0);
        assertData(gate);
        assertEvent(gate, EndOfPartitionEvent.class);
        getChannel(gate, 2).onBuffer(dataBuffer(), 0, 0);
        getChannel(gate, 2).onBuffer(endOfPartition(), 1, 0);
        assertData(gate);
        assertEvent(gate, EndOfPartitionEvent.class);
        assertEquals(1, target.getTriggeredCheckpointCounter());
        assertThat(target.getTriggeredCheckpointOptions(), contains(unaligned(CheckpointType.CHECKPOINT, getDefault())));
    }
}

Example 5 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentOnAnnouncementForSecondCheckpoint.

/**
 * This test tries to make sure that the first time out happens after processing {@link
 * EventAnnouncement} but before/during processing the first {@link CheckpointBarrier} of at
 * least second checkpoint.
 */
@Test
public void testTimeoutAlignmentOnAnnouncementForSecondCheckpoint() throws Exception {
    int numChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build();
    long alignmentTimeout = 100;
    performFirstCheckpoint(numChannels, target, gate, alignmentTimeout);
    assertEquals(1, target.getTriggeredCheckpointCounter());
    Buffer checkpointBarrier = withTimeout(2, alignmentTimeout);
    for (int i = 0; i < numChannels; i++) {
        (getChannel(gate, i)).onBuffer(dataBuffer(), 1, 0);
        (getChannel(gate, i)).onBuffer(checkpointBarrier.retainBuffer(), 2, 0);
    }
    assertEquals(1, target.getTriggeredCheckpointCounter());
    for (int i = 0; i < numChannels; i++) {
        assertAnnouncement(gate);
    }
    assertEquals(1, target.getTriggeredCheckpointCounter());
    clock.advanceTime(alignmentTimeout * 4, TimeUnit.MILLISECONDS);
    // the barrier should overtake the data buffers
    assertBarrier(gate);
    assertEquals(2, target.getTriggeredCheckpointCounter());
}