Examples with TestCheckpointedInputGateBuilder org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder used on opensource projects

Example 6 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testActiveTimeoutAlignmentOnAnnouncement.

@Test
public void testActiveTimeoutAlignmentOnAnnouncement() throws Exception {
    int numChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build()) {
        long alignmentTimeout = 10;
        Buffer checkpointBarrier = withTimeout(alignmentTimeout);
        getChannel(gate, 0).onBuffer(dataBuffer(), 0, 0);
        getChannel(gate, 0).onBuffer(dataBuffer(), 1, 0);
        getChannel(gate, 0).onBuffer(checkpointBarrier.retainBuffer(), 2, 0);
        getChannel(gate, 1).onBuffer(dataBuffer(), 0, 0);
        getChannel(gate, 1).onBuffer(checkpointBarrier.retainBuffer(), 1, 0);
        assertEquals(0, target.getTriggeredCheckpointCounter());
        assertAnnouncement(gate);
        assertAnnouncement(gate);
        // the announcement should time out causing the barriers to overtake the data buffers
        clock.advanceTime(alignmentTimeout + 1, TimeUnit.MILLISECONDS);
        assertBarrier(gate);
        assertBarrier(gate);
        assertEquals(1, target.getTriggeredCheckpointCounter());
        assertThat(target.getTriggeredCheckpointOptions(), contains(unaligned(CheckpointType.CHECKPOINT, getDefault())));
        // Followed by overtaken buffers
        assertData(gate);
        assertData(gate);
        assertData(gate);
    }
}

Example 7 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentBeforeFirstBarrier.

@Test
public void testTimeoutAlignmentBeforeFirstBarrier() throws Exception {
    // given: Local channels.
    int numChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withTestChannels().withMailboxExecutor().build();
    long alignedCheckpointTimeout = 100;
    // when: Aligned checkpoint timeout expired before the first barrier received.
    Buffer checkpointBarrier = withTimeout(1, alignedCheckpointTimeout);
    clock.advanceTime(alignedCheckpointTimeout + 1, TimeUnit.MILLISECONDS);
    ((TestInputChannel) gate.getChannel(0)).read(checkpointBarrier.retainBuffer());
    // then: The UC is triggered as soon as the first barrier is received.
    assertBarrier(gate);
    assertEquals(1, target.getTriggeredCheckpointCounter());
}

Example 8 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testMetricsAlternation.

@Test
public void testMetricsAlternation() throws Exception {
    int numChannels = 2;
    int bufferSize = 1000;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).build();
    long startNanos = clock.relativeTimeNanos();
    long checkpoint1CreationTime = clock.relativeTimeMillis() - 10;
    sendBarrier(1, checkpoint1CreationTime, CHECKPOINT, gate, 0);
    sendData(bufferSize, 0, gate);
    sendData(bufferSize, 1, gate);
    clock.advanceTime(6, TimeUnit.MILLISECONDS);
    sendBarrier(1, checkpoint1CreationTime, CHECKPOINT, gate, 1);
    sendData(bufferSize, 0, gate);
    assertMetrics(target, gate.getCheckpointBarrierHandler(), 1L, startNanos, 6_000_000L, 10_000_000L, bufferSize * 2);
    startNanos = clock.relativeTimeNanos();
    long checkpoint2CreationTime = clock.relativeTimeMillis() - 5;
    sendBarrier(2, checkpoint2CreationTime, SavepointType.savepoint(SavepointFormatType.CANONICAL), gate, 0);
    sendData(bufferSize, 1, gate);
    assertMetrics(target, gate.getCheckpointBarrierHandler(), 2L, startNanos, 0L, 5_000_000L, bufferSize * 2);
    clock.advanceTime(5, TimeUnit.MILLISECONDS);
    sendBarrier(2, checkpoint2CreationTime, SavepointType.savepoint(SavepointFormatType.CANONICAL), gate, 1);
    sendData(bufferSize, 0, gate);
    assertMetrics(target, gate.getCheckpointBarrierHandler(), 2L, startNanos, 5_000_000L, 5_000_000L, bufferSize);
    startNanos = clock.relativeTimeNanos();
    long checkpoint3CreationTime = clock.relativeTimeMillis() - 7;
    send(barrier(3, checkpoint3CreationTime, unaligned(CheckpointType.CHECKPOINT, getDefault())), 0, gate);
    sendData(bufferSize, 0, gate);
    sendData(bufferSize, 1, gate);
    assertMetrics(target, gate.getCheckpointBarrierHandler(), 3L, startNanos, 0L, 7_000_000L, -1L);
    clock.advanceTime(10, TimeUnit.MILLISECONDS);
    send(barrier(3, checkpoint2CreationTime, unaligned(CheckpointType.CHECKPOINT, getDefault())), 1, gate);
    assertMetrics(target, gate.getCheckpointBarrierHandler(), 3L, startNanos, 10_000_000L, 7_000_000L, bufferSize * 2);
}

Example 9 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testNoActiveTimeoutAlignmentAfterAbort.

@Test
public void testNoActiveTimeoutAlignmentAfterAbort() throws Exception {
    int numberOfChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numberOfChannels, getTestBarrierHandlerFactory(target)).withTestChannels().withSyncExecutor().build();
    long alignmentTimeout = 100;
    Buffer checkpointBarrier = withTimeout(alignmentTimeout);
    send(checkpointBarrier, 0, gate);
    send(toBuffer(new CancelCheckpointMarker(1L), true), 0, gate);
    send(toBuffer(new CancelCheckpointMarker(1L), true), 1, gate);
    clock.advanceTime(alignmentTimeout + 1, TimeUnit.MILLISECONDS);
    assertThat(target.getTriggeredCheckpointOptions().size(), equalTo(0));
}

Example 10 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testAlternation.

@Test
public void testAlternation() throws Exception {
    int numBarriers = 123;
    int numChannels = 123;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).build()) {
        List<Long> barriers = new ArrayList<>();
        for (long barrier = 0; barrier < numBarriers; barrier++) {
            barriers.add(barrier);
            SnapshotType type = barrier % 2 == 0 ? CHECKPOINT : SavepointType.savepoint(SavepointFormatType.CANONICAL);
            for (int channel = 0; channel < numChannels; channel++) {
                send(barrier(barrier, clock.relativeTimeMillis(), alignedNoTimeout(type, getDefault())).retainBuffer(), channel, gate);
            }
        }
        assertEquals(barriers, target.triggeredCheckpoints);
    }
}