Examples with TestCheckpointedInputGateBuilder org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder used on opensource projects

Example 21 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testMetricsSingleChannel.

@Test
public void testMetricsSingleChannel() throws Exception {
    int numChannels = 1;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).build();
    long checkpoint1CreationTime = clock.relativeTimeMillis() - 10;
    long startNanos = clock.relativeTimeNanos();
    sendData(1000, 0, gate);
    sendBarrier(1, checkpoint1CreationTime, CHECKPOINT, gate, 0);
    sendData(1000, 0, gate);
    clock.advanceTime(6, TimeUnit.MILLISECONDS);
    assertMetrics(target, gate.getCheckpointBarrierHandler(), 1L, startNanos, 0L, 10_000_000L, 0);
    long checkpoint2CreationTime = clock.relativeTimeMillis() - 5;
    startNanos = clock.relativeTimeNanos();
    sendData(1000, 0, gate);
    sendBarrier(2, checkpoint2CreationTime, SavepointType.savepoint(SavepointFormatType.CANONICAL), gate, 0);
    sendData(1000, 0, gate);
    clock.advanceTime(5, TimeUnit.MILLISECONDS);
    assertMetrics(target, gate.getCheckpointBarrierHandler(), 2L, startNanos, 0L, 5_000_000L, 0);
}

Example 22 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentOnUnalignedCheckpoint.

/**
 * First we process aligned {@link CheckpointBarrier} and after that we receive an already
 * unaligned {@link CheckpointBarrier}, that has timed out on an upstream task.
 */
@Test
public void testTimeoutAlignmentOnUnalignedCheckpoint() throws Exception {
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(3, getTestBarrierHandlerFactory(target)).withChannelStateWriter(channelStateWriter).withRemoteChannels().withMailboxExecutor().build();
    getChannel(gate, 0).onBuffer(withTimeout(Integer.MAX_VALUE).retainBuffer(), 0, 0);
    assertAnnouncement(gate);
    assertBarrier(gate);
    getChannel(gate, 1).onBuffer(dataBuffer(), 0, 0);
    getChannel(gate, 1).onBuffer(dataBuffer(), 1, 0);
    getChannel(gate, 1).onBuffer(toBuffer(new CheckpointBarrier(1, clock.relativeTimeMillis(), unaligned(CheckpointType.CHECKPOINT, getDefault())), true).retainBuffer(), 2, 0);
    assertBarrier(gate);
    assertEquals(2, channelStateWriter.getAddedInput().get(getChannel(gate, 1).getChannelInfo()).size());
    assertEquals(1, target.getTriggeredCheckpointCounter());
}

Example 23 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentAfterProcessingBarrier.

@Test
public void testTimeoutAlignmentAfterProcessingBarrier() throws Exception {
    int numChannels = 3;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build()) {
        send(barrier(1, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), Integer.MAX_VALUE)), 2, gate);
        assertEquals(0, target.getTriggeredCheckpointCounter());
        testTimeoutBarrierOnTwoChannels(target, gate, Integer.MAX_VALUE);
    }
}

Example 24 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testActiveTimeoutAfterAnnouncementPassiveTimeout.

@Test
public void testActiveTimeoutAfterAnnouncementPassiveTimeout() throws Exception {
    int numChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build()) {
        long alignmentTimeout = 10;
        Buffer checkpointBarrier = withTimeout(alignmentTimeout);
        getChannel(gate, 0).onBuffer(dataBuffer(), 0, 0);
        getChannel(gate, 0).onBuffer(dataBuffer(), 1, 0);
        getChannel(gate, 0).onBuffer(checkpointBarrier.retainBuffer(), 2, 0);
        getChannel(gate, 1).onBuffer(dataBuffer(), 0, 0);
        getChannel(gate, 1).onBuffer(checkpointBarrier.retainBuffer(), 1, 0);
        assertEquals(0, target.getTriggeredCheckpointCounter());
        assertAnnouncement(gate);
        clock.advanceTimeWithoutRunningCallables(alignmentTimeout + 1, TimeUnit.MILLISECONDS);
        // the announcement should passively time out causing the barriers to overtake the data
        // buffers
        assertAnnouncement(gate);
        // we simulate active time out firing after the passive one
        clock.executeCallables();
        assertBarrier(gate);
        assertBarrier(gate);
        assertEquals(1, target.getTriggeredCheckpointCounter());
        assertThat(target.getTriggeredCheckpointOptions(), contains(unaligned(CheckpointType.CHECKPOINT, getDefault())));
        // Followed by overtaken buffers
        assertData(gate);
        assertData(gate);
        assertData(gate);
    }
}

Example 25 with TestCheckpointedInputGateBuilder

use of org.apache.flink.streaming.util.TestCheckpointedInputGateBuilder in project flink by apache.

the class AlternatingCheckpointsTest method testPassiveTimeoutAlignmentOnAnnouncement.

@Test
public void testPassiveTimeoutAlignmentOnAnnouncement() throws Exception {
    int numChannels = 2;
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(numChannels, getTestBarrierHandlerFactory(target)).withRemoteChannels().withMailboxExecutor().build();
    long alignmentTimeout = 100;
    Buffer checkpointBarrier = withTimeout(alignmentTimeout);
    (getChannel(gate, 0)).onBuffer(checkpointBarrier.retainBuffer(), 0, 0);
    assertEquals(0, target.getTriggeredCheckpointCounter());
    assertAnnouncement(gate);
    assertBarrier(gate);
    clock.advanceTimeWithoutRunningCallables(alignmentTimeout * 4, TimeUnit.MILLISECONDS);
    (getChannel(gate, 1)).onBuffer(checkpointBarrier.retainBuffer(), 0, 0);
    assertAnnouncement(gate);
    assertBarrier(gate);
    assertEquals(1, target.getTriggeredCheckpointCounter());
}