Examples with CheckpointBarrier org.apache.flink.runtime.io.network.api.CheckpointBarrier used on opensource projects

Example 46 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class AlternatingCheckpointsTest method testSwitchToUnalignedByUpstream.

/**
 * If a checkpoint announcement was processed from one channel and then UC-barrier arrives on
 * another channel, this UC barrier should be processed by the UC controller.
 */
@Test
public void testSwitchToUnalignedByUpstream() throws Exception {
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(2, getTestBarrierHandlerFactory(target)).build()) {
        CheckpointBarrier aligned = new CheckpointBarrier(1, clock.relativeTimeMillis(), alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), Integer.MAX_VALUE));
        send(toBuffer(new EventAnnouncement(aligned, 0), true), 0, // process announcement but not the barrier
        gate);
        assertEquals(0, target.triggeredCheckpointCounter);
        send(toBuffer(aligned.asUnaligned(), true), 1, // pretend it came from upstream before the first (AC) barrier was picked
        gate);
        // up
        assertEquals(1, target.triggeredCheckpointCounter);
    }
}

Example 47 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class AlternatingCheckpointsTest method testChannelResetOnNewBarrier.

/**
 * Upon subsuming (or canceling) a checkpoint, channels should be notified regardless of whether
 * UC controller is currently being used or not. Otherwise, channels may not capture in-flight
 * buffers.
 */
@Test
public void testChannelResetOnNewBarrier() throws Exception {
    RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    try (CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(2, getTestBarrierHandlerFactory(new ValidatingCheckpointHandler())).withChannelStateWriter(stateWriter).withRemoteChannels().withMailboxExecutor().build()) {
        sendBarrier(0, clock.relativeTimeMillis(), SavepointType.savepoint(SavepointFormatType.CANONICAL), gate, // using AC because UC would require ordering in gate while polling
        0);
        ((RemoteInputChannel) gate.getChannel(0)).onBuffer(createBuffer(1024), 1, // to be captured
        0);
        send(toBuffer(new CheckpointBarrier(1, clock.relativeTimeMillis(), unaligned(CheckpointType.CHECKPOINT, getDefault())), true), 1, gate);
        assertFalse(stateWriter.getAddedInput().isEmpty());
    }
}

Example 48 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class AlternatingCheckpointsTest method testTimeoutAlignmentOnUnalignedCheckpoint.

/**
 * First we process aligned {@link CheckpointBarrier} and after that we receive an already
 * unaligned {@link CheckpointBarrier}, that has timed out on an upstream task.
 */
@Test
public void testTimeoutAlignmentOnUnalignedCheckpoint() throws Exception {
    ValidatingCheckpointHandler target = new ValidatingCheckpointHandler();
    RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    CheckpointedInputGate gate = new TestCheckpointedInputGateBuilder(3, getTestBarrierHandlerFactory(target)).withChannelStateWriter(channelStateWriter).withRemoteChannels().withMailboxExecutor().build();
    getChannel(gate, 0).onBuffer(withTimeout(Integer.MAX_VALUE).retainBuffer(), 0, 0);
    assertAnnouncement(gate);
    assertBarrier(gate);
    getChannel(gate, 1).onBuffer(dataBuffer(), 0, 0);
    getChannel(gate, 1).onBuffer(dataBuffer(), 1, 0);
    getChannel(gate, 1).onBuffer(toBuffer(new CheckpointBarrier(1, clock.relativeTimeMillis(), unaligned(CheckpointType.CHECKPOINT, getDefault())), true).retainBuffer(), 2, 0);
    assertBarrier(gate);
    assertEquals(2, channelStateWriter.getAddedInput().get(getChannel(gate, 1).getChannelInfo()).size());
    assertEquals(1, target.getTriggeredCheckpointCounter());
}

Example 49 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class UnalignedCheckpointsCancellationTest method test.

@Test
public void test() throws Exception {
    TestInvokable invokable = new TestInvokable();
    final SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(numChannels).setChannelFactory(InputChannelBuilder::buildLocalChannel).build();
    SingleCheckpointBarrierHandler unaligner = SingleCheckpointBarrierHandler.createUnalignedCheckpointBarrierHandler(TestSubtaskCheckpointCoordinator.INSTANCE, "test", invokable, SystemClock.getInstance(), true, inputGate);
    for (RuntimeEvent e : events) {
        if (e instanceof CancelCheckpointMarker) {
            unaligner.processCancellationBarrier((CancelCheckpointMarker) e, new InputChannelInfo(0, channel));
        } else if (e instanceof CheckpointBarrier) {
            unaligner.processBarrier((CheckpointBarrier) e, new InputChannelInfo(0, channel), false);
        } else {
            throw new IllegalArgumentException("unexpected event type: " + e);
        }
    }
    assertEquals("expectAbortCheckpoint", expectAbortCheckpoint, invokable.checkpointAborted);
    assertEquals("expectTriggerCheckpoint", expectTriggerCheckpoint, invokable.checkpointTriggered);
}

Example 50 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method testSourceCheckpointFirst.

/**
 * In this scenario: 1. checkpoint is triggered via RPC and source is blocked 2. network inputs
 * are processed until CheckpointBarriers are processed 3. aligned checkpoint is performed
 */
@Test
public void testSourceCheckpointFirst() throws Exception {
    try (StreamTaskMailboxTestHarness<String> testHarness = buildTestHarness(objectReuse)) {
        testHarness.setAutoProcess(false);
        ArrayDeque<Object> expectedOutput = new ArrayDeque<>();
        CheckpointBarrier barrier = createBarrier(testHarness);
        addRecordsAndBarriers(testHarness, barrier);
        Future<Boolean> checkpointFuture = testHarness.getStreamTask().triggerCheckpointAsync(metaData, barrier.getCheckpointOptions());
        processSingleStepUntil(testHarness, checkpointFuture::isDone);
        expectedOutput.add(new StreamRecord<>("44", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("44", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("47.0", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("47.0", TimestampAssigner.NO_TIMESTAMP));
        ArrayList<Object> actualOutput = new ArrayList<>(testHarness.getOutput());
        assertThat(actualOutput.subList(0, expectedOutput.size()), containsInAnyOrder(expectedOutput.toArray()));
        assertThat(actualOutput.get(expectedOutput.size()), equalTo(barrier));
    }
}