Examples with CheckpointBarrier org.apache.flink.runtime.io.network.api.CheckpointBarrier used on opensource projects

Example 51 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method testSourceCheckpointLast.

/**
 * In this scenario: 1a. network inputs are processed until CheckpointBarriers are processed 1b.
 * source records are processed at the same time 2. checkpoint is triggered via RPC 3. aligned
 * checkpoint is performed
 */
@Test
public void testSourceCheckpointLast() throws Exception {
    try (StreamTaskMailboxTestHarness<String> testHarness = buildTestHarness(objectReuse)) {
        testHarness.setAutoProcess(false);
        ArrayDeque<Object> expectedOutput = new ArrayDeque<>();
        CheckpointBarrier barrier = createBarrier(testHarness);
        addRecordsAndBarriers(testHarness, barrier);
        testHarness.processAll();
        Future<Boolean> checkpointFuture = testHarness.getStreamTask().triggerCheckpointAsync(metaData, barrier.getCheckpointOptions());
        processSingleStepUntil(testHarness, checkpointFuture::isDone);
        expectedOutput.add(new StreamRecord<>("42", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("42", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("42", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("44", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("44", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("47.0", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("47.0", TimestampAssigner.NO_TIMESTAMP));
        ArrayList<Object> actualOutput = new ArrayList<>(testHarness.getOutput());
        assertThat(actualOutput.subList(0, expectedOutput.size()), containsInAnyOrder(expectedOutput.toArray()));
        assertThat(actualOutput.get(expectedOutput.size()), equalTo(barrier));
    }
}

Example 52 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method testSourceCheckpointLastUnaligned.

/**
 * In this scenario: 1. network inputs are processed until CheckpointBarriers are processed 2.
 * there are no source records to be processed 3. checkpoint is triggered on first received
 * CheckpointBarrier 4. unaligned checkpoint is performed at some point of time blocking the
 * source 5. more source records are added, that shouldn't be processed
 */
@Test
public void testSourceCheckpointLastUnaligned() throws Exception {
    boolean unaligned = true;
    try (StreamTaskMailboxTestHarness<String> testHarness = buildTestHarness(unaligned, objectReuse)) {
        testHarness.setAutoProcess(false);
        ArrayDeque<Object> expectedOutput = new ArrayDeque<>();
        addNetworkRecords(testHarness);
        CheckpointBarrier barrier = createBarrier(testHarness);
        addBarriers(testHarness, barrier);
        testHarness.processAll();
        addSourceRecords(testHarness, 1, 1337, 1337, 1337);
        testHarness.processAll();
        expectedOutput.add(new StreamRecord<>("44", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("44", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("47.0", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("47.0", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(barrier);
        assertThat(testHarness.getOutput(), containsInAnyOrder(expectedOutput.toArray()));
    }
}

Example 53 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method testOnlyOneSource.

@Test
public void testOnlyOneSource() throws Exception {
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).modifyExecutionConfig(applyObjectReuse(objectReuse)).addSourceInput(new SourceOperatorFactory<>(new MockSource(Boundedness.BOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).setupOutputForSingletonOperatorChain(new MapToStringMultipleInputOperatorFactory(1)).build()) {
        testHarness.setAutoProcess(false);
        ArrayDeque<Object> expectedOutput = new ArrayDeque<>();
        addSourceRecords(testHarness, 0, 42, 43, 44);
        processSingleStepUntil(testHarness, () -> !testHarness.getOutput().isEmpty());
        expectedOutput.add(new StreamRecord<>("42", TimestampAssigner.NO_TIMESTAMP));
        CheckpointBarrier barrier = createBarrier(testHarness);
        Future<Boolean> checkpointFuture = testHarness.getStreamTask().triggerCheckpointAsync(metaData, barrier.getCheckpointOptions());
        processSingleStepUntil(testHarness, checkpointFuture::isDone);
        ArrayList<Object> actualOutput = new ArrayList<>(testHarness.getOutput());
        assertThat(actualOutput.subList(0, expectedOutput.size()), containsInAnyOrder(expectedOutput.toArray()));
        assertThat(actualOutput.get(expectedOutput.size()), equalTo(barrier));
    }
}

Example 54 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method testStopWithSavepointDrainWaitsForSourcesFinish.

/**
 * In this scenario:
 *
 * <ul>
 *   <li>Network inputs are processed until CheckpointBarriers for synchronous savepoint are
 *       processed.
 *   <li>RPC for stop-with-savepoint comes for sources
 *   <li>Sources keep being invoked until they return END_OF_DATA
 *   <li>Synchronous savepoint is triggered
 * </ul>
 */
@Test
public void testStopWithSavepointDrainWaitsForSourcesFinish() throws Exception {
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).setCollectNetworkEvents().modifyExecutionConfig(applyObjectReuse(objectReuse)).modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).addInput(BasicTypeInfo.STRING_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.DOUBLE_TYPE_INFO).setupOutputForSingletonOperatorChain(new MapToStringMultipleInputOperatorFactory(4, true)).build()) {
        testHarness.setAutoProcess(false);
        ArrayDeque<Object> expectedOutput = new ArrayDeque<>();
        CheckpointBarrier barrier = createStopWithSavepointDrainBarrier();
        testHarness.processElement(new StreamRecord<>("44", TimestampAssigner.NO_TIMESTAMP), 0);
        testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0);
        testHarness.processEvent(barrier, 0);
        testHarness.processElement(new StreamRecord<>(47d, TimestampAssigner.NO_TIMESTAMP), 1);
        testHarness.processEvent(new EndOfData(StopMode.DRAIN), 1);
        testHarness.processEvent(barrier, 1);
        addSourceRecords(testHarness, 1, Boundedness.CONTINUOUS_UNBOUNDED, 1, 2);
        addSourceRecords(testHarness, 2, Boundedness.CONTINUOUS_UNBOUNDED, 3, 4);
        testHarness.processAll();
        Future<Boolean> checkpointFuture = testHarness.getStreamTask().triggerCheckpointAsync(metaData, barrier.getCheckpointOptions());
        processSingleStepUntil(testHarness, checkpointFuture::isDone);
        expectedOutput.add(new StreamRecord<>("3", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("47.0", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("44", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("1", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("4", TimestampAssigner.NO_TIMESTAMP));
        expectedOutput.add(new StreamRecord<>("2", TimestampAssigner.NO_TIMESTAMP));
        ArrayList<Object> actualOutput = new ArrayList<>(testHarness.getOutput());
        assertThat(actualOutput.subList(0, expectedOutput.size()), containsInAnyOrder(expectedOutput.toArray()));
        assertThat(actualOutput.subList(actualOutput.size() - 3, actualOutput.size()), contains(new StreamRecord<>("FINISH"), new EndOfData(StopMode.DRAIN), barrier));
    }
}

Example 55 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method createBarrier.

private CheckpointBarrier createBarrier(StreamTaskMailboxTestHarness<String> testHarness) {
    StreamConfig config = testHarness.getStreamTask().getConfiguration();
    CheckpointOptions checkpointOptions = CheckpointOptions.forConfig(CheckpointType.CHECKPOINT, CheckpointStorageLocationReference.getDefault(), config.isExactlyOnceCheckpointMode(), config.isUnalignedCheckpointsEnabled(), config.getAlignedCheckpointTimeout().toMillis());
    return new CheckpointBarrier(metaData.getCheckpointId(), metaData.getTimestamp(), checkpointOptions);
}