Examples with CheckpointBarrier org.apache.flink.runtime.io.network.api.CheckpointBarrier used on opensource projects

Example 26 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class StreamTaskFinalCheckpointsTest method testOperatorSkipLifeCycleIfFinishedOnRestore.

@Test
public void testOperatorSkipLifeCycleIfFinishedOnRestore() throws Exception {
    try (StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO, 3).setCollectNetworkEvents().setTaskStateSnapshot(1, TaskStateSnapshot.FINISHED_ON_RESTORE).setupOperatorChain(new TestFinishedOnRestoreStreamOperator()).chain(new TestFinishedOnRestoreStreamOperator(), StringSerializer.INSTANCE).finish().build()) {
        // Finish the restore, including state initialization and open.
        harness.processAll();
        // Try trigger a checkpoint.
        harness.getTaskStateManager().getWaitForReportLatch().reset();
        CheckpointMetaData checkpointMetaData = new CheckpointMetaData(2, 2);
        CheckpointOptions checkpointOptions = new CheckpointOptions(CheckpointType.CHECKPOINT, getDefault());
        harness.streamTask.triggerCheckpointOnBarrier(checkpointMetaData, checkpointOptions, new CheckpointMetricsBuilder().setBytesProcessedDuringAlignment(0).setAlignmentDurationNanos(0));
        harness.getTaskStateManager().getWaitForReportLatch().await();
        assertEquals(2, harness.getTaskStateManager().getReportedCheckpointId());
        // Checkpoint notification.
        harness.streamTask.notifyCheckpointCompleteAsync(2);
        harness.streamTask.notifyCheckpointAbortAsync(3, 2);
        harness.processAll();
        // Finish & close operators.
        harness.processElement(Watermark.MAX_WATERMARK, 0, 0);
        harness.processElement(Watermark.MAX_WATERMARK, 0, 1);
        harness.processElement(Watermark.MAX_WATERMARK, 0, 2);
        harness.waitForTaskCompletion();
        harness.finishProcessing();
        assertThat(harness.getOutput(), contains(new CheckpointBarrier(checkpointMetaData.getCheckpointId(), checkpointMetaData.getTimestamp(), checkpointOptions), Watermark.MAX_WATERMARK, new EndOfData(StopMode.DRAIN)));
    }
}

Example 27 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class SourceOperatorStreamTaskTest method executeAndWaitForCheckpoint.

private TaskStateSnapshot executeAndWaitForCheckpoint(long checkpointId, TaskStateSnapshot initialSnapshot, IntStream expectedRecords) throws Exception {
    try (StreamTaskMailboxTestHarness<Integer> testHarness = createTestHarness(checkpointId, initialSnapshot)) {
        // Add records to the splits.
        MockSourceSplit split = getAndMaybeAssignSplit(testHarness);
        // Add records to the split and update expected output.
        addRecords(split, NUM_RECORDS);
        // Process all the records.
        testHarness.processAll();
        CheckpointOptions checkpointOptions = CheckpointOptions.forCheckpointWithDefaultLocation();
        triggerCheckpointWaitForFinish(testHarness, checkpointId, checkpointOptions);
        // Build expected output to verify the results
        Queue<Object> expectedOutput = new LinkedList<>();
        expectedRecords.forEach(r -> expectedOutput.offer(new StreamRecord<>(r, TimestampAssigner.NO_TIMESTAMP)));
        // Add barrier to the expected output.
        expectedOutput.add(new CheckpointBarrier(checkpointId, checkpointId, checkpointOptions));
        assertEquals(checkpointId, testHarness.taskStateManager.getReportedCheckpointId());
        assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
        return testHarness.taskStateManager.getLastJobManagerTaskStateSnapshot();
    }
}

Example 28 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class SourceOperatorStreamTaskTest method testSnapshotAndAdvanceToEndOfEventTime.

@Test
public void testSnapshotAndAdvanceToEndOfEventTime() throws Exception {
    final int checkpointId = 1;
    try (StreamTaskMailboxTestHarness<Integer> testHarness = createTestHarness(checkpointId, null)) {
        getAndMaybeAssignSplit(testHarness);
        final CheckpointOptions checkpointOptions = new CheckpointOptions(SavepointType.terminate(SavepointFormatType.CANONICAL), CheckpointStorageLocationReference.getDefault());
        triggerCheckpointWaitForFinish(testHarness, checkpointId, checkpointOptions);
        Queue<Object> expectedOutput = new LinkedList<>();
        expectedOutput.add(Watermark.MAX_WATERMARK);
        expectedOutput.add(new EndOfData(StopMode.DRAIN));
        expectedOutput.add(new CheckpointBarrier(checkpointId, checkpointId, checkpointOptions));
        assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    }
}

Example 29 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink by apache.

the class TwoInputStreamTaskTest method testOvertakingCheckpointBarriers.

/**
 * This test verifies that checkpoint barriers and barrier buffers work correctly with
 * concurrent checkpoint barriers where one checkpoint is "overtaking" another checkpoint, i.e.
 * some inputs receive barriers from an earlier checkpoint, thereby blocking, then all inputs
 * receive barriers from a later checkpoint.
 */
@Test
public void testOvertakingCheckpointBarriers() throws Exception {
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, 2, 2, new int[] { 1, 2 }, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);
    streamConfig.setOperatorID(new OperatorID());
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<>(42, initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<>(1337, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<>("42", initialTime));
    expectedOutput.add(new StreamRecord<>("1337", initialTime));
    testHarness.waitForInputProcessing();
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Now give a later barrier to all inputs, this should unblock the first channel
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);
    expectedOutput.add(new CancelCheckpointMarker(0));
    expectedOutput.add(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()));
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // Then give the earlier barrier, these should be ignored
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);
    testHarness.waitForInputProcessing();
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 30 with CheckpointBarrier

use of org.apache.flink.runtime.io.network.api.CheckpointBarrier in project flink-mirror by flink-ci.

the class RecordWriterTest method testBroadcastEventMixedRecords.

/**
 * Tests broadcasting events when records have been emitted.
 */
@Test
public void testBroadcastEventMixedRecords() throws Exception {
    Random rand = new XORShiftRandom();
    int numberOfChannels = 4;
    int bufferSize = 32;
    // serialized length
    int lenBytes = 4;
    ResultPartition partition = createResultPartition(bufferSize, numberOfChannels);
    RecordWriter<ByteArrayIO> writer = createRecordWriter(partition);
    CheckpointBarrier barrier = new CheckpointBarrier(Integer.MAX_VALUE + 1292L, Integer.MAX_VALUE + 199L, CheckpointOptions.forCheckpointWithDefaultLocation());
    // Emit records on some channels first (requesting buffers), then
    // broadcast the event. The record buffers should be emitted first, then
    // the event. After the event, no new buffer should be requested.
    // (i) Smaller than the buffer size
    byte[] bytes = new byte[bufferSize / 2];
    rand.nextBytes(bytes);
    writer.emit(new ByteArrayIO(bytes));
    // (ii) Larger than the buffer size
    bytes = new byte[bufferSize + 1];
    rand.nextBytes(bytes);
    writer.emit(new ByteArrayIO(bytes));
    // (iii) Exactly the buffer size
    bytes = new byte[bufferSize - lenBytes];
    rand.nextBytes(bytes);
    writer.emit(new ByteArrayIO(bytes));
    // (iv) Broadcast the event
    writer.broadcastEvent(barrier);
    if (isBroadcastWriter) {
        assertEquals(3, partition.getBufferPool().bestEffortGetNumOfUsedBuffers());
        for (int i = 0; i < numberOfChannels; i++) {
            // 3 buffer + 1 event
            assertEquals(4, partition.getNumberOfQueuedBuffers(i));
            ResultSubpartitionView view = partition.createSubpartitionView(i, new NoOpBufferAvailablityListener());
            for (int j = 0; j < 3; j++) {
                assertTrue(parseBuffer(view.getNextBuffer().buffer(), 0).isBuffer());
            }
            BufferOrEvent boe = parseBuffer(view.getNextBuffer().buffer(), i);
            assertTrue(boe.isEvent());
            assertEquals(barrier, boe.getEvent());
        }
    } else {
        assertEquals(4, partition.getBufferPool().bestEffortGetNumOfUsedBuffers());
        ResultSubpartitionView[] views = new ResultSubpartitionView[4];
        // 1 buffer + 1 event
        assertEquals(2, partition.getNumberOfQueuedBuffers(0));
        views[0] = partition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
        assertTrue(parseBuffer(views[0].getNextBuffer().buffer(), 0).isBuffer());
        // 2 buffers + 1 event
        assertEquals(3, partition.getNumberOfQueuedBuffers(1));
        views[1] = partition.createSubpartitionView(1, new NoOpBufferAvailablityListener());
        assertTrue(parseBuffer(views[1].getNextBuffer().buffer(), 1).isBuffer());
        assertTrue(parseBuffer(views[1].getNextBuffer().buffer(), 1).isBuffer());
        // 1 buffer + 1 event
        assertEquals(2, partition.getNumberOfQueuedBuffers(2));
        views[2] = partition.createSubpartitionView(2, new NoOpBufferAvailablityListener());
        assertTrue(parseBuffer(views[2].getNextBuffer().buffer(), 2).isBuffer());
        views[3] = partition.createSubpartitionView(3, new NoOpBufferAvailablityListener());
        // 0 buffers + 1 event
        assertEquals(1, partition.getNumberOfQueuedBuffers(3));
        // every queue's last element should be the event
        for (int i = 0; i < numberOfChannels; i++) {
            BufferOrEvent boe = parseBuffer(views[i].getNextBuffer().buffer(), i);
            assertTrue(boe.isEvent());
            assertEquals(barrier, boe.getEvent());
        }
    }
}