Examples with EndOfData org.apache.flink.runtime.io.network.api.EndOfData used on opensource projects

Example 6 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class StreamTaskFinalCheckpointsTest method testTriggeringCheckpointWithFinishedChannels.

private void testTriggeringCheckpointWithFinishedChannels(CheckpointOptions checkpointOptions) throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        try (StreamTaskMailboxTestHarness<String> testHarness = createTestHarness(partitionWriters, new CompletingCheckpointResponder(), checkpointOptions.isUnalignedCheckpoint() || checkpointOptions.isTimeoutable())) {
            int numChannels = testHarness.inputGates[0].getInputGate().getNumberOfInputChannels();
            int[] resumedCount = new int[numChannels];
            for (int i = 0; i < numChannels; ++i) {
                TestInputChannel inputChannel = (TestInputChannel) testHarness.inputGates[0].getInputGate().getChannel(i);
                inputChannel.setActionOnResumed(() -> resumedCount[inputChannel.getChannelIndex()]++);
            }
            // Tests triggering checkpoint when all the inputs are alive.
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2, checkpointOptions);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            assertArrayEquals(new int[] { 0, 0, 0 }, resumedCount);
            // Tests triggering checkpoint after some inputs have received EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 4, checkpointOptions);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(4, testHarness.getTaskStateManager().getReportedCheckpointId());
            assertArrayEquals(new int[] { 0, 0, 0 }, resumedCount);
            // Tests triggering checkpoint after received all the inputs have received
            // EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 1);
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 2);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 1);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 2);
            checkpointFuture = triggerCheckpoint(testHarness, 6, checkpointOptions);
            // Notifies the result partition that all records are processed after the
            // last checkpoint is triggered.
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            // The checkpoint 6 would be triggered successfully.
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(6, testHarness.getTaskStateManager().getReportedCheckpointId());
            assertArrayEquals(new int[] { 0, 0, 0 }, resumedCount);
            // Each result partition should have emitted 3 barriers and 1 EndOfUserRecordsEvent.
            for (ResultPartition resultPartition : partitionWriters) {
                assertEquals(4, resultPartition.getNumberOfQueuedBuffers());
            }
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}

Example 7 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class StreamTaskFinalCheckpointsTest method testOperatorSkipLifeCycleIfFinishedOnRestore.

@Test
public void testOperatorSkipLifeCycleIfFinishedOnRestore() throws Exception {
    try (StreamTaskMailboxTestHarness<String> harness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO, 3).setCollectNetworkEvents().setTaskStateSnapshot(1, TaskStateSnapshot.FINISHED_ON_RESTORE).setupOperatorChain(new TestFinishedOnRestoreStreamOperator()).chain(new TestFinishedOnRestoreStreamOperator(), StringSerializer.INSTANCE).finish().build()) {
        // Finish the restore, including state initialization and open.
        harness.processAll();
        // Try trigger a checkpoint.
        harness.getTaskStateManager().getWaitForReportLatch().reset();
        CheckpointMetaData checkpointMetaData = new CheckpointMetaData(2, 2);
        CheckpointOptions checkpointOptions = new CheckpointOptions(CheckpointType.CHECKPOINT, getDefault());
        harness.streamTask.triggerCheckpointOnBarrier(checkpointMetaData, checkpointOptions, new CheckpointMetricsBuilder().setBytesProcessedDuringAlignment(0).setAlignmentDurationNanos(0));
        harness.getTaskStateManager().getWaitForReportLatch().await();
        assertEquals(2, harness.getTaskStateManager().getReportedCheckpointId());
        // Checkpoint notification.
        harness.streamTask.notifyCheckpointCompleteAsync(2);
        harness.streamTask.notifyCheckpointAbortAsync(3, 2);
        harness.processAll();
        // Finish & close operators.
        harness.processElement(Watermark.MAX_WATERMARK, 0, 0);
        harness.processElement(Watermark.MAX_WATERMARK, 0, 1);
        harness.processElement(Watermark.MAX_WATERMARK, 0, 2);
        harness.waitForTaskCompletion();
        harness.finishProcessing();
        assertThat(harness.getOutput(), contains(new CheckpointBarrier(checkpointMetaData.getCheckpointId(), checkpointMetaData.getTimestamp(), checkpointOptions), Watermark.MAX_WATERMARK, new EndOfData(StopMode.DRAIN)));
    }
}

Example 8 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class SourceOperatorStreamTaskTest method testSnapshotAndAdvanceToEndOfEventTime.

@Test
public void testSnapshotAndAdvanceToEndOfEventTime() throws Exception {
    final int checkpointId = 1;
    try (StreamTaskMailboxTestHarness<Integer> testHarness = createTestHarness(checkpointId, null)) {
        getAndMaybeAssignSplit(testHarness);
        final CheckpointOptions checkpointOptions = new CheckpointOptions(SavepointType.terminate(SavepointFormatType.CANONICAL), CheckpointStorageLocationReference.getDefault());
        triggerCheckpointWaitForFinish(testHarness, checkpointId, checkpointOptions);
        Queue<Object> expectedOutput = new LinkedList<>();
        expectedOutput.add(Watermark.MAX_WATERMARK);
        expectedOutput.add(new EndOfData(StopMode.DRAIN));
        expectedOutput.add(new CheckpointBarrier(checkpointId, checkpointId, checkpointOptions));
        assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    }
}

Example 9 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class SourceOperatorStreamTaskTest method testSkipExecutionIfFinishedOnRestore.

@Test
public void testSkipExecutionIfFinishedOnRestore() throws Exception {
    TaskStateSnapshot taskStateSnapshot = TaskStateSnapshot.FINISHED_ON_RESTORE;
    LifeCycleMonitorSource testingSource = new LifeCycleMonitorSource(Boundedness.CONTINUOUS_UNBOUNDED, 10);
    SourceOperatorFactory<Integer> sourceOperatorFactory = new SourceOperatorFactory<>(testingSource, WatermarkStrategy.noWatermarks());
    List<Object> output = new ArrayList<>();
    try (StreamTaskMailboxTestHarness<Integer> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(SourceOperatorStreamTask::new, BasicTypeInfo.INT_TYPE_INFO).setTaskStateSnapshot(1, taskStateSnapshot).addAdditionalOutput(new RecordOrEventCollectingResultPartitionWriter<StreamElement>(output, new StreamElementSerializer<>(IntSerializer.INSTANCE)) {

        @Override
        public void notifyEndOfData(StopMode mode) throws IOException {
            broadcastEvent(new EndOfData(mode), false);
        }
    }).setupOperatorChain(sourceOperatorFactory).chain(new TestFinishedOnRestoreStreamOperator(), StringSerializer.INSTANCE).finish().build()) {
        testHarness.getStreamTask().invoke();
        testHarness.processAll();
        assertThat(output, contains(Watermark.MAX_WATERMARK, new EndOfData(StopMode.DRAIN)));
        LifeCycleMonitorSourceReader sourceReader = (LifeCycleMonitorSourceReader) ((SourceOperator<?, ?>) testHarness.getStreamTask().getMainOperator()).getSourceReader();
        sourceReader.getLifeCycleMonitor().assertCallTimes(0, LifeCyclePhase.values());
    }
}

Example 10 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class StreamTaskFinalCheckpointsTest method testWaitingForFinalCheckpoint.

@Test
public void testWaitingForFinalCheckpoint() throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        int lastCheckpointId = 6;
        CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder();
        try (StreamTaskMailboxTestHarness<String> testHarness = createTestHarness(partitionWriters, checkpointResponder, false)) {
            // Tests triggering checkpoint when all the inputs are alive.
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after some inputs have received EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 4);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(4, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after received all the inputs have received
            // EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 1);
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 2);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 1);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 2);
            checkpointFuture = triggerCheckpoint(testHarness, lastCheckpointId);
            // Notifies the result partition that all records are processed after the
            // last checkpoint is triggered.
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            // The checkpoint 6 would be triggered successfully.
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(6, testHarness.getTaskStateManager().getReportedCheckpointId());
            assertEquals(6, testHarness.getTaskStateManager().getNotifiedCompletedCheckpointId());
            // Each result partition should have emitted 3 barriers and 1 EndOfUserRecordsEvent.
            for (ResultPartition resultPartition : partitionWriters) {
                assertEquals(4, resultPartition.getNumberOfQueuedBuffers());
            }
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}