Search in sources :

Example 11 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class StreamTaskFinalCheckpointsTest method testReportOperatorsFinishedInCheckpoint.

@Test
public void testReportOperatorsFinishedInCheckpoint() throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        final CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder();
        try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO, 1).addAdditionalOutput(partitionWriters).setCheckpointResponder(checkpointResponder).modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).setupOperatorChain(new StatefulOperator()).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
            checkpointResponder.setHandlers(testHarness.streamTask::notifyCheckpointCompleteAsync, testHarness.streamTask::notifyCheckpointAbortAsync);
            // Trigger the first checkpoint before we call operators' finish method.
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            assertFalse(testHarness.getTaskStateManager().getJobManagerTaskStateSnapshotsByCheckpointId().get(2L).isTaskFinished());
            // Trigger the first checkpoint after we call operators' finish method.
            // The checkpoint is added to the mailbox and will be processed in the
            // mailbox loop after call operators' finish method in the afterInvoke()
            // method.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 4);
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            testHarness.processAll();
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertTrue(testHarness.getTaskStateManager().getJobManagerTaskStateSnapshotsByCheckpointId().get(4L).isTaskFinished());
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}
Also used : EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) Deadline(org.apache.flink.api.common.time.Deadline) CheckpointMetricsBuilder(org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder) TestCheckpointResponder(org.apache.flink.runtime.taskmanager.TestCheckpointResponder) SavepointType(org.apache.flink.runtime.checkpoint.SavepointType) StringSerializer(org.apache.flink.api.common.typeutils.base.StringSerializer) BasicTypeInfo(org.apache.flink.api.common.typeinfo.BasicTypeInfo) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) ListState(org.apache.flink.api.common.state.ListState) Future(java.util.concurrent.Future) CheckpointStorageLocationReference.getDefault(org.apache.flink.runtime.state.CheckpointStorageLocationReference.getDefault) Duration(java.time.Duration) ListStateDescriptor(org.apache.flink.api.common.state.ListStateDescriptor) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) CheckpointType(org.apache.flink.runtime.checkpoint.CheckpointType) TestInputChannel(org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel) CheckpointOptions(org.apache.flink.runtime.checkpoint.CheckpointOptions) Collectors(java.util.stream.Collectors) StopMode(org.apache.flink.runtime.io.network.api.StopMode) PipelinedResultPartition(org.apache.flink.runtime.io.network.partition.PipelinedResultPartition) CheckpointBarrier(org.apache.flink.runtime.io.network.api.CheckpointBarrier) Matchers.contains(org.hamcrest.Matchers.contains) Assert.assertFalse(org.junit.Assert.assertFalse) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) CheckpointMetrics(org.apache.flink.runtime.checkpoint.CheckpointMetrics) OneInputStreamOperator(org.apache.flink.streaming.api.operators.OneInputStreamOperator) OneShotLatch(org.apache.flink.core.testutils.OneShotLatch) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) CheckpointMetaData(org.apache.flink.runtime.checkpoint.CheckpointMetaData) EndOfPartitionEvent(org.apache.flink.runtime.io.network.api.EndOfPartitionEvent) Watermark(org.apache.flink.streaming.api.watermark.Watermark) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) ResultPartitionType(org.apache.flink.runtime.io.network.partition.ResultPartitionType) CompletableFuture(java.util.concurrent.CompletableFuture) STRING_TYPE_INFO(org.apache.flink.api.common.typeinfo.BasicTypeInfo.STRING_TYPE_INFO) CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) SourceFunction(org.apache.flink.streaming.api.functions.source.SourceFunction) StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) Assert.assertArrayEquals(org.junit.Assert.assertArrayEquals) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Nullable(javax.annotation.Nullable) CheckpointStorageLocationReference(org.apache.flink.runtime.state.CheckpointStorageLocationReference) FlinkRuntimeException(org.apache.flink.util.FlinkRuntimeException) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) AbstractStreamOperator(org.apache.flink.streaming.api.operators.AbstractStreamOperator) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) JobID(org.apache.flink.api.common.JobID) PartitionTestUtils(org.apache.flink.runtime.io.network.partition.PartitionTestUtils) CommonTestUtils(org.apache.flink.runtime.testutils.CommonTestUtils) StreamSource(org.apache.flink.streaming.api.operators.StreamSource) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) StateInitializationContext(org.apache.flink.runtime.state.StateInitializationContext) CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) PipelinedResultPartition(org.apache.flink.runtime.io.network.partition.PipelinedResultPartition) EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) Test(org.junit.Test)

Example 12 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class SourceOperatorStreamTaskTest method testEmittingMaxWatermarkAfterReadingAllRecords.

@Test
public void testEmittingMaxWatermarkAfterReadingAllRecords() throws Exception {
    try (StreamTaskMailboxTestHarness<Integer> testHarness = createTestHarness()) {
        testHarness.processAll();
        testHarness.finishProcessing();
        Queue<Object> expectedOutput = new LinkedList<>();
        expectedOutput.add(Watermark.MAX_WATERMARK);
        expectedOutput.add(new EndOfData(StopMode.DRAIN));
        assertThat(testHarness.getOutput().toArray(), equalTo(expectedOutput.toArray()));
    }
}
Also used : EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) LinkedList(java.util.LinkedList) Test(org.junit.Test)

Example 13 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class TwoInputStreamTaskTest method testSkipExecutionsIfFinishedOnRestore.

@Test
public void testSkipExecutionsIfFinishedOnRestore() throws Exception {
    OperatorID nonSourceOperatorId = new OperatorID();
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(TwoInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).setCollectNetworkEvents().modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).setTaskStateSnapshot(1, TaskStateSnapshot.FINISHED_ON_RESTORE).setupOperatorChain(nonSourceOperatorId, new TestFinishedOnRestoreStreamOperator()).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
        testHarness.processElement(Watermark.MAX_WATERMARK, 0);
        testHarness.processElement(Watermark.MAX_WATERMARK, 1);
        testHarness.waitForTaskCompletion();
        assertThat(testHarness.getOutput(), contains(Watermark.MAX_WATERMARK, new EndOfData(StopMode.DRAIN)));
    }
}
Also used : EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) Test(org.junit.Test)

Example 14 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class BoundedBlockingResultPartition method notifyEndOfData.

@Override
public void notifyEndOfData(StopMode mode) throws IOException {
    if (!hasNotifiedEndOfUserRecords) {
        broadcastEvent(new EndOfData(mode), false);
        hasNotifiedEndOfUserRecords = true;
    }
}
Also used : EndOfData(org.apache.flink.runtime.io.network.api.EndOfData)

Example 15 with EndOfData

use of org.apache.flink.runtime.io.network.api.EndOfData in project flink by apache.

the class PipelinedResultPartition method notifyEndOfData.

@Override
public void notifyEndOfData(StopMode mode) throws IOException {
    synchronized (lock) {
        if (!hasNotifiedEndOfUserRecords) {
            broadcastEvent(new EndOfData(mode), false);
            hasNotifiedEndOfUserRecords = true;
        }
    }
}
Also used : EndOfData(org.apache.flink.runtime.io.network.api.EndOfData)

Aggregations

EndOfData (org.apache.flink.runtime.io.network.api.EndOfData)21 Test (org.junit.Test)12 CheckpointOptions (org.apache.flink.runtime.checkpoint.CheckpointOptions)6 CheckpointBarrier (org.apache.flink.runtime.io.network.api.CheckpointBarrier)6 ResultPartitionWriter (org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter)6 ResultPartition (org.apache.flink.runtime.io.network.partition.ResultPartition)6 CompletingCheckpointResponder (org.apache.flink.streaming.util.CompletingCheckpointResponder)6 ArrayList (java.util.ArrayList)5 CheckpointMetaData (org.apache.flink.runtime.checkpoint.CheckpointMetaData)5 SourceOperatorFactory (org.apache.flink.streaming.api.operators.SourceOperatorFactory)5 IOException (java.io.IOException)4 EndOfPartitionEvent (org.apache.flink.runtime.io.network.api.EndOfPartitionEvent)4 StopMode (org.apache.flink.runtime.io.network.api.StopMode)4 OperatorID (org.apache.flink.runtime.jobgraph.OperatorID)4 Duration (java.time.Duration)3 ArrayDeque (java.util.ArrayDeque)3 Collections (java.util.Collections)3 CompletableFuture (java.util.concurrent.CompletableFuture)3 JobID (org.apache.flink.api.common.JobID)3 BasicTypeInfo (org.apache.flink.api.common.typeinfo.BasicTypeInfo)3