Search in sources :

Example 6 with CompletingCheckpointResponder

use of org.apache.flink.streaming.util.CompletingCheckpointResponder in project flink by apache.

the class StreamTaskFinalCheckpointsTest method testWaitingForFinalCheckpoint.

@Test
public void testWaitingForFinalCheckpoint() throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        int lastCheckpointId = 6;
        CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder();
        try (StreamTaskMailboxTestHarness<String> testHarness = createTestHarness(partitionWriters, checkpointResponder, false)) {
            // Tests triggering checkpoint when all the inputs are alive.
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after some inputs have received EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 4);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(4, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after received all the inputs have received
            // EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 1);
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 2);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 1);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 2);
            checkpointFuture = triggerCheckpoint(testHarness, lastCheckpointId);
            // Notifies the result partition that all records are processed after the
            // last checkpoint is triggered.
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            // The checkpoint 6 would be triggered successfully.
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(6, testHarness.getTaskStateManager().getReportedCheckpointId());
            assertEquals(6, testHarness.getTaskStateManager().getNotifiedCompletedCheckpointId());
            // Each result partition should have emitted 3 barriers and 1 EndOfUserRecordsEvent.
            for (ResultPartition resultPartition : partitionWriters) {
                assertEquals(4, resultPartition.getNumberOfQueuedBuffers());
            }
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}
Also used : EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) PipelinedResultPartition(org.apache.flink.runtime.io.network.partition.PipelinedResultPartition) Test(org.junit.Test)

Example 7 with CompletingCheckpointResponder

use of org.apache.flink.streaming.util.CompletingCheckpointResponder in project flink by apache.

the class StreamTaskFinalCheckpointsTest method testReportOperatorsFinishedInCheckpoint.

@Test
public void testReportOperatorsFinishedInCheckpoint() throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        final CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder();
        try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO, 1).addAdditionalOutput(partitionWriters).setCheckpointResponder(checkpointResponder).modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).setupOperatorChain(new StatefulOperator()).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
            checkpointResponder.setHandlers(testHarness.streamTask::notifyCheckpointCompleteAsync, testHarness.streamTask::notifyCheckpointAbortAsync);
            // Trigger the first checkpoint before we call operators' finish method.
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            assertFalse(testHarness.getTaskStateManager().getJobManagerTaskStateSnapshotsByCheckpointId().get(2L).isTaskFinished());
            // Trigger the first checkpoint after we call operators' finish method.
            // The checkpoint is added to the mailbox and will be processed in the
            // mailbox loop after call operators' finish method in the afterInvoke()
            // method.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 4);
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            testHarness.processAll();
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertTrue(testHarness.getTaskStateManager().getJobManagerTaskStateSnapshotsByCheckpointId().get(4L).isTaskFinished());
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}
Also used : EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) Deadline(org.apache.flink.api.common.time.Deadline) CheckpointMetricsBuilder(org.apache.flink.runtime.checkpoint.CheckpointMetricsBuilder) TestCheckpointResponder(org.apache.flink.runtime.taskmanager.TestCheckpointResponder) SavepointType(org.apache.flink.runtime.checkpoint.SavepointType) StringSerializer(org.apache.flink.api.common.typeutils.base.StringSerializer) BasicTypeInfo(org.apache.flink.api.common.typeinfo.BasicTypeInfo) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) ListState(org.apache.flink.api.common.state.ListState) Future(java.util.concurrent.Future) CheckpointStorageLocationReference.getDefault(org.apache.flink.runtime.state.CheckpointStorageLocationReference.getDefault) Duration(java.time.Duration) ListStateDescriptor(org.apache.flink.api.common.state.ListStateDescriptor) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) CheckpointType(org.apache.flink.runtime.checkpoint.CheckpointType) TestInputChannel(org.apache.flink.runtime.io.network.partition.consumer.TestInputChannel) CheckpointOptions(org.apache.flink.runtime.checkpoint.CheckpointOptions) Collectors(java.util.stream.Collectors) StopMode(org.apache.flink.runtime.io.network.api.StopMode) PipelinedResultPartition(org.apache.flink.runtime.io.network.partition.PipelinedResultPartition) CheckpointBarrier(org.apache.flink.runtime.io.network.api.CheckpointBarrier) Matchers.contains(org.hamcrest.Matchers.contains) Assert.assertFalse(org.junit.Assert.assertFalse) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) CheckpointMetrics(org.apache.flink.runtime.checkpoint.CheckpointMetrics) OneInputStreamOperator(org.apache.flink.streaming.api.operators.OneInputStreamOperator) OneShotLatch(org.apache.flink.core.testutils.OneShotLatch) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) CheckpointMetaData(org.apache.flink.runtime.checkpoint.CheckpointMetaData) EndOfPartitionEvent(org.apache.flink.runtime.io.network.api.EndOfPartitionEvent) Watermark(org.apache.flink.streaming.api.watermark.Watermark) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) ResultPartitionType(org.apache.flink.runtime.io.network.partition.ResultPartitionType) CompletableFuture(java.util.concurrent.CompletableFuture) STRING_TYPE_INFO(org.apache.flink.api.common.typeinfo.BasicTypeInfo.STRING_TYPE_INFO) CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) SourceFunction(org.apache.flink.streaming.api.functions.source.SourceFunction) StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) Assert.assertArrayEquals(org.junit.Assert.assertArrayEquals) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Nullable(javax.annotation.Nullable) CheckpointStorageLocationReference(org.apache.flink.runtime.state.CheckpointStorageLocationReference) FlinkRuntimeException(org.apache.flink.util.FlinkRuntimeException) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) AbstractStreamOperator(org.apache.flink.streaming.api.operators.AbstractStreamOperator) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) JobID(org.apache.flink.api.common.JobID) PartitionTestUtils(org.apache.flink.runtime.io.network.partition.PartitionTestUtils) CommonTestUtils(org.apache.flink.runtime.testutils.CommonTestUtils) StreamSource(org.apache.flink.streaming.api.operators.StreamSource) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) StateInitializationContext(org.apache.flink.runtime.state.StateInitializationContext) CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) PipelinedResultPartition(org.apache.flink.runtime.io.network.partition.PipelinedResultPartition) EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) Test(org.junit.Test)

Example 8 with CompletingCheckpointResponder

use of org.apache.flink.streaming.util.CompletingCheckpointResponder in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method testTriggerCheckpointWithFinishedChannelsAndSourceChain.

private void testTriggerCheckpointWithFinishedChannelsAndSourceChain(CheckpointOptions checkpointOptions) throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder();
        try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).modifyStreamConfig(config -> {
            config.setCheckpointingEnabled(true);
            config.setUnalignedCheckpointsEnabled(checkpointOptions.isUnalignedCheckpoint() || checkpointOptions.isTimeoutable());
        }).modifyExecutionConfig(applyObjectReuse(objectReuse)).setCheckpointResponder(checkpointResponder).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MultipleInputStreamTaskTest.LifeCycleTrackingMockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MultipleInputStreamTaskTest.LifeCycleTrackingMockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addAdditionalOutput(partitionWriters).setupOperatorChain(new MapToStringMultipleInputOperatorFactory(4)).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
            checkpointResponder.setHandlers(testHarness.streamTask::notifyCheckpointCompleteAsync, testHarness.streamTask::notifyCheckpointAbortAsync);
            testHarness.getStreamTask().getCheckpointBarrierHandler().get();
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2, checkpointOptions);
            testHarness.processAll();
            // The checkpoint 2 would be aligned after received all the EndOfPartitionEvent.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 1, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 1, 0);
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after all the inputs have received EndOfPartition.
            checkpointFuture = triggerCheckpoint(testHarness, 4, checkpointOptions);
            // Notifies the result partition that all records are processed after the
            // last checkpoint is triggered.
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            // The checkpoint 4 would be triggered successfully.
            testHarness.processAll();
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(4, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Each result partition should have emitted 2 barriers and 1 EndOfUserRecordsEvent.
            for (ResultPartition resultPartition : partitionWriters) {
                assertEquals(3, resultPartition.getNumberOfQueuedBuffers());
            }
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}
Also used : CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) RecordOrEventCollectingResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.RecordOrEventCollectingResultPartitionWriter) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) SourceOperatorFactory(org.apache.flink.streaming.api.operators.SourceOperatorFactory) StreamTaskFinalCheckpointsTest.triggerCheckpoint(org.apache.flink.streaming.runtime.tasks.StreamTaskFinalCheckpointsTest.triggerCheckpoint) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) MapToStringMultipleInputOperatorFactory(org.apache.flink.streaming.runtime.tasks.MultipleInputStreamTaskTest.MapToStringMultipleInputOperatorFactory)

Example 9 with CompletingCheckpointResponder

use of org.apache.flink.streaming.util.CompletingCheckpointResponder in project flink by apache.

the class MultipleInputStreamTaskTest method testTriggeringCheckpointWithFinishedChannels.

private void testTriggeringCheckpointWithFinishedChannels(CheckpointOptions checkpointOptions) throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder();
        try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.DOUBLE_TYPE_INFO).addAdditionalOutput(partitionWriters).setCheckpointResponder(checkpointResponder).modifyStreamConfig(config -> {
            config.setCheckpointingEnabled(true);
            config.setUnalignedCheckpointsEnabled(checkpointOptions.isUnalignedCheckpoint() || checkpointOptions.isTimeoutable());
        }).setupOperatorChain(new MapToStringMultipleInputOperatorFactory(3)).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
            checkpointResponder.setHandlers(testHarness.streamTask::notifyCheckpointCompleteAsync, testHarness.streamTask::notifyCheckpointAbortAsync);
            testHarness.getStreamTask().getCheckpointBarrierHandler().get();
            // Tests triggering checkpoint when all the inputs are alive.
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2, checkpointOptions);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after some inputs have received EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 4, checkpointOptions);
            processMailTillCheckpointSucceeds(testHarness, checkpointFuture);
            assertEquals(4, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after all the inputs have received EndOfPartition.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 1, 0);
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 2, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 1, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 2, 0);
            checkpointFuture = triggerCheckpoint(testHarness, 6, checkpointOptions);
            // Notifies the result partition that all records are processed after the
            // last checkpoint is triggered.
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            // The checkpoint 6 would be triggered successfully.
            testHarness.processAll();
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(6, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Each result partition should have emitted 3 barriers and 1 EndOfUserRecordsEvent.
            for (ResultPartition resultPartition : partitionWriters) {
                assertEquals(4, resultPartition.getNumberOfQueuedBuffers());
            }
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}
Also used : EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) TaskIOMetricGroup(org.apache.flink.runtime.metrics.groups.TaskIOMetricGroup) Arrays(java.util.Arrays) TestCheckpointResponder(org.apache.flink.runtime.taskmanager.TestCheckpointResponder) SharedObjects(org.apache.flink.testutils.junit.SharedObjects) NoMoreSplitsEvent(org.apache.flink.runtime.source.event.NoMoreSplitsEvent) AbstractStreamOperatorFactory(org.apache.flink.streaming.api.operators.AbstractStreamOperatorFactory) Duration(java.time.Duration) Map(java.util.Map) WatermarkStatus(org.apache.flink.streaming.runtime.watermarkstatus.WatermarkStatus) TypeSerializer(org.apache.flink.api.common.typeutils.TypeSerializer) BoundedOneInput(org.apache.flink.streaming.api.operators.BoundedOneInput) Serializable(java.io.Serializable) StopMode(org.apache.flink.runtime.io.network.api.StopMode) MetricNames(org.apache.flink.runtime.metrics.MetricNames) CheckpointBarrier(org.apache.flink.runtime.io.network.api.CheckpointBarrier) Matchers.contains(org.hamcrest.Matchers.contains) WatermarkMetricOperator(org.apache.flink.streaming.runtime.tasks.OneInputStreamTaskTest.WatermarkMetricOperator) OneInputStreamOperator(org.apache.flink.streaming.api.operators.OneInputStreamOperator) Boundedness(org.apache.flink.api.connector.source.Boundedness) Counter(org.apache.flink.metrics.Counter) RunWith(org.junit.runner.RunWith) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) TimestampAssigner(org.apache.flink.api.common.eventtime.TimestampAssigner) DataOutputView(org.apache.flink.core.memory.DataOutputView) AbstractInput(org.apache.flink.streaming.api.operators.AbstractInput) ArrayList(java.util.ArrayList) CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) InternalOperatorMetricGroup(org.apache.flink.runtime.metrics.groups.InternalOperatorMetricGroup) Gauge(org.apache.flink.metrics.Gauge) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) TestHarnessUtil(org.apache.flink.streaming.util.TestHarnessUtil) Before(org.junit.Before) CheckpointStorageLocationReference(org.apache.flink.runtime.state.CheckpointStorageLocationReference) SourceReader(org.apache.flink.api.connector.source.SourceReader) Parameter(org.junit.runners.Parameterized.Parameter) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) InterceptingTaskMetricGroup(org.apache.flink.runtime.metrics.util.InterceptingTaskMetricGroup) AddSplitEvent(org.apache.flink.runtime.source.event.AddSplitEvent) StreamMultipleInputProcessor(org.apache.flink.streaming.runtime.io.StreamMultipleInputProcessor) AbstractStreamOperator(org.apache.flink.streaming.api.operators.AbstractStreamOperator) StreamOperator(org.apache.flink.streaming.api.operators.StreamOperator) JobID(org.apache.flink.api.common.JobID) UnregisteredMetricGroups(org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups) StreamTaskFinalCheckpointsTest.processMailTillCheckpointSucceeds(org.apache.flink.streaming.runtime.tasks.StreamTaskFinalCheckpointsTest.processMailTillCheckpointSucceeds) Assert(org.junit.Assert) ArrayDeque(java.util.ArrayDeque) Assert.assertEquals(org.junit.Assert.assertEquals) Input(org.apache.flink.streaming.api.operators.Input) LifeCycleMonitorMultipleInputOperatorFactory(org.apache.flink.streaming.runtime.tasks.MultipleInputStreamTaskChainedSourcesCheckpointingTest.LifeCycleMonitorMultipleInputOperatorFactory) WatermarkGenerator(org.apache.flink.api.common.eventtime.WatermarkGenerator) SavepointType(org.apache.flink.runtime.checkpoint.SavepointType) StringSerializer(org.apache.flink.api.common.typeutils.base.StringSerializer) StreamTaskFinalCheckpointsTest.triggerCheckpoint(org.apache.flink.streaming.runtime.tasks.StreamTaskFinalCheckpointsTest.triggerCheckpoint) BasicTypeInfo(org.apache.flink.api.common.typeinfo.BasicTypeInfo) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) IntSerializer(org.apache.flink.api.common.typeutils.base.IntSerializer) TypeSerializerSnapshot(org.apache.flink.api.common.typeutils.TypeSerializerSnapshot) Parameterized(org.junit.runners.Parameterized) SourceReaderContext(org.apache.flink.api.connector.source.SourceReaderContext) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) CheckpointType(org.apache.flink.runtime.checkpoint.CheckpointType) InterceptingOperatorMetricGroup(org.apache.flink.runtime.metrics.util.InterceptingOperatorMetricGroup) BoundedMultiInput(org.apache.flink.streaming.api.operators.BoundedMultiInput) MockSourceReader(org.apache.flink.api.connector.source.mocks.MockSourceReader) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) WatermarkStrategy(org.apache.flink.api.common.eventtime.WatermarkStrategy) MockSourceSplitSerializer(org.apache.flink.api.connector.source.mocks.MockSourceSplitSerializer) CheckpointOptions(org.apache.flink.runtime.checkpoint.CheckpointOptions) MultipleInputStreamOperator(org.apache.flink.streaming.api.operators.MultipleInputStreamOperator) List(java.util.List) SerializedValue(org.apache.flink.util.SerializedValue) Preconditions.checkArgument(org.apache.flink.util.Preconditions.checkArgument) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) ExecutionConfig(org.apache.flink.api.common.ExecutionConfig) CancelCheckpointMarker(org.apache.flink.runtime.io.network.api.CancelCheckpointMarker) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) CheckpointResponder(org.apache.flink.runtime.taskmanager.CheckpointResponder) CheckpointMetrics(org.apache.flink.runtime.checkpoint.CheckpointMetrics) StreamConfig(org.apache.flink.streaming.api.graph.StreamConfig) SavepointFormatType(org.apache.flink.core.execution.SavepointFormatType) CheckpointMetaData(org.apache.flink.runtime.checkpoint.CheckpointMetaData) Parameters(org.junit.runners.Parameterized.Parameters) CoreMatchers.not(org.hamcrest.CoreMatchers.not) EndOfPartitionEvent(org.apache.flink.runtime.io.network.api.EndOfPartitionEvent) AbstractStreamOperatorV2(org.apache.flink.streaming.api.operators.AbstractStreamOperatorV2) Watermark(org.apache.flink.streaming.api.watermark.Watermark) ResultPartitionType(org.apache.flink.runtime.io.network.partition.ResultPartitionType) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Metric(org.apache.flink.metrics.Metric) SourceOperatorFactory(org.apache.flink.streaming.api.operators.SourceOperatorFactory) MockSourceSplit(org.apache.flink.api.connector.source.mocks.MockSourceSplit) TaskMetricGroup(org.apache.flink.runtime.metrics.groups.TaskMetricGroup) StreamRecord(org.apache.flink.streaming.runtime.streamrecord.StreamRecord) MockSource(org.apache.flink.api.connector.source.mocks.MockSource) OperatorMetricGroup(org.apache.flink.metrics.groups.OperatorMetricGroup) DataInputView(org.apache.flink.core.memory.DataInputView) SharedReference(org.apache.flink.testutils.junit.SharedReference) StreamOperatorParameters(org.apache.flink.streaming.api.operators.StreamOperatorParameters) WatermarkOutput(org.apache.flink.api.common.eventtime.WatermarkOutput) IsMapContaining(org.hamcrest.collection.IsMapContaining) Consumer(java.util.function.Consumer) ExecutionAttemptID(org.apache.flink.runtime.executiongraph.ExecutionAttemptID) Rule(org.junit.Rule) PartitionTestUtils(org.apache.flink.runtime.io.network.partition.PartitionTestUtils) LatencyMarker(org.apache.flink.streaming.runtime.streamrecord.LatencyMarker) Collections(java.util.Collections) CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) StreamTaskFinalCheckpointsTest.triggerCheckpoint(org.apache.flink.streaming.runtime.tasks.StreamTaskFinalCheckpointsTest.triggerCheckpoint) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) EndOfData(org.apache.flink.runtime.io.network.api.EndOfData)

Aggregations

CompletingCheckpointResponder (org.apache.flink.streaming.util.CompletingCheckpointResponder)9 ResultPartitionWriter (org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter)8 ResultPartition (org.apache.flink.runtime.io.network.partition.ResultPartition)8 Test (org.junit.Test)7 EndOfData (org.apache.flink.runtime.io.network.api.EndOfData)6 PipelinedResultPartition (org.apache.flink.runtime.io.network.partition.PipelinedResultPartition)6 JobID (org.apache.flink.api.common.JobID)5 CheckpointMetrics (org.apache.flink.runtime.checkpoint.CheckpointMetrics)5 TaskStateSnapshot (org.apache.flink.runtime.checkpoint.TaskStateSnapshot)5 ExecutionAttemptID (org.apache.flink.runtime.executiongraph.ExecutionAttemptID)5 CheckpointMetaData (org.apache.flink.runtime.checkpoint.CheckpointMetaData)4 Duration (java.time.Duration)3 Collections (java.util.Collections)3 CompletableFuture (java.util.concurrent.CompletableFuture)3 BasicTypeInfo (org.apache.flink.api.common.typeinfo.BasicTypeInfo)3 StringSerializer (org.apache.flink.api.common.typeutils.base.StringSerializer)3 SavepointFormatType (org.apache.flink.core.execution.SavepointFormatType)3 CheckpointOptions (org.apache.flink.runtime.checkpoint.CheckpointOptions)3 CheckpointType (org.apache.flink.runtime.checkpoint.CheckpointType)3 SavepointType (org.apache.flink.runtime.checkpoint.SavepointType)3