Search in sources :

Example 11 with SourceOperatorFactory

use of org.apache.flink.streaming.api.operators.SourceOperatorFactory in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method testTriggerCheckpointWithFinishedChannelsAndSourceChain.

private void testTriggerCheckpointWithFinishedChannelsAndSourceChain(CheckpointOptions checkpointOptions) throws Exception {
    ResultPartition[] partitionWriters = new ResultPartition[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        CompletingCheckpointResponder checkpointResponder = new CompletingCheckpointResponder();
        try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).modifyStreamConfig(config -> {
            config.setCheckpointingEnabled(true);
            config.setUnalignedCheckpointsEnabled(checkpointOptions.isUnalignedCheckpoint() || checkpointOptions.isTimeoutable());
        }).modifyExecutionConfig(applyObjectReuse(objectReuse)).setCheckpointResponder(checkpointResponder).addInput(BasicTypeInfo.INT_TYPE_INFO).addInput(BasicTypeInfo.STRING_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MultipleInputStreamTaskTest.LifeCycleTrackingMockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addSourceInput(new SourceOperatorFactory<>(new MultipleInputStreamTaskTest.LifeCycleTrackingMockSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addAdditionalOutput(partitionWriters).setupOperatorChain(new MapToStringMultipleInputOperatorFactory(4)).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
            checkpointResponder.setHandlers(testHarness.streamTask::notifyCheckpointCompleteAsync, testHarness.streamTask::notifyCheckpointAbortAsync);
            testHarness.getStreamTask().getCheckpointBarrierHandler().get();
            CompletableFuture<Boolean> checkpointFuture = triggerCheckpoint(testHarness, 2, checkpointOptions);
            testHarness.processAll();
            // The checkpoint 2 would be aligned after received all the EndOfPartitionEvent.
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 0, 0);
            testHarness.processEvent(new EndOfData(StopMode.DRAIN), 1, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 0, 0);
            testHarness.processEvent(EndOfPartitionEvent.INSTANCE, 1, 0);
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(2, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Tests triggering checkpoint after all the inputs have received EndOfPartition.
            checkpointFuture = triggerCheckpoint(testHarness, 4, checkpointOptions);
            // Notifies the result partition that all records are processed after the
            // last checkpoint is triggered.
            checkpointFuture.thenAccept((ignored) -> {
                for (ResultPartition resultPartition : partitionWriters) {
                    resultPartition.onSubpartitionAllDataProcessed(0);
                }
            });
            // The checkpoint 4 would be triggered successfully.
            testHarness.processAll();
            testHarness.finishProcessing();
            assertTrue(checkpointFuture.isDone());
            testHarness.getTaskStateManager().getWaitForReportLatch().await();
            assertEquals(4, testHarness.getTaskStateManager().getReportedCheckpointId());
            // Each result partition should have emitted 2 barriers and 1 EndOfUserRecordsEvent.
            for (ResultPartition resultPartition : partitionWriters) {
                assertEquals(3, resultPartition.getNumberOfQueuedBuffers());
            }
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}
Also used : CompletingCheckpointResponder(org.apache.flink.streaming.util.CompletingCheckpointResponder) RecordOrEventCollectingResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.RecordOrEventCollectingResultPartitionWriter) ResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.ResultPartitionWriter) SourceOperatorFactory(org.apache.flink.streaming.api.operators.SourceOperatorFactory) StreamTaskFinalCheckpointsTest.triggerCheckpoint(org.apache.flink.streaming.runtime.tasks.StreamTaskFinalCheckpointsTest.triggerCheckpoint) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) MapToStringMultipleInputOperatorFactory(org.apache.flink.streaming.runtime.tasks.MultipleInputStreamTaskTest.MapToStringMultipleInputOperatorFactory)

Example 12 with SourceOperatorFactory

use of org.apache.flink.streaming.api.operators.SourceOperatorFactory in project flink by apache.

the class MultipleInputStreamTaskChainedSourcesCheckpointingTest method testSkipExecutionsIfFinishedOnRestoreWithSourceChained.

@Test
public void testSkipExecutionsIfFinishedOnRestoreWithSourceChained() throws Exception {
    OperatorID firstSourceOperatorId = new OperatorID();
    OperatorID secondSourceOperatorId = new OperatorID();
    OperatorID nonSourceOperatorId = new OperatorID();
    List<Object> output = new ArrayList<>();
    try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(MultipleInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO).modifyStreamConfig(config -> config.setCheckpointingEnabled(true)).modifyExecutionConfig(applyObjectReuse(objectReuse)).addInput(BasicTypeInfo.INT_TYPE_INFO).addAdditionalOutput(new RecordOrEventCollectingResultPartitionWriter<StreamElement>(output, new StreamElementSerializer<>(IntSerializer.INSTANCE)) {

        @Override
        public void notifyEndOfData(StopMode mode) throws IOException {
            broadcastEvent(new EndOfData(mode), false);
        }
    }).addSourceInput(firstSourceOperatorId, new SourceOperatorFactory<>(new SourceOperatorStreamTaskTest.LifeCycleMonitorSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).addSourceInput(secondSourceOperatorId, new SourceOperatorFactory<>(new SourceOperatorStreamTaskTest.LifeCycleMonitorSource(Boundedness.CONTINUOUS_UNBOUNDED, 1), WatermarkStrategy.noWatermarks()), BasicTypeInfo.INT_TYPE_INFO).setTaskStateSnapshot(1, TaskStateSnapshot.FINISHED_ON_RESTORE).setupOperatorChain(nonSourceOperatorId, new LifeCycleMonitorMultipleInputOperatorFactory()).chain(new TestFinishedOnRestoreStreamOperator(), StringSerializer.INSTANCE).finish().build()) {
        testHarness.processElement(Watermark.MAX_WATERMARK);
        assertThat(output, is(empty()));
        testHarness.waitForTaskCompletion();
        assertThat(output, contains(Watermark.MAX_WATERMARK, new EndOfData(StopMode.DRAIN)));
        for (StreamOperatorWrapper<?, ?> wrapper : testHarness.getStreamTask().operatorChain.getAllOperators()) {
            if (wrapper.getStreamOperator() instanceof SourceOperator<?, ?>) {
                SourceOperatorStreamTaskTest.LifeCycleMonitorSourceReader sourceReader = (SourceOperatorStreamTaskTest.LifeCycleMonitorSourceReader) ((SourceOperator<?, ?>) wrapper.getStreamOperator()).getSourceReader();
                sourceReader.getLifeCycleMonitor().assertCallTimes(0, LifeCyclePhase.values());
            }
        }
    }
}
Also used : ArrayList(java.util.ArrayList) StopMode(org.apache.flink.runtime.io.network.api.StopMode) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) SourceOperatorFactory(org.apache.flink.streaming.api.operators.SourceOperatorFactory) EndOfData(org.apache.flink.runtime.io.network.api.EndOfData) SourceOperator(org.apache.flink.streaming.api.operators.SourceOperator) RecordOrEventCollectingResultPartitionWriter(org.apache.flink.runtime.io.network.api.writer.RecordOrEventCollectingResultPartitionWriter) Test(org.junit.Test)

Aggregations

SourceOperatorFactory (org.apache.flink.streaming.api.operators.SourceOperatorFactory)12 Test (org.junit.Test)8 ArrayList (java.util.ArrayList)6 MockSource (org.apache.flink.api.connector.source.mocks.MockSource)6 EndOfData (org.apache.flink.runtime.io.network.api.EndOfData)4 OperatorID (org.apache.flink.runtime.jobgraph.OperatorID)4 HashMap (java.util.HashMap)3 ExecutionConfig (org.apache.flink.api.common.ExecutionConfig)3 TaskStateSnapshot (org.apache.flink.runtime.checkpoint.TaskStateSnapshot)3 StreamTaskFinalCheckpointsTest.triggerCheckpoint (org.apache.flink.streaming.runtime.tasks.StreamTaskFinalCheckpointsTest.triggerCheckpoint)3 IOException (java.io.IOException)2 ArrayDeque (java.util.ArrayDeque)2 List (java.util.List)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 JobID (org.apache.flink.api.common.JobID)2 Counter (org.apache.flink.metrics.Counter)2 OperatorMetricGroup (org.apache.flink.metrics.groups.OperatorMetricGroup)2 CheckpointMetaData (org.apache.flink.runtime.checkpoint.CheckpointMetaData)2 CheckpointMetrics (org.apache.flink.runtime.checkpoint.CheckpointMetrics)2