Examples with StreamConfig org.apache.flink.streaming.api.graph.StreamConfig used on opensource projects

Example 21 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class OneInputStreamTaskTest method testSnapshottingAndRestoring.

/**
 * Tests that the stream operator can snapshot and restore the operator state of chained
 * operators.
 */
@Test
public void testSnapshottingAndRestoring() throws Exception {
    final Deadline deadline = Deadline.fromNow(Duration.ofMinutes(2));
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    IdentityKeySelector<String> keySelector = new IdentityKeySelector<>();
    testHarness.configureForKeyedStream(keySelector, BasicTypeInfo.STRING_TYPE_INFO);
    long checkpointId = 1L;
    long checkpointTimestamp = 1L;
    int numberChainedTasks = 11;
    StreamConfig streamConfig = testHarness.getStreamConfig();
    configureChainedTestingStreamOperator(streamConfig, numberChainedTasks);
    TestTaskStateManager taskStateManager = testHarness.taskStateManager;
    // reset number of restore calls
    TestingStreamOperator.numberRestoreCalls = 0;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    final OneInputStreamTask<String, String> streamTask = testHarness.getTask();
    CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, checkpointTimestamp);
    streamTask.triggerCheckpointAsync(checkpointMetaData, CheckpointOptions.forCheckpointWithDefaultLocation()).get();
    // since no state was set, there shouldn't be restore calls
    assertEquals(0, TestingStreamOperator.numberRestoreCalls);
    taskStateManager.getWaitForReportLatch().await();
    assertEquals(checkpointId, taskStateManager.getReportedCheckpointId());
    testHarness.endInput();
    testHarness.waitForTaskCompletion(deadline.timeLeft().toMillis());
    final OneInputStreamTaskTestHarness<String, String> restoredTaskHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    restoredTaskHarness.configureForKeyedStream(keySelector, BasicTypeInfo.STRING_TYPE_INFO);
    restoredTaskHarness.setTaskStateSnapshot(checkpointId, taskStateManager.getLastJobManagerTaskStateSnapshot());
    StreamConfig restoredTaskStreamConfig = restoredTaskHarness.getStreamConfig();
    configureChainedTestingStreamOperator(restoredTaskStreamConfig, numberChainedTasks);
    TaskStateSnapshot stateHandles = taskStateManager.getLastJobManagerTaskStateSnapshot();
    Assert.assertEquals(numberChainedTasks, stateHandles.getSubtaskStateMappings().size());
    TestingStreamOperator.numberRestoreCalls = 0;
    // transfer state to new harness
    restoredTaskHarness.taskStateManager.restoreLatestCheckpointState(taskStateManager.getJobManagerTaskStateSnapshotsByCheckpointId());
    restoredTaskHarness.invoke();
    restoredTaskHarness.endInput();
    restoredTaskHarness.waitForTaskCompletion(deadline.timeLeft().toMillis());
    // restore of every chained operator should have been called
    assertEquals(numberChainedTasks, TestingStreamOperator.numberRestoreCalls);
    TestingStreamOperator.numberRestoreCalls = 0;
    TestingStreamOperator.numberSnapshotCalls = 0;
}

Example 22 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class OneInputStreamTaskTest method testCheckpointBarrierMetrics.

/**
 * Tests the checkpoint related metrics are registered into {@link TaskIOMetricGroup} correctly
 * while generating the {@link OneInputStreamTask}.
 */
@Test
public void testCheckpointBarrierMetrics() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setStreamOperator(new TestOperator());
    final Map<String, Metric> metrics = new ConcurrentHashMap<>();
    final TaskMetricGroup taskMetricGroup = StreamTaskTestHarness.createTaskMetricGroup(metrics);
    final StreamMockEnvironment environment = testHarness.createEnvironment();
    environment.setTaskMetricGroup(taskMetricGroup);
    testHarness.invoke(environment);
    testHarness.waitForTaskRunning();
    assertThat(metrics, IsMapContaining.hasKey(MetricNames.CHECKPOINT_ALIGNMENT_TIME));
    assertThat(metrics, IsMapContaining.hasKey(MetricNames.CHECKPOINT_START_DELAY_TIME));
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}

Example 23 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class OneInputStreamTaskTest method testWatermarkAndWatermarkStatusForwarding.

/**
 * This test verifies that watermarks and watermark statuses are correctly forwarded. This also
 * checks whether watermarks are forwarded only when we have received watermarks from all
 * inputs. The forwarded watermark must be the minimum of the watermarks of all active inputs.
 */
@Test
public void testWatermarkAndWatermarkStatusForwarding() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 2, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamMap<String, String> mapOperator = new StreamMap<>(new IdentityMap());
    streamConfig.setStreamOperator(mapOperator);
    streamConfig.setOperatorID(new OperatorID());
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processElement(new Watermark(initialTime), 0, 0);
    testHarness.processElement(new Watermark(initialTime), 0, 1);
    testHarness.processElement(new Watermark(initialTime), 1, 0);
    // now the output should still be empty
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(new Watermark(initialTime), 1, 1);
    // now the watermark should have propagated, Map simply forward Watermarks
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // contrary to checkpoint barriers these elements are not blocked by watermarks
    testHarness.processElement(new StreamRecord<>("Hello", initialTime));
    testHarness.processElement(new StreamRecord<>("Ciao", initialTime));
    expectedOutput.add(new StreamRecord<>("Hello", initialTime));
    expectedOutput.add(new StreamRecord<>("Ciao", initialTime));
    testHarness.processElement(new Watermark(initialTime + 4), 0, 0);
    testHarness.processElement(new Watermark(initialTime + 3), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 3), 1, 0);
    testHarness.processElement(new Watermark(initialTime + 2), 1, 1);
    // check whether we get the minimum of all the watermarks, this must also only occur in
    // the output after the two StreamRecords
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 2));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // advance watermark from one of the inputs, now we should get a new one since the
    // minimum increases
    testHarness.processElement(new Watermark(initialTime + 4), 1, 1);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 3));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // advance the other two inputs, now we should get a new one since the
    // minimum increases again
    testHarness.processElement(new Watermark(initialTime + 4), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 4), 1, 0);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 4));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // test whether idle input channels are acknowledged correctly when forwarding watermarks
    testHarness.processElement(WatermarkStatus.IDLE, 0, 1);
    testHarness.processElement(WatermarkStatus.IDLE, 1, 0);
    testHarness.processElement(new Watermark(initialTime + 6), 0, 0);
    testHarness.processElement(new Watermark(initialTime + 5), 1, // this watermark should be advanced first
    1);
    // once this is acknowledged,
    testHarness.processElement(WatermarkStatus.IDLE, 1, 1);
    // watermark (initial + 6) should be forwarded
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 5));
    expectedOutput.add(new Watermark(initialTime + 6));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // make all input channels idle and check that the operator's idle status is forwarded
    testHarness.processElement(WatermarkStatus.IDLE, 0, 0);
    testHarness.waitForInputProcessing();
    expectedOutput.add(WatermarkStatus.IDLE);
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // make some input channels active again and check that the operator's active status is
    // forwarded only once
    testHarness.processElement(WatermarkStatus.ACTIVE, 1, 0);
    testHarness.processElement(WatermarkStatus.ACTIVE, 0, 1);
    testHarness.waitForInputProcessing();
    expectedOutput.add(WatermarkStatus.ACTIVE);
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    assertEquals(2, resultElements.size());
}

Example 24 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class OneInputStreamTaskTest method testQuiesceTimerServiceAfterOpClose.

@Test
public void testQuiesceTimerServiceAfterOpClose() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 2, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setStreamOperator(new TestOperator());
    streamConfig.setOperatorID(new OperatorID());
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    SystemProcessingTimeService timeService = (SystemProcessingTimeService) testHarness.getTimerService();
    // verify that the timer service is running
    Assert.assertTrue(timeService.isAlive());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    timeService.shutdownService();
}

Example 25 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class SourceExternalCheckpointTriggerTest method testCheckpointsTriggeredBySource.

@Test
@SuppressWarnings("unchecked")
public void testCheckpointsTriggeredBySource() throws Exception {
    // set up the basic test harness
    final StreamTaskTestHarness<Long> testHarness = new StreamTaskTestHarness<>(SourceStreamTask::new, BasicTypeInfo.LONG_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    testHarness.getExecutionConfig().setLatencyTrackingInterval(-1);
    final long numElements = 10;
    final long checkpointEvery = 3;
    // set up the source function
    ExternalCheckpointsSource source = new ExternalCheckpointsSource(numElements, checkpointEvery);
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamSource<Long, ?> sourceOperator = new StreamSource<>(source);
    streamConfig.setStreamOperator(sourceOperator);
    streamConfig.setOperatorID(new OperatorID());
    // this starts the source thread
    testHarness.invoke();
    final StreamTask<Long, ?> sourceTask = testHarness.getTask();
    ready.await();
    // now send an external trigger that should be ignored
    assertTrue(sourceTask.triggerCheckpointAsync(new CheckpointMetaData(32, 829), CheckpointOptions.forCheckpointWithDefaultLocation()).get());
    // step by step let the source thread emit elements
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 1L);
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 2L);
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 3L);
    verifyCheckpointBarrier(testHarness.getOutput(), 1L);
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 4L);
    // now send an regular trigger command that should be ignored
    assertTrue(sourceTask.triggerCheckpointAsync(new CheckpointMetaData(34, 900), CheckpointOptions.forCheckpointWithDefaultLocation()).get());
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 5L);
    sync.trigger();
    verifyNextElement(testHarness.getOutput(), 6L);
    verifyCheckpointBarrier(testHarness.getOutput(), 2L);
    for (long l = 7L, checkpoint = 3L; l <= numElements; l++) {
        sync.trigger();
        verifyNextElement(testHarness.getOutput(), l);
        if (l % checkpointEvery == 0) {
            verifyCheckpointBarrier(testHarness.getOutput(), checkpoint++);
        }
    }
// done!
}