Examples with StreamConfig org.apache.flink.streaming.api.graph.StreamConfig used on opensource projects

Example 51 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class AsyncWaitOperatorTest method testClosingWithBlockedEmitter.

/**
	 * Test case for FLINK-5638: Tests that the async wait operator can be closed even if the
	 * emitter is currently waiting on the checkpoint lock (e.g. in the case of two chained async
	 * wait operators where the latter operator's queue is currently full).
	 *
	 * Note that this test does not enforce the exact strict ordering because with the fix it is no
	 * longer possible. However, it provokes the described situation without the fix.
	 */
@Test(timeout = 10000L)
public void testClosingWithBlockedEmitter() throws Exception {
    final Object lock = new Object();
    ArgumentCaptor<Throwable> failureReason = ArgumentCaptor.forClass(Throwable.class);
    Environment environment = mock(Environment.class);
    when(environment.getMetricGroup()).thenReturn(new UnregisteredTaskMetricsGroup());
    when(environment.getTaskManagerInfo()).thenReturn(new TestingTaskManagerRuntimeInfo());
    when(environment.getUserClassLoader()).thenReturn(getClass().getClassLoader());
    when(environment.getTaskInfo()).thenReturn(new TaskInfo("testTask", 1, 0, 1, 0));
    doNothing().when(environment).failExternally(failureReason.capture());
    StreamTask<?, ?> containingTask = mock(StreamTask.class);
    when(containingTask.getEnvironment()).thenReturn(environment);
    when(containingTask.getCheckpointLock()).thenReturn(lock);
    when(containingTask.getProcessingTimeService()).thenReturn(new TestProcessingTimeService());
    StreamConfig streamConfig = mock(StreamConfig.class);
    doReturn(IntSerializer.INSTANCE).when(streamConfig).getTypeSerializerIn1(any(ClassLoader.class));
    final OneShotLatch closingLatch = new OneShotLatch();
    final OneShotLatch outputLatch = new OneShotLatch();
    Output<StreamRecord<Integer>> output = mock(Output.class);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            assertTrue("Output should happen under the checkpoint lock.", Thread.currentThread().holdsLock(lock));
            outputLatch.trigger();
            // wait until we're in the closing method of the operator
            while (!closingLatch.isTriggered()) {
                lock.wait();
            }
            return null;
        }
    }).when(output).collect(any(StreamRecord.class));
    AsyncWaitOperator<Integer, Integer> operator = new TestAsyncWaitOperator<>(new MyAsyncFunction(), 1000L, 1, AsyncDataStream.OutputMode.ORDERED, closingLatch);
    operator.setup(containingTask, streamConfig, output);
    operator.open();
    synchronized (lock) {
        operator.processElement(new StreamRecord<>(42));
    }
    outputLatch.await();
    synchronized (lock) {
        operator.close();
    }
    // check that no concurrent exception has occurred
    try {
        verify(environment, never()).failExternally(any(Throwable.class));
    } catch (Error e) {
        // add the exception occurring in the emitter thread (root cause) as a suppressed
        // exception
        e.addSuppressed(failureReason.getValue());
        throw e;
    }
}

Example 52 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskTest method testFailingCheckpointStreamOperator.

@Test
public void testFailingCheckpointStreamOperator() throws Exception {
    final long checkpointId = 42L;
    final long timestamp = 1L;
    TaskInfo mockTaskInfo = mock(TaskInfo.class);
    when(mockTaskInfo.getTaskNameWithSubtasks()).thenReturn("foobar");
    when(mockTaskInfo.getIndexOfThisSubtask()).thenReturn(0);
    Environment mockEnvironment = mock(Environment.class);
    when(mockEnvironment.getTaskInfo()).thenReturn(mockTaskInfo);
    StreamTask<?, AbstractStreamOperator<?>> streamTask = mock(StreamTask.class, Mockito.CALLS_REAL_METHODS);
    CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointId, timestamp);
    streamTask.setEnvironment(mockEnvironment);
    // mock the operators
    StreamOperator<?> streamOperator1 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    StreamOperator<?> streamOperator2 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    StreamOperator<?> streamOperator3 = mock(StreamOperator.class, withSettings().extraInterfaces(StreamCheckpointedOperator.class));
    // mock the returned snapshots
    OperatorSnapshotResult operatorSnapshotResult1 = mock(OperatorSnapshotResult.class);
    OperatorSnapshotResult operatorSnapshotResult2 = mock(OperatorSnapshotResult.class);
    final Exception testException = new Exception("Test exception");
    when(streamOperator1.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult1);
    when(streamOperator2.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(operatorSnapshotResult2);
    when(streamOperator3.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenThrow(testException);
    // mock the returned legacy snapshots
    StreamStateHandle streamStateHandle1 = mock(StreamStateHandle.class);
    StreamStateHandle streamStateHandle2 = mock(StreamStateHandle.class);
    StreamStateHandle streamStateHandle3 = mock(StreamStateHandle.class);
    when(streamOperator1.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle1);
    when(streamOperator2.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle2);
    when(streamOperator3.snapshotLegacyOperatorState(anyLong(), anyLong(), any(CheckpointOptions.class))).thenReturn(streamStateHandle3);
    // set up the task
    StreamOperator<?>[] streamOperators = { streamOperator1, streamOperator2, streamOperator3 };
    OperatorChain<Void, AbstractStreamOperator<Void>> operatorChain = mock(OperatorChain.class);
    when(operatorChain.getAllOperators()).thenReturn(streamOperators);
    Whitebox.setInternalState(streamTask, "isRunning", true);
    Whitebox.setInternalState(streamTask, "lock", new Object());
    Whitebox.setInternalState(streamTask, "operatorChain", operatorChain);
    Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry());
    Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration()));
    try {
        streamTask.triggerCheckpoint(checkpointMetaData, CheckpointOptions.forFullCheckpoint());
        fail("Expected test exception here.");
    } catch (Exception e) {
        assertEquals(testException, e.getCause());
    }
    verify(operatorSnapshotResult1).cancel();
    verify(operatorSnapshotResult2).cancel();
    verify(streamStateHandle1).discardState();
    verify(streamStateHandle2).discardState();
    verify(streamStateHandle3).discardState();
}

Example 53 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class OneInputStreamTaskTest method configureChainedTestingStreamOperator.

//==============================================================================================
// Utility functions and classes
//==============================================================================================
private void configureChainedTestingStreamOperator(StreamConfig streamConfig, int numberChainedTasks, long seed, long recoveryTimestamp) {
    Preconditions.checkArgument(numberChainedTasks >= 1, "The operator chain must at least " + "contain one operator.");
    Random random = new Random(seed);
    TestingStreamOperator<Integer, Integer> previousOperator = new TestingStreamOperator<>(random.nextLong(), recoveryTimestamp);
    streamConfig.setStreamOperator(previousOperator);
    // create the chain of operators
    Map<Integer, StreamConfig> chainedTaskConfigs = new HashMap<>(numberChainedTasks - 1);
    List<StreamEdge> outputEdges = new ArrayList<>(numberChainedTasks - 1);
    for (int chainedIndex = 1; chainedIndex < numberChainedTasks; chainedIndex++) {
        TestingStreamOperator<Integer, Integer> chainedOperator = new TestingStreamOperator<>(random.nextLong(), recoveryTimestamp);
        StreamConfig chainedConfig = new StreamConfig(new Configuration());
        chainedConfig.setStreamOperator(chainedOperator);
        chainedTaskConfigs.put(chainedIndex, chainedConfig);
        StreamEdge outputEdge = new StreamEdge(new StreamNode(null, chainedIndex - 1, null, null, null, null, null), new StreamNode(null, chainedIndex, null, null, null, null, null), 0, Collections.<String>emptyList(), null, null);
        outputEdges.add(outputEdge);
    }
    streamConfig.setChainedOutputs(outputEdges);
    streamConfig.setTransitiveChainedTaskConfigs(chainedTaskConfigs);
}

Example 54 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class OneInputStreamTaskTest method testWatermarkAndStreamStatusForwarding.

/**
	 * This test verifies that watermarks and stream statuses are correctly forwarded. This also checks whether
	 * watermarks are forwarded only when we have received watermarks from all inputs. The
	 * forwarded watermark must be the minimum of the watermarks of all active inputs.
	 */
@Test
@SuppressWarnings("unchecked")
public void testWatermarkAndStreamStatusForwarding() throws Exception {
    final OneInputStreamTask<String, String> mapTask = new OneInputStreamTask<String, String>();
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<String, String>(mapTask, 2, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamMap<String, String> mapOperator = new StreamMap<String, String>(new IdentityMap());
    streamConfig.setStreamOperator(mapOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processElement(new Watermark(initialTime), 0, 0);
    testHarness.processElement(new Watermark(initialTime), 0, 1);
    testHarness.processElement(new Watermark(initialTime), 1, 0);
    // now the output should still be empty
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(new Watermark(initialTime), 1, 1);
    // now the watermark should have propagated, Map simply forward Watermarks
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // contrary to checkpoint barriers these elements are not blocked by watermarks
    testHarness.processElement(new StreamRecord<String>("Hello", initialTime));
    testHarness.processElement(new StreamRecord<String>("Ciao", initialTime));
    expectedOutput.add(new StreamRecord<String>("Hello", initialTime));
    expectedOutput.add(new StreamRecord<String>("Ciao", initialTime));
    testHarness.processElement(new Watermark(initialTime + 4), 0, 0);
    testHarness.processElement(new Watermark(initialTime + 3), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 3), 1, 0);
    testHarness.processElement(new Watermark(initialTime + 2), 1, 1);
    // check whether we get the minimum of all the watermarks, this must also only occur in
    // the output after the two StreamRecords
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 2));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // advance watermark from one of the inputs, now we should get a new one since the
    // minimum increases
    testHarness.processElement(new Watermark(initialTime + 4), 1, 1);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 3));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // advance the other two inputs, now we should get a new one since the
    // minimum increases again
    testHarness.processElement(new Watermark(initialTime + 4), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 4), 1, 0);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 4));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // test whether idle input channels are acknowledged correctly when forwarding watermarks
    testHarness.processElement(StreamStatus.IDLE, 0, 1);
    testHarness.processElement(StreamStatus.IDLE, 1, 0);
    testHarness.processElement(new Watermark(initialTime + 6), 0, 0);
    // this watermark should be advanced first
    testHarness.processElement(new Watermark(initialTime + 5), 1, 1);
    // once this is acknowledged,
    testHarness.processElement(StreamStatus.IDLE, 1, 1);
    // watermark (initial + 6) should be forwarded
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 5));
    expectedOutput.add(new Watermark(initialTime + 6));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // make all input channels idle and check that the operator's idle status is forwarded
    testHarness.processElement(StreamStatus.IDLE, 0, 0);
    testHarness.waitForInputProcessing();
    expectedOutput.add(StreamStatus.IDLE);
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // make some input channels active again and check that the operator's active status is forwarded only once
    testHarness.processElement(StreamStatus.ACTIVE, 1, 0);
    testHarness.processElement(StreamStatus.ACTIVE, 0, 1);
    testHarness.waitForInputProcessing();
    expectedOutput.add(StreamStatus.ACTIVE);
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    assertEquals(2, resultElements.size());
}

Example 55 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class TwoInputStreamTaskTest method testWatermarkAndStreamStatusForwarding.

/**
	 * This test verifies that watermarks and stream statuses are correctly forwarded. This also checks whether
	 * watermarks are forwarded only when we have received watermarks from all inputs. The
	 * forwarded watermark must be the minimum of the watermarks of all active inputs.
	 */
@Test
@SuppressWarnings("unchecked")
public void testWatermarkAndStreamStatusForwarding() throws Exception {
    final TwoInputStreamTask<String, Integer, String> coMapTask = new TwoInputStreamTask<String, Integer, String>();
    final TwoInputStreamTaskTestHarness<String, Integer, String> testHarness = new TwoInputStreamTaskTestHarness<String, Integer, String>(coMapTask, 2, 2, new int[] { 1, 2 }, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, Integer, String> coMapOperator = new CoStreamMap<String, Integer, String>(new IdentityMap());
    streamConfig.setStreamOperator(coMapOperator);
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processElement(new Watermark(initialTime), 0, 0);
    testHarness.processElement(new Watermark(initialTime), 0, 1);
    testHarness.processElement(new Watermark(initialTime), 1, 0);
    // now the output should still be empty
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(new Watermark(initialTime), 1, 1);
    // now the watermark should have propagated, Map simply forward Watermarks
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // contrary to checkpoint barriers these elements are not blocked by watermarks
    testHarness.processElement(new StreamRecord<String>("Hello", initialTime), 0, 0);
    testHarness.processElement(new StreamRecord<Integer>(42, initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<String>("Hello", initialTime));
    expectedOutput.add(new StreamRecord<String>("42", initialTime));
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processElement(new Watermark(initialTime + 4), 0, 0);
    testHarness.processElement(new Watermark(initialTime + 3), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 3), 1, 0);
    testHarness.processElement(new Watermark(initialTime + 2), 1, 1);
    // check whether we get the minimum of all the watermarks, this must also only occur in
    // the output after the two StreamRecords
    expectedOutput.add(new Watermark(initialTime + 2));
    testHarness.waitForInputProcessing();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // advance watermark from one of the inputs, now we should get a new one since the
    // minimum increases
    testHarness.processElement(new Watermark(initialTime + 4), 1, 1);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 3));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // advance the other two inputs, now we should get a new one since the
    // minimum increases again
    testHarness.processElement(new Watermark(initialTime + 4), 0, 1);
    testHarness.processElement(new Watermark(initialTime + 4), 1, 0);
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 4));
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // test whether idle input channels are acknowledged correctly when forwarding watermarks
    testHarness.processElement(StreamStatus.IDLE, 0, 1);
    testHarness.processElement(StreamStatus.IDLE, 1, 0);
    testHarness.processElement(new Watermark(initialTime + 6), 0, 0);
    // this watermark should be advanced first
    testHarness.processElement(new Watermark(initialTime + 5), 1, 1);
    // once this is acknowledged,
    testHarness.processElement(StreamStatus.IDLE, 1, 1);
    // watermark (initial + 6) should be forwarded
    testHarness.waitForInputProcessing();
    expectedOutput.add(new Watermark(initialTime + 5));
    // We don't expect to see Watermark(6) here because the idle status of one
    // input doesn't propagate to the other input. That is, if input 1 is at WM 6 and input
    // two was at WM 5 before going to IDLE then the output watermark will not jump to WM 6.
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // make all input channels idle and check that the operator's idle status is forwarded
    testHarness.processElement(StreamStatus.IDLE, 0, 0);
    testHarness.waitForInputProcessing();
    expectedOutput.add(StreamStatus.IDLE);
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    // make some input channels active again and check that the operator's active status is forwarded only once
    testHarness.processElement(StreamStatus.ACTIVE, 1, 0);
    testHarness.processElement(StreamStatus.ACTIVE, 0, 1);
    testHarness.waitForInputProcessing();
    expectedOutput.add(StreamStatus.ACTIVE);
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    Assert.assertEquals(2, resultElements.size());
}