Examples with StreamConfig org.apache.flink.streaming.api.graph.StreamConfig used on opensource projects

Example 26 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class SourceStreamTaskTest method testClosingAllOperatorsOnChainProperly.

@Test
public void testClosingAllOperatorsOnChainProperly() throws Exception {
    final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(SourceStreamTask::new, STRING_TYPE_INFO);
    testHarness.setupOperatorChain(new OperatorID(), new OutputRecordInCloseTestSource<>("Source0", new FromElementsFunction<>(StringSerializer.INSTANCE, "Hello"))).chain(new OperatorID(), new TestBoundedOneInputStreamOperator("Operator1"), STRING_TYPE_INFO.createSerializer(new ExecutionConfig())).finish();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    testHarness.invoke();
    testHarness.waitForTaskCompletion();
    ArrayList<Object> expected = new ArrayList<>();
    Collections.addAll(expected, new StreamRecord<>("Hello"), new StreamRecord<>("[Source0]: End of input"), Watermark.MAX_WATERMARK, new StreamRecord<>("[Source0]: Finish"), new StreamRecord<>("[Operator1]: End of input"), new StreamRecord<>("[Operator1]: Finish"));
    final Object[] output = testHarness.getOutput().toArray();
    assertArrayEquals("Output was not correct.", expected.toArray(), output);
}

Example 27 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class SourceStreamTaskTest method testOpenClose.

/**
 * This test verifies that open() and close() are correctly called by the StreamTask.
 */
@Test
public void testOpenClose() throws Exception {
    final StreamTaskTestHarness<String> testHarness = new StreamTaskTestHarness<>(SourceStreamTask::new, STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamSource<String, ?> sourceOperator = new StreamSource<>(new OpenCloseTestSource());
    streamConfig.setStreamOperator(sourceOperator);
    streamConfig.setOperatorID(new OperatorID());
    testHarness.invoke();
    testHarness.waitForTaskCompletion();
    assertTrue("RichFunction methods where not called.", OpenCloseTestSource.closeCalled);
    List<String> resultElements = TestHarnessUtil.getRawElementsFromOutput(testHarness.getOutput());
    Assert.assertEquals(10, resultElements.size());
}

Example 28 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class OneInputStreamTaskTest method testCheckpointBarriers.

/**
 * This test verifies that checkpoint barriers are correctly forwarded.
 */
@Test
public void testCheckpointBarriers() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, 2, 2, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamMap<String, String> mapOperator = new StreamMap<>(new IdentityMap());
    streamConfig.setStreamOperator(mapOperator);
    streamConfig.setOperatorID(new OperatorID());
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    long initialTime = 0L;
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 0);
    // These elements should be forwarded, since we did not yet receive a checkpoint barrier
    // on that input, only add to same input, otherwise we would not know the ordering
    // of the output since the Task might read the inputs in any order
    testHarness.processElement(new StreamRecord<>("Hello-1-1", initialTime), 1, 1);
    testHarness.processElement(new StreamRecord<>("Ciao-1-1", initialTime), 1, 1);
    expectedOutput.add(new StreamRecord<>("Hello-1-1", initialTime));
    expectedOutput.add(new StreamRecord<>("Ciao-1-1", initialTime));
    testHarness.waitForInputProcessing();
    // we should not yet see the barrier, only the two elements from non-blocked input
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 0, 1);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 0);
    testHarness.processEvent(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()), 1, 1);
    testHarness.waitForInputProcessing();
    // now we should see the barrier
    expectedOutput.add(new CheckpointBarrier(0, 0, CheckpointOptions.forCheckpointWithDefaultLocation()));
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}

Example 29 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class OperatorChainTest method setupOperatorChain.

// ------------------------------------------------------------------------
// Operator Chain Setup Utils
// ------------------------------------------------------------------------
@SafeVarargs
public static <T, OP extends StreamOperator<T>> OperatorChain<T, OP> setupOperatorChain(OneInputStreamOperator<T, T>... operators) throws Exception {
    checkNotNull(operators);
    checkArgument(operators.length > 0);
    try (MockEnvironment env = MockEnvironment.builder().build()) {
        final StreamTask<?, ?> containingTask = new MockStreamTaskBuilder(env).build();
        final StreamConfig cfg = new StreamConfig(new Configuration());
        cfg.setOperatorID(new OperatorID());
        cfg.setStateKeySerializer(new StringSerializer());
        final List<StreamOperatorWrapper<?, ?>> operatorWrappers = new ArrayList<>();
        // initial output goes to nowhere
        @SuppressWarnings({ "unchecked", "rawtypes" }) WatermarkGaugeExposingOutput<StreamRecord<T>> lastWriter = new BroadcastingOutputCollector<>(new Output[0]);
        // build the reverse operators array
        for (int i = 0; i < operators.length; i++) {
            int operatorIndex = operators.length - i - 1;
            OneInputStreamOperator<T, T> op = operators[operatorIndex];
            if (op instanceof SetupableStreamOperator) {
                ((SetupableStreamOperator) op).setup(containingTask, cfg, lastWriter);
            }
            lastWriter = new ChainingOutput<>(op, null);
            ProcessingTimeService processingTimeService = null;
            if (op instanceof AbstractStreamOperator) {
                processingTimeService = ((AbstractStreamOperator) op).getProcessingTimeService();
            }
            operatorWrappers.add(new StreamOperatorWrapper<>(op, Optional.ofNullable(processingTimeService), containingTask.getMailboxExecutorFactory().createExecutor(i), operatorIndex == 0));
        }
        @SuppressWarnings("unchecked") final StreamOperatorWrapper<T, OP> headOperatorWrapper = (StreamOperatorWrapper<T, OP>) operatorWrappers.get(operatorWrappers.size() - 1);
        return new RegularOperatorChain<>(operatorWrappers, new RecordWriterOutput<?>[0], lastWriter, headOperatorWrapper);
    }
}

Example 30 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskCancellationBarrierTest method testDeclineCallOnCancelBarrierTwoInputs.

/**
 * This test verifies (for two input tasks) that the Stream tasks react the following way to
 * receiving a checkpoint cancellation barrier: - send a "decline checkpoint" notification out
 * (to the JobManager) - emit a cancellation barrier downstream.
 */
@Test
public void testDeclineCallOnCancelBarrierTwoInputs() throws Exception {
    TwoInputStreamTaskTestHarness<String, String, String> testHarness = new TwoInputStreamTaskTestHarness<>(TwoInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    CoStreamMap<String, String, String> op = new CoStreamMap<>(new UnionCoMap());
    streamConfig.setStreamOperator(op);
    streamConfig.setOperatorID(new OperatorID());
    StreamMockEnvironment environment = spy(testHarness.createEnvironment());
    // start the task
    testHarness.invoke(environment);
    testHarness.waitForTaskRunning();
    // emit cancellation barriers
    testHarness.processEvent(new CancelCheckpointMarker(2L), 0, 0);
    testHarness.processEvent(new CancelCheckpointMarker(2L), 1, 0);
    testHarness.waitForInputProcessing();
    // the decline call should go to the coordinator
    verify(environment, times(1)).declineCheckpoint(eq(2L), argThat(new AlignedCheckpointsTest.CheckpointExceptionMatcher(CheckpointFailureReason.CHECKPOINT_DECLINED_ON_CANCELLATION_BARRIER)));
    // a cancellation barrier should be downstream
    Object result = testHarness.getOutput().poll();
    assertNotNull("nothing emitted", result);
    assertTrue("wrong type emitted", result instanceof CancelCheckpointMarker);
    assertEquals("wrong checkpoint id", 2L, ((CancelCheckpointMarker) result).getCheckpointId());
    // cancel and shutdown
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
}