Examples with StreamConfig org.apache.flink.streaming.api.graph.StreamConfig used on opensource projects

Example 76 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class AbstractUdfStreamOperatorLifecycleTest method testLifeCycleFull.

@Test
public void testLifeCycleFull() throws Exception {
    ACTUAL_ORDER_TRACKING.clear();
    Configuration taskManagerConfig = new Configuration();
    StreamConfig cfg = new StreamConfig(new Configuration());
    MockSourceFunction srcFun = new MockSourceFunction();
    cfg.setStreamOperator(new LifecycleTrackingStreamSource<>(srcFun, true));
    cfg.setOperatorID(new OperatorID());
    cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
    try (ShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build()) {
        Task task = StreamTaskTest.createTask(SourceStreamTask.class, shuffleEnvironment, cfg, taskManagerConfig);
        task.startTaskThread();
        LifecycleTrackingStreamSource.runStarted.await();
        // wait for clean termination
        task.getExecutingThread().join();
        assertEquals(ExecutionState.FINISHED, task.getExecutionState());
        assertEquals(EXPECTED_CALL_ORDER_FULL, ACTUAL_ORDER_TRACKING);
    }
}

Example 77 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamSourceOperatorWatermarksTest method setupSourceOperator.

// ------------------------------------------------------------------------
@SuppressWarnings("unchecked")
private static <T> MockStreamTask setupSourceOperator(StreamSource<T, ?> operator, TimeCharacteristic timeChar, long watermarkInterval, final TimerService timeProvider) throws Exception {
    ExecutionConfig executionConfig = new ExecutionConfig();
    executionConfig.setAutoWatermarkInterval(watermarkInterval);
    StreamConfig cfg = new StreamConfig(new Configuration());
    cfg.setStateBackend(new MemoryStateBackend());
    cfg.setTimeCharacteristic(timeChar);
    cfg.setOperatorID(new OperatorID());
    Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);
    MockStreamTask mockTask = new MockStreamTaskBuilder(env).setConfig(cfg).setExecutionConfig(executionConfig).setTimerService(timeProvider).build();
    operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
    return mockTask;
}

Example 78 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamSourceOperatorWatermarksTest method setupSourceStreamTask.

private static <T> StreamTaskTestHarness<T> setupSourceStreamTask(StreamSource<T, ?> sourceOperator, TypeInformation<T> outputType, final boolean cancelImmediatelyAfterCreation) {
    final StreamTaskTestHarness<T> testHarness = new StreamTaskTestHarness<>((env) -> {
        SourceStreamTask<T, ?, ?> sourceTask = new SourceStreamTask<>(env);
        if (cancelImmediatelyAfterCreation) {
            try {
                sourceTask.cancel();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        }
        return sourceTask;
    }, outputType);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setStreamOperator(sourceOperator);
    streamConfig.setOperatorID(new OperatorID());
    streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
    return testHarness;
}

Example 79 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskTimerTest method startTestHarness.

private StreamTaskTestHarness<?> startTestHarness() throws Exception {
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(OneInputStreamTask::new, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    // Making it impossible to execute the throughput calculation even once during the test.
    final Configuration taskConfig = testHarness.getTaskManagerRuntimeInfo().getConfiguration();
    taskConfig.set(TaskManagerOptions.BUFFER_DEBLOAT_ENABLED, true);
    taskConfig.set(TaskManagerOptions.BUFFER_DEBLOAT_PERIOD, Duration.ofMinutes(10));
    StreamConfig streamConfig = testHarness.getStreamConfig();
    streamConfig.setChainIndex(0);
    streamConfig.setStreamOperator(new StreamMap<String, String>(new DummyMapFunction<>()));
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    return testHarness;
}

Example 80 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamOperatorChainingTest method testMultiChaining.

/**
 * Verify that multi-chaining works.
 */
private void testMultiChaining(StreamExecutionEnvironment env) throws Exception {
    // set parallelism to 2 to avoid chaining with source in case when available processors is
    // 1.
    env.setParallelism(2);
    // the actual elements will not be used
    DataStream<Integer> input = env.fromElements(1, 2, 3);
    sink1Results = new ArrayList<>();
    sink2Results = new ArrayList<>();
    input = input.map(value -> value);
    input.map(value -> "First: " + value).addSink(new SinkFunction<String>() {

        @Override
        public void invoke(String value, Context ctx) throws Exception {
            sink1Results.add(value);
        }
    });
    input.map(value -> "Second: " + value).addSink(new SinkFunction<String>() {

        @Override
        public void invoke(String value, Context ctx) throws Exception {
            sink2Results.add(value);
        }
    });
    // be build our own StreamTask and OperatorChain
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    Assert.assertTrue(jobGraph.getVerticesSortedTopologicallyFromSources().size() == 2);
    JobVertex chainedVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    Configuration configuration = chainedVertex.getConfiguration();
    StreamConfig streamConfig = new StreamConfig(configuration);
    StreamMap<Integer, Integer> headOperator = streamConfig.getStreamOperator(Thread.currentThread().getContextClassLoader());
    try (MockEnvironment environment = createMockEnvironment(chainedVertex.getName())) {
        StreamTask<Integer, StreamMap<Integer, Integer>> mockTask = createMockTask(streamConfig, environment);
        OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = createOperatorChain(streamConfig, environment, mockTask);
        headOperator.setup(mockTask, streamConfig, operatorChain.getMainOperatorOutput());
        operatorChain.initializeStateAndOpenOperators(null);
        headOperator.processElement(new StreamRecord<>(1));
        headOperator.processElement(new StreamRecord<>(2));
        headOperator.processElement(new StreamRecord<>(3));
        assertThat(sink1Results, contains("First: 1", "First: 2", "First: 3"));
        assertThat(sink2Results, contains("Second: 1", "Second: 2", "Second: 3"));
    }
}