Examples with StreamConfig org.apache.flink.streaming.api.graph.StreamConfig used on opensource projects

Example 56 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskTimerTest method checkScheduledTimestampe.

@Test
public void checkScheduledTimestampe() {
    try {
        final OneInputStreamTask<String, String> mapTask = new OneInputStreamTask<>();
        final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(mapTask, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
        testHarness.setupOutputForSingletonOperatorChain();
        StreamConfig streamConfig = testHarness.getStreamConfig();
        StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
        streamConfig.setStreamOperator(mapOperator);
        testHarness.invoke();
        testHarness.waitForTaskRunning();
        final AtomicReference<Throwable> errorRef = new AtomicReference<>();
        final long t1 = System.currentTimeMillis();
        final long t2 = System.currentTimeMillis() - 200;
        final long t3 = System.currentTimeMillis() + 100;
        final long t4 = System.currentTimeMillis() + 200;
        ProcessingTimeService timeService = mapTask.getProcessingTimeService();
        timeService.registerTimer(t1, new ValidatingProcessingTimeCallback(errorRef, t1, 0));
        timeService.registerTimer(t2, new ValidatingProcessingTimeCallback(errorRef, t2, 1));
        timeService.registerTimer(t3, new ValidatingProcessingTimeCallback(errorRef, t3, 2));
        timeService.registerTimer(t4, new ValidatingProcessingTimeCallback(errorRef, t4, 3));
        long deadline = System.currentTimeMillis() + 20000;
        while (errorRef.get() == null && ValidatingProcessingTimeCallback.numInSequence < 4 && System.currentTimeMillis() < deadline) {
            Thread.sleep(100);
        }
        // handle errors
        if (errorRef.get() != null) {
            errorRef.get().printStackTrace();
            fail(errorRef.get().getMessage());
        }
        assertEquals(4, ValidatingProcessingTimeCallback.numInSequence);
        testHarness.endInput();
        testHarness.waitForTaskCompletion();
        // wait until the trigger thread is shut down. otherwise, the other tests may become unstable
        deadline = System.currentTimeMillis() + 4000;
        while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
            Thread.sleep(10);
        }
        assertEquals("Trigger timer thread did not properly shut down", 0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 57 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StreamTaskTimerTest method testOpenCloseAndTimestamps.

@Test
public void testOpenCloseAndTimestamps() throws Exception {
    final OneInputStreamTask<String, String> mapTask = new OneInputStreamTask<>();
    final OneInputStreamTaskTestHarness<String, String> testHarness = new OneInputStreamTaskTestHarness<>(mapTask, BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    testHarness.setupOutputForSingletonOperatorChain();
    StreamConfig streamConfig = testHarness.getStreamConfig();
    StreamMap<String, String> mapOperator = new StreamMap<>(new DummyMapFunction<String>());
    streamConfig.setStreamOperator(mapOperator);
    testHarness.invoke();
    testHarness.waitForTaskRunning();
    // first one spawns thread
    mapTask.getProcessingTimeService().registerTimer(System.currentTimeMillis(), new ProcessingTimeCallback() {

        @Override
        public void onProcessingTime(long timestamp) {
        }
    });
    assertEquals(1, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
    testHarness.endInput();
    testHarness.waitForTaskCompletion();
    // thread needs to die in time
    long deadline = System.currentTimeMillis() + 4000;
    while (StreamTask.TRIGGER_THREAD_GROUP.activeCount() > 0 && System.currentTimeMillis() < deadline) {
        Thread.sleep(10);
    }
    assertEquals("Trigger timer thread did not properly shut down", 0, StreamTask.TRIGGER_THREAD_GROUP.activeCount());
}

Example 58 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class BootstrapTransformationTest method testStreamConfig.

@Test
public void testStreamConfig() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSource<String> input = env.fromElements("");
    BootstrapTransformation<String> transformation = OperatorTransformation.bootstrapWith(input).keyBy(new CustomKeySelector()).transform(new ExampleKeyedStateBootstrapFunction());
    StreamConfig config = transformation.getConfig(OperatorIDGenerator.fromUid("uid"), new MemoryStateBackend(), new Configuration(), null);
    KeySelector selector = config.getStatePartitioner(0, Thread.currentThread().getContextClassLoader());
    Assert.assertEquals("Incorrect key selector forwarded to stream operator", CustomKeySelector.class, selector.getClass());
}

Example 59 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class BootstrapTransformation method writeOperatorSubtaskStates.

private MapPartitionOperator<T, TaggedOperatorSubtaskState> writeOperatorSubtaskStates(OperatorID operatorID, @Nullable StateBackend stateBackend, Configuration additionalConfig, Path savepointPath, int localMaxParallelism) {
    DataSet<T> input = dataSet;
    if (originalKeySelector != null) {
        input = dataSet.partitionCustom(new KeyGroupRangePartitioner(localMaxParallelism), hashKeySelector);
    }
    StreamOperator<TaggedOperatorSubtaskState> operator = factory.createOperator(System.currentTimeMillis(), savepointPath);
    operator = dataSet.clean(operator);
    final StreamConfig config = getConfig(operatorID, stateBackend, additionalConfig, operator);
    BoundedOneInputStreamTaskRunner<T> operatorRunner = new BoundedOneInputStreamTaskRunner<>(config, localMaxParallelism, timestamper);
    MapPartitionOperator<T, TaggedOperatorSubtaskState> subtaskStates = input.mapPartition(operatorRunner).name(operatorID.toHexString());
    if (operator instanceof BroadcastStateBootstrapOperator) {
        subtaskStates = subtaskStates.setParallelism(1);
    } else {
        int currentParallelism = getParallelism(subtaskStates);
        if (currentParallelism > localMaxParallelism) {
            subtaskStates.setParallelism(localMaxParallelism);
        }
    }
    return subtaskStates;
}

Example 60 with StreamConfig

use of org.apache.flink.streaming.api.graph.StreamConfig in project flink by apache.

the class StateBootstrapTransformation method writeOperatorSubtaskStates.

private SingleOutputStreamOperator<TaggedOperatorSubtaskState> writeOperatorSubtaskStates(OperatorID operatorID, StateBackend stateBackend, Configuration additionalConfig, Path savepointPath, int localMaxParallelism) {
    StreamOperator<TaggedOperatorSubtaskState> operator = factory.createOperator(System.currentTimeMillis(), savepointPath);
    operator = stream.getExecutionEnvironment().clean(operator);
    final StreamConfig config = getConfig(operatorID, stateBackend, additionalConfig, operator);
    BootstrapStreamTaskRunner<T> operatorRunner = new BootstrapStreamTaskRunner<>(config, localMaxParallelism);
    DataStream<T> input = stream;
    if (keySelector != null) {
        input = stream.keyBy(this.keySelector);
    }
    SingleOutputStreamOperator<TaggedOperatorSubtaskState> subtaskStates = input.transform(operatorID.toHexString(), TypeInformation.of(TaggedOperatorSubtaskState.class), operatorRunner).setMaxParallelism(localMaxParallelism);
    if (operator instanceof BroadcastStateBootstrapOperator) {
        subtaskStates = subtaskStates.setParallelism(1);
    } else {
        int currentParallelism = getParallelism(subtaskStates);
        if (currentParallelism > localMaxParallelism) {
            subtaskStates.setParallelism(localMaxParallelism);
        }
    }
    return subtaskStates;
}