Examples with SingleOutputStreamOperator org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator used on opensource projects

Example 1 with SingleOutputStreamOperator

use of org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator in project flink by apache.

the class StreamGraphGeneratorTest method testUnalignedCheckpointDisabledOnPointwise.

@Test
public void testUnalignedCheckpointDisabledOnPointwise() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(42);
    DataStream<Long> source1 = env.fromSequence(1L, 10L);
    DataStream<Long> map1 = source1.forward().map(l -> l);
    DataStream<Long> source2 = env.fromSequence(2L, 11L);
    DataStream<Long> map2 = source2.shuffle().map(l -> l);
    final MapStateDescriptor<Long, Long> descriptor = new MapStateDescriptor<>("broadcast", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
    final BroadcastStream<Long> broadcast = map1.broadcast(descriptor);
    final SingleOutputStreamOperator<Long> joined = map2.connect(broadcast).process(new BroadcastProcessFunction<Long, Long, Long>() {

        @Override
        public void processElement(Long value, ReadOnlyContext ctx, Collector<Long> out) {
        }

        @Override
        public void processBroadcastElement(Long value, Context ctx, Collector<Long> out) {
        }
    });
    DataStream<Long> map3 = joined.shuffle().map(l -> l);
    DataStream<Long> map4 = map3.rescale().map(l -> l).setParallelism(1337);
    StreamGraph streamGraph = env.getStreamGraph();
    assertEquals(7, streamGraph.getStreamNodes().size());
    // forward
    assertThat(edge(streamGraph, source1, map1), supportsUnalignedCheckpoints(false));
    // shuffle
    assertThat(edge(streamGraph, source2, map2), supportsUnalignedCheckpoints(true));
    // broadcast, but other channel is forwarded
    assertThat(edge(streamGraph, map1, joined), supportsUnalignedCheckpoints(false));
    // forward
    assertThat(edge(streamGraph, map2, joined), supportsUnalignedCheckpoints(false));
    // shuffle
    assertThat(edge(streamGraph, joined, map3), supportsUnalignedCheckpoints(true));
    // rescale
    assertThat(edge(streamGraph, map3, map4), supportsUnalignedCheckpoints(false));
}

Example 2 with SingleOutputStreamOperator

use of org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator in project flink by apache.

the class StateDescriptorPassingTest method testReduceWindowState.

@Test
public void testReduceWindowState() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);
    DataStream<File> src = env.fromElements(new File("/")).assignTimestampsAndWatermarks(WatermarkStrategy.<File>forMonotonousTimestamps().withTimestampAssigner((file, ts) -> System.currentTimeMillis()));
    SingleOutputStreamOperator<?> result = src.keyBy(new KeySelector<File, String>() {

        @Override
        public String getKey(File value) {
            return null;
        }
    }).window(TumblingEventTimeWindows.of(Time.milliseconds(1000))).reduce(new ReduceFunction<File>() {

        @Override
        public File reduce(File value1, File value2) {
            return null;
        }
    });
    validateStateDescriptorConfigured(result);
}

Example 3 with SingleOutputStreamOperator

use of org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator in project flink by apache.

the class StateDescriptorPassingTest method testApplyWindowAllState.

@Test
public void testApplyWindowAllState() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);
    // simulate ingestion time
    DataStream<File> src = env.fromElements(new File("/")).assignTimestampsAndWatermarks(WatermarkStrategy.<File>forMonotonousTimestamps().withTimestampAssigner((file, ts) -> System.currentTimeMillis()));
    SingleOutputStreamOperator<?> result = src.windowAll(TumblingEventTimeWindows.of(Time.milliseconds(1000))).apply(new AllWindowFunction<File, String, TimeWindow>() {

        @Override
        public void apply(TimeWindow window, Iterable<File> input, Collector<String> out) {
        }
    });
    validateListStateDescriptorConfigured(result);
}

Example 4 with SingleOutputStreamOperator

use of org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator in project flink by apache.

the class StateDescriptorPassingTest method testApplyWindowState.

@Test
public void testApplyWindowState() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);
    DataStream<File> src = env.fromElements(new File("/")).assignTimestampsAndWatermarks(WatermarkStrategy.<File>forMonotonousTimestamps().withTimestampAssigner((file, ts) -> System.currentTimeMillis()));
    SingleOutputStreamOperator<?> result = src.keyBy(new KeySelector<File, String>() {

        @Override
        public String getKey(File value) {
            return null;
        }
    }).window(TumblingEventTimeWindows.of(Time.milliseconds(1000))).apply(new WindowFunction<File, String, String, TimeWindow>() {

        @Override
        public void apply(String s, TimeWindow window, Iterable<File> input, Collector<String> out) {
        }
    });
    validateListStateDescriptorConfigured(result);
}

Example 5 with SingleOutputStreamOperator

use of org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator in project flink by apache.

the class StateDescriptorPassingTest method testProcessWindowState.

@Test
public void testProcessWindowState() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);
    DataStream<File> src = env.fromElements(new File("/")).assignTimestampsAndWatermarks(WatermarkStrategy.<File>forMonotonousTimestamps().withTimestampAssigner((file, ts) -> System.currentTimeMillis()));
    SingleOutputStreamOperator<?> result = src.keyBy(new KeySelector<File, String>() {

        @Override
        public String getKey(File value) {
            return null;
        }
    }).window(TumblingEventTimeWindows.of(Time.milliseconds(1000))).process(new ProcessWindowFunction<File, String, String, TimeWindow>() {

        @Override
        public void process(String s, Context ctx, Iterable<File> input, Collector<String> out) {
        }
    });
    validateListStateDescriptorConfigured(result);
}