Examples with DataStream org.apache.flink.streaming.api.datastream.DataStream used on opensource projects

Example 31 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StateDescriptorPassingTest method testApplyWindowAllState.

@Test
public void testApplyWindowAllState() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);
    // simulate ingestion time
    DataStream<File> src = env.fromElements(new File("/")).assignTimestampsAndWatermarks(WatermarkStrategy.<File>forMonotonousTimestamps().withTimestampAssigner((file, ts) -> System.currentTimeMillis()));
    SingleOutputStreamOperator<?> result = src.windowAll(TumblingEventTimeWindows.of(Time.milliseconds(1000))).apply(new AllWindowFunction<File, String, TimeWindow>() {

        @Override
        public void apply(TimeWindow window, Iterable<File> input, Collector<String> out) {
        }
    });
    validateListStateDescriptorConfigured(result);
}

Example 32 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StateDescriptorPassingTest method testApplyWindowState.

@Test
public void testApplyWindowState() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);
    DataStream<File> src = env.fromElements(new File("/")).assignTimestampsAndWatermarks(WatermarkStrategy.<File>forMonotonousTimestamps().withTimestampAssigner((file, ts) -> System.currentTimeMillis()));
    SingleOutputStreamOperator<?> result = src.keyBy(new KeySelector<File, String>() {

        @Override
        public String getKey(File value) {
            return null;
        }
    }).window(TumblingEventTimeWindows.of(Time.milliseconds(1000))).apply(new WindowFunction<File, String, String, TimeWindow>() {

        @Override
        public void apply(String s, TimeWindow window, Iterable<File> input, Collector<String> out) {
        }
    });
    validateListStateDescriptorConfigured(result);
}

Example 33 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StateDescriptorPassingTest method testProcessWindowState.

@Test
public void testProcessWindowState() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);
    DataStream<File> src = env.fromElements(new File("/")).assignTimestampsAndWatermarks(WatermarkStrategy.<File>forMonotonousTimestamps().withTimestampAssigner((file, ts) -> System.currentTimeMillis()));
    SingleOutputStreamOperator<?> result = src.keyBy(new KeySelector<File, String>() {

        @Override
        public String getKey(File value) {
            return null;
        }
    }).window(TumblingEventTimeWindows.of(Time.milliseconds(1000))).process(new ProcessWindowFunction<File, String, String, TimeWindow>() {

        @Override
        public void process(String s, Context ctx, Iterable<File> input, Collector<String> out) {
        }
    });
    validateListStateDescriptorConfigured(result);
}

Example 34 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StreamGraphGeneratorTest method testMaxParallelismWithConnectedKeyedStream.

/**
 * Tests that the max parallelism is properly set for connected streams.
 */
@Test
public void testMaxParallelismWithConnectedKeyedStream() {
    int maxParallelism = 42;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Integer> input1 = env.fromElements(1, 2, 3, 4).setMaxParallelism(128);
    DataStream<Integer> input2 = env.fromElements(1, 2, 3, 4).setMaxParallelism(129);
    env.getConfig().setMaxParallelism(maxParallelism);
    DataStream<Integer> keyedResult = input1.connect(input2).keyBy(value -> value, value -> value).map(new NoOpIntCoMap());
    keyedResult.addSink(new DiscardingSink<>());
    StreamGraph graph = env.getStreamGraph();
    StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId());
    StreamPartitioner<?> streamPartitioner1 = keyedResultNode.getInEdges().get(0).getPartitioner();
    StreamPartitioner<?> streamPartitioner2 = keyedResultNode.getInEdges().get(1).getPartitioner();
}

Example 35 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StreamGraphGeneratorTest method testConflictSlotSharingGroup.

@Test(expected = IllegalArgumentException.class)
public void testConflictSlotSharingGroup() {
    final SlotSharingGroup ssg = SlotSharingGroup.newBuilder("ssg").setCpuCores(1).setTaskHeapMemoryMB(100).build();
    final SlotSharingGroup ssgConflict = SlotSharingGroup.newBuilder("ssg").setCpuCores(2).setTaskHeapMemoryMB(200).build();
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Integer> source = env.fromElements(1).slotSharingGroup(ssg);
    source.map(value -> value).slotSharingGroup(ssgConflict).addSink(new DiscardingSink<>()).slotSharingGroup(ssgConflict);
    env.getStreamGraph();
}