Examples with DataStream org.apache.flink.streaming.api.datastream.DataStream used on opensource projects

Example 26 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StreamGraphGeneratorTest method testSetSlotSharingResource.

@Test
public void testSetSlotSharingResource() {
    final String slotSharingGroup1 = "a";
    final String slotSharingGroup2 = "b";
    final ResourceProfile resourceProfile1 = ResourceProfile.fromResources(1, 10);
    final ResourceProfile resourceProfile2 = ResourceProfile.fromResources(2, 20);
    final ResourceProfile resourceProfile3 = ResourceProfile.fromResources(3, 30);
    final Map<String, ResourceProfile> slotSharingGroupResource = new HashMap<>();
    slotSharingGroupResource.put(slotSharingGroup1, resourceProfile1);
    slotSharingGroupResource.put(slotSharingGroup2, resourceProfile2);
    slotSharingGroupResource.put(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP, resourceProfile3);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3).slotSharingGroup(slotSharingGroup1);
    final DataStream<Integer> mapDataStream1 = sourceDataStream.map(x -> x + 1).slotSharingGroup(slotSharingGroup2);
    final DataStream<Integer> mapDataStream2 = mapDataStream1.map(x -> x * 2);
    final List<Transformation<?>> transformations = new ArrayList<>();
    transformations.add(sourceDataStream.getTransformation());
    transformations.add(mapDataStream1.getTransformation());
    transformations.add(mapDataStream2.getTransformation());
    // all stream nodes share default group by default
    final StreamGraph streamGraph = new StreamGraphGenerator(transformations, env.getConfig(), env.getCheckpointConfig()).setSlotSharingGroupResource(slotSharingGroupResource).generate();
    assertThat(streamGraph.getSlotSharingGroupResource(slotSharingGroup1).get(), equalTo(resourceProfile1));
    assertThat(streamGraph.getSlotSharingGroupResource(slotSharingGroup2).get(), equalTo(resourceProfile2));
    assertThat(streamGraph.getSlotSharingGroupResource(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP).get(), equalTo(resourceProfile3));
}

Example 27 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StreamGraphGeneratorTest method testAutoMaxParallelism.

/**
 * Tests that the max parallelism is automatically set to the parallelism if it has not been
 * specified.
 */
@Test
public void testAutoMaxParallelism() {
    int globalParallelism = 42;
    int mapParallelism = 17;
    int maxParallelism = 21;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(globalParallelism);
    DataStream<Integer> source = env.fromElements(1, 2, 3);
    DataStream<Integer> keyedResult1 = source.keyBy(value -> value).map(new NoOpIntMap());
    DataStream<Integer> keyedResult2 = keyedResult1.keyBy(value -> value).map(new NoOpIntMap()).setParallelism(mapParallelism);
    DataStream<Integer> keyedResult3 = keyedResult2.keyBy(value -> value).map(new NoOpIntMap()).setMaxParallelism(maxParallelism);
    DataStream<Integer> keyedResult4 = keyedResult3.keyBy(value -> value).map(new NoOpIntMap()).setMaxParallelism(maxParallelism).setParallelism(mapParallelism);
    keyedResult4.addSink(new DiscardingSink<>());
    StreamGraph graph = env.getStreamGraph();
    StreamNode keyedResult3Node = graph.getStreamNode(keyedResult3.getId());
    StreamNode keyedResult4Node = graph.getStreamNode(keyedResult4.getId());
    assertEquals(maxParallelism, keyedResult3Node.getMaxParallelism());
    assertEquals(maxParallelism, keyedResult4Node.getMaxParallelism());
}

Example 28 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StreamGraphGeneratorTest method testUnalignedCheckpointDisabledOnPointwise.

@Test
public void testUnalignedCheckpointDisabledOnPointwise() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(42);
    DataStream<Long> source1 = env.fromSequence(1L, 10L);
    DataStream<Long> map1 = source1.forward().map(l -> l);
    DataStream<Long> source2 = env.fromSequence(2L, 11L);
    DataStream<Long> map2 = source2.shuffle().map(l -> l);
    final MapStateDescriptor<Long, Long> descriptor = new MapStateDescriptor<>("broadcast", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.LONG_TYPE_INFO);
    final BroadcastStream<Long> broadcast = map1.broadcast(descriptor);
    final SingleOutputStreamOperator<Long> joined = map2.connect(broadcast).process(new BroadcastProcessFunction<Long, Long, Long>() {

        @Override
        public void processElement(Long value, ReadOnlyContext ctx, Collector<Long> out) {
        }

        @Override
        public void processBroadcastElement(Long value, Context ctx, Collector<Long> out) {
        }
    });
    DataStream<Long> map3 = joined.shuffle().map(l -> l);
    DataStream<Long> map4 = map3.rescale().map(l -> l).setParallelism(1337);
    StreamGraph streamGraph = env.getStreamGraph();
    assertEquals(7, streamGraph.getStreamNodes().size());
    // forward
    assertThat(edge(streamGraph, source1, map1), supportsUnalignedCheckpoints(false));
    // shuffle
    assertThat(edge(streamGraph, source2, map2), supportsUnalignedCheckpoints(true));
    // broadcast, but other channel is forwarded
    assertThat(edge(streamGraph, map1, joined), supportsUnalignedCheckpoints(false));
    // forward
    assertThat(edge(streamGraph, map2, joined), supportsUnalignedCheckpoints(false));
    // shuffle
    assertThat(edge(streamGraph, joined, map3), supportsUnalignedCheckpoints(true));
    // rescale
    assertThat(edge(streamGraph, map3, map4), supportsUnalignedCheckpoints(false));
}

Example 29 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StreamGraphGeneratorTest method testSetupOfKeyGroupPartitioner.

/**
 * Tests that the KeyGroupStreamPartitioner are properly set up with the correct value of
 * maximum parallelism.
 */
@Test
public void testSetupOfKeyGroupPartitioner() {
    int maxParallelism = 42;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setMaxParallelism(maxParallelism);
    DataStream<Integer> source = env.fromElements(1, 2, 3);
    DataStream<Integer> keyedResult = source.keyBy(value -> value).map(new NoOpIntMap());
    keyedResult.addSink(new DiscardingSink<>());
    StreamGraph graph = env.getStreamGraph();
    StreamNode keyedResultNode = graph.getStreamNode(keyedResult.getId());
    StreamPartitioner<?> streamPartitioner = keyedResultNode.getInEdges().get(0).getPartitioner();
}

Example 30 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StateDescriptorPassingTest method testReduceWindowState.

@Test
public void testReduceWindowState() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.registerTypeWithKryoSerializer(File.class, JavaSerializer.class);
    DataStream<File> src = env.fromElements(new File("/")).assignTimestampsAndWatermarks(WatermarkStrategy.<File>forMonotonousTimestamps().withTimestampAssigner((file, ts) -> System.currentTimeMillis()));
    SingleOutputStreamOperator<?> result = src.keyBy(new KeySelector<File, String>() {

        @Override
        public String getKey(File value) {
            return null;
        }
    }).window(TumblingEventTimeWindows.of(Time.milliseconds(1000))).reduce(new ReduceFunction<File>() {

        @Override
        public File reduce(File value1, File value2) {
            return null;
        }
    });
    validateStateDescriptorConfigured(result);
}