Examples with DataStream org.apache.flink.streaming.api.datastream.DataStream used on opensource projects

Example 41 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class SelfConnectionITCase method differentDataStreamDifferentChain.

/**
 * We connect two different data streams in different chains to a CoMap. (This is not actually
 * self-connect.)
 */
@Test
public void differentDataStreamDifferentChain() throws Exception {
    TestListResultSink<String> resultSink = new TestListResultSink<>();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(3);
    DataStream<Integer> src = env.fromElements(1, 3, 5).disableChaining();
    DataStream<String> stringMap = src.flatMap(new FlatMapFunction<Integer, String>() {

        @Override
        public void flatMap(Integer value, Collector<String> out) throws Exception {
            out.collect("x " + value);
        }
    }).keyBy(String::length);
    DataStream<Long> longMap = src.map(value -> (long) (value + 1)).keyBy(Long::intValue);
    stringMap.connect(longMap).map(new CoMapFunction<String, Long, String>() {

        @Override
        public String map1(String value) {
            return value;
        }

        @Override
        public String map2(Long value) {
            return value.toString();
        }
    }).addSink(resultSink);
    env.execute();
    List<String> expected = Arrays.asList("x 1", "x 3", "x 5", "2", "4", "6");
    List<String> result = resultSink.getResult();
    Collections.sort(expected);
    Collections.sort(result);
    assertEquals(expected, result);
}

Example 42 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StreamingJobGraphGeneratorTest method testPartitionTypesInBatchMode.

@Test
public void testPartitionTypesInBatchMode() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setRuntimeMode(RuntimeExecutionMode.BATCH);
    env.setParallelism(4);
    env.disableOperatorChaining();
    DataStream<Integer> source = env.fromElements(1);
    source.map(value -> value).setParallelism(1).rescale().map(value -> value).rebalance().map(value -> value).keyBy(value -> value).map(value -> value).addSink(new DiscardingSink<>());
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertThat(verticesSorted.get(0), /* source - forward */
    hasOutputPartitionType(ResultPartitionType.BLOCKING));
    assertThat(verticesSorted.get(1), /* rescale */
    hasOutputPartitionType(ResultPartitionType.BLOCKING));
    assertThat(verticesSorted.get(2), /* rebalance */
    hasOutputPartitionType(ResultPartitionType.BLOCKING));
    assertThat(verticesSorted.get(3), /* keyBy */
    hasOutputPartitionType(ResultPartitionType.BLOCKING));
    assertThat(verticesSorted.get(4), /* forward - sink */
    hasOutputPartitionType(ResultPartitionType.BLOCKING));
}

Example 43 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StreamingJobGraphGeneratorTest method testExchangeModeUndefined.

/**
 * Test setting exchange mode to {@link StreamExchangeMode#UNDEFINED}.
 */
@Test
public void testExchangeModeUndefined() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // fromElements -> Map -> Print
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    DataStream<Integer> partitionAfterSourceDataStream = new DataStream<>(env, new PartitionTransformation<>(sourceDataStream.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.UNDEFINED));
    DataStream<Integer> mapDataStream = partitionAfterSourceDataStream.map(value -> value).setParallelism(1);
    DataStream<Integer> partitionAfterMapDataStream = new DataStream<>(env, new PartitionTransformation<>(mapDataStream.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.UNDEFINED));
    partitionAfterMapDataStream.print().setParallelism(2);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(2, verticesSorted.size());
    // it can be chained with UNDEFINED exchange mode
    JobVertex sourceAndMapVertex = verticesSorted.get(0);
    // UNDEFINED exchange mode is translated into PIPELINED_BOUNDED result partition by default
    assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceAndMapVertex.getProducedDataSets().get(0).getResultType());
}

Example 44 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StreamingJobGraphGeneratorTest method testExchangeModePipelined.

/**
 * Test setting exchange mode to {@link StreamExchangeMode#PIPELINED}.
 */
@Test
public void testExchangeModePipelined() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // fromElements -> Map -> Print
    DataStream<Integer> sourceDataStream = env.fromElements(1, 2, 3);
    DataStream<Integer> partitionAfterSourceDataStream = new DataStream<>(env, new PartitionTransformation<>(sourceDataStream.getTransformation(), new ForwardPartitioner<>(), StreamExchangeMode.PIPELINED));
    DataStream<Integer> mapDataStream = partitionAfterSourceDataStream.map(value -> value).setParallelism(1);
    DataStream<Integer> partitionAfterMapDataStream = new DataStream<>(env, new PartitionTransformation<>(mapDataStream.getTransformation(), new RescalePartitioner<>(), StreamExchangeMode.PIPELINED));
    partitionAfterMapDataStream.print().setParallelism(2);
    JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
    List<JobVertex> verticesSorted = jobGraph.getVerticesSortedTopologicallyFromSources();
    assertEquals(2, verticesSorted.size());
    // it can be chained with PIPELINED exchange mode
    JobVertex sourceAndMapVertex = verticesSorted.get(0);
    // PIPELINED exchange mode is translated into PIPELINED_BOUNDED result partition
    assertEquals(ResultPartitionType.PIPELINED_BOUNDED, sourceAndMapVertex.getProducedDataSets().get(0).getResultType());
}

Example 45 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StreamingJobGraphGeneratorTest method createJobGraphForManagedMemoryFractionTest.

private JobGraph createJobGraphForManagedMemoryFractionTest(final List<ResourceSpec> resourceSpecs, final List<Map<ManagedMemoryUseCase, Integer>> operatorScopeUseCaseWeights, final List<Set<ManagedMemoryUseCase>> slotScopeUseCases) throws Exception {
    final Method opMethod = getSetResourcesMethodAndSetAccessible(SingleOutputStreamOperator.class);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Integer> source = env.addSource(new ParallelSourceFunction<Integer>() {

        @Override
        public void run(SourceContext<Integer> ctx) {
        }

        @Override
        public void cancel() {
        }
    });
    opMethod.invoke(source, resourceSpecs.get(0));
    // CHAIN(source -> map1) in default slot sharing group
    final DataStream<Integer> map1 = source.map((MapFunction<Integer, Integer>) value -> value);
    opMethod.invoke(map1, resourceSpecs.get(1));
    // CHAIN(map2) in default slot sharing group
    final DataStream<Integer> map2 = map1.rebalance().map((MapFunction<Integer, Integer>) value -> value);
    opMethod.invoke(map2, resourceSpecs.get(2));
    // CHAIN(map3) in test slot sharing group
    final DataStream<Integer> map3 = map2.rebalance().map(value -> value).slotSharingGroup("test");
    opMethod.invoke(map3, resourceSpecs.get(3));
    declareManagedMemoryUseCaseForTranformation(source.getTransformation(), operatorScopeUseCaseWeights.get(0), slotScopeUseCases.get(0));
    declareManagedMemoryUseCaseForTranformation(map1.getTransformation(), operatorScopeUseCaseWeights.get(1), slotScopeUseCases.get(1));
    declareManagedMemoryUseCaseForTranformation(map2.getTransformation(), operatorScopeUseCaseWeights.get(2), slotScopeUseCases.get(2));
    declareManagedMemoryUseCaseForTranformation(map3.getTransformation(), operatorScopeUseCaseWeights.get(3), slotScopeUseCases.get(3));
    return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}