Examples with DataStream org.apache.flink.streaming.api.datastream.DataStream used on opensource projects

Example 86 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class BlockingShuffleITCase method createJobGraph.

private JobGraph createJobGraph(int numRecordsToSend, boolean deletePartitionFile) {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0L));
    env.setBufferTimeout(-1);
    env.setParallelism(numTaskManagers * numSlotsPerTaskManager);
    DataStream<String> source = env.addSource(new StringSource(numRecordsToSend));
    source.rebalance().map((MapFunction<String, String>) value -> value).broadcast().addSink(new VerifySink(deletePartitionFile));
    StreamGraph streamGraph = env.getStreamGraph();
    streamGraph.setGlobalStreamExchangeMode(GlobalStreamExchangeMode.ALL_EDGES_BLOCKING);
    // a scheduler supporting batch jobs is required for this job graph, because it contains
    // blocking data exchanges.
    // The scheduler is selected based on the JobType.
    streamGraph.setJobType(JobType.BATCH);
    return StreamingJobGraphGenerator.createJobGraph(streamGraph);
}

Example 87 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class SourceNAryInputChainingITCase method createProgramWithUnionInput.

/**
 * Creates a DataStream program as shown below.
 *
 * <pre>
 *                                   +--------------+
 *             (src 1) --> (map) --> |              |
 *                                   |              |
 *            (src 2) --+            |    N-Ary     |
 *                      +-- UNION -> |   Operator   |
 *   (src 3) -> (map) --+            |              |
 *                                   |              |
 *                       (src 4) --> |              |
 *                                   +--------------+
 * </pre>
 */
private DataStream<Long> createProgramWithUnionInput() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(PARALLELISM);
    env.getConfig().enableObjectReuse();
    final DataStream<Long> source1 = env.fromSource(new NumberSequenceSource(1L, 10L), WatermarkStrategy.noWatermarks(), "source-1");
    final DataStream<Long> source2 = env.fromSource(new NumberSequenceSource(11L, 20L), WatermarkStrategy.noWatermarks(), "source-2");
    final DataStream<Long> source3 = env.fromSource(new NumberSequenceSource(21L, 30L), WatermarkStrategy.noWatermarks(), "source-3");
    final DataStream<Long> source4 = env.fromSource(new NumberSequenceSource(31L, 40L), WatermarkStrategy.noWatermarks(), "source-4");
    return nAryInputStreamOperation(source1.map((v) -> v), source2.union(source3), source4);
}