Examples with NumberSequenceSource org.apache.flink.api.connector.source.lib.NumberSequenceSource used on opensource projects

Example 6 with NumberSequenceSource

use of org.apache.flink.api.connector.source.lib.NumberSequenceSource in project flink by apache.

the class StreamingJobGraphGeneratorTest method createGraphWithMultipleInputs.

public JobGraph createGraphWithMultipleInputs(boolean chain, String... inputNames) {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    MultipleInputTransformation<Long> transform = new MultipleInputTransformation<>("mit", new UnusedOperatorFactory(), Types.LONG, env.getParallelism());
    Arrays.stream(inputNames).map(name -> env.fromSource(new NumberSequenceSource(1, 2), WatermarkStrategy.noWatermarks(), name).getTransformation()).forEach(transform::addInput);
    transform.setChainingStrategy(chain ? ChainingStrategy.HEAD_WITH_SOURCES : ChainingStrategy.NEVER);
    env.addOperator(transform);
    return StreamingJobGraphGenerator.createJobGraph(env.getStreamGraph());
}

Example 7 with NumberSequenceSource

use of org.apache.flink.api.connector.source.lib.NumberSequenceSource in project flink by apache.

the class AlignedWatermarksITCase method getJobGraph.

private JobGraph getJobGraph() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setAutoWatermarkInterval(0L);
    env.setParallelism(1);
    DataStream<Long> slowSource = env.fromSource(new NumberSequenceSource(0, Long.MAX_VALUE), WatermarkStrategy.forGenerator(ctx -> new PunctuatedGenerator()).withWatermarkAlignment("group-1", Duration.ofMillis(MAX_DRIFT), UPDATE_INTERVAL).withTimestampAssigner((r, t) -> r), SLOW_SOURCE_NAME).map(new RichMapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            Thread.sleep(10);
            return value;
        }
    });
    DataStream<Long> fastSource = env.fromSource(new NumberSequenceSource(0, Long.MAX_VALUE), WatermarkStrategy.forGenerator(ctx -> new PunctuatedGenerator()).withWatermarkAlignment("group-1", Duration.ofMillis(MAX_DRIFT), UPDATE_INTERVAL).withTimestampAssigner((r, t) -> r), FAST_SOURCE_NAME).map(new RichMapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            Thread.sleep(1);
            return value;
        }
    });
    slowSource.union(fastSource).addSink(new DiscardingSink<>());
    return env.getStreamGraph().getJobGraph();
}

Example 8 with NumberSequenceSource

use of org.apache.flink.api.connector.source.lib.NumberSequenceSource in project flink by apache.

the class SourceNAryInputChainingITCase method createProgramWithUnionInput.

/**
 * Creates a DataStream program as shown below.
 *
 * <pre>
 *                                   +--------------+
 *             (src 1) --> (map) --> |              |
 *                                   |              |
 *            (src 2) --+            |    N-Ary     |
 *                      +-- UNION -> |   Operator   |
 *   (src 3) -> (map) --+            |              |
 *                                   |              |
 *                       (src 4) --> |              |
 *                                   +--------------+
 * </pre>
 */
private DataStream<Long> createProgramWithUnionInput() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(PARALLELISM);
    env.getConfig().enableObjectReuse();
    final DataStream<Long> source1 = env.fromSource(new NumberSequenceSource(1L, 10L), WatermarkStrategy.noWatermarks(), "source-1");
    final DataStream<Long> source2 = env.fromSource(new NumberSequenceSource(11L, 20L), WatermarkStrategy.noWatermarks(), "source-2");
    final DataStream<Long> source3 = env.fromSource(new NumberSequenceSource(21L, 30L), WatermarkStrategy.noWatermarks(), "source-3");
    final DataStream<Long> source4 = env.fromSource(new NumberSequenceSource(31L, 40L), WatermarkStrategy.noWatermarks(), "source-4");
    return nAryInputStreamOperation(source1.map((v) -> v), source2.union(source3), source4);
}

Example 9 with NumberSequenceSource

use of org.apache.flink.api.connector.source.lib.NumberSequenceSource in project flink by apache.

the class SourceNAryInputChainingITCase method createProgramWithSourcesOnly.

/**
 * Creates a DataStream program as shown below.
 *
 * <pre>
 *               +--------------+
 *   (src 1) --> |              |
 *               |     N-Ary    |
 *   (src 2) --> |              |
 *               |   Operator   |
 *   (src 3) --> |              |
 *               +--------------+
 * </pre>
 */
private DataStream<Long> createProgramWithSourcesOnly() {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(PARALLELISM);
    env.getConfig().enableObjectReuse();
    final DataStream<Long> source1 = env.fromSource(new NumberSequenceSource(1L, 10L), WatermarkStrategy.noWatermarks(), "source-1");
    final DataStream<Long> source2 = env.fromSource(new NumberSequenceSource(11L, 20L), WatermarkStrategy.noWatermarks(), "source-2");
    final DataStream<Long> source3 = env.fromSource(new NumberSequenceSource(21L, 30L), WatermarkStrategy.noWatermarks(), "source-3");
    return nAryInputStreamOperation(source1, source2, source3);
}