Examples with DataStream org.apache.flink.streaming.api.datastream.DataStream used on opensource projects

Example 66 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class StreamOperatorChainingTest method testMultiChaining.

/**
 * Verify that multi-chaining works.
 */
private void testMultiChaining(StreamExecutionEnvironment env) throws Exception {
    // set parallelism to 2 to avoid chaining with source in case when available processors is
    // 1.
    env.setParallelism(2);
    // the actual elements will not be used
    DataStream<Integer> input = env.fromElements(1, 2, 3);
    sink1Results = new ArrayList<>();
    sink2Results = new ArrayList<>();
    input = input.map(value -> value);
    input.map(value -> "First: " + value).addSink(new SinkFunction<String>() {

        @Override
        public void invoke(String value, Context ctx) throws Exception {
            sink1Results.add(value);
        }
    });
    input.map(value -> "Second: " + value).addSink(new SinkFunction<String>() {

        @Override
        public void invoke(String value, Context ctx) throws Exception {
            sink2Results.add(value);
        }
    });
    // be build our own StreamTask and OperatorChain
    JobGraph jobGraph = env.getStreamGraph().getJobGraph();
    Assert.assertTrue(jobGraph.getVerticesSortedTopologicallyFromSources().size() == 2);
    JobVertex chainedVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    Configuration configuration = chainedVertex.getConfiguration();
    StreamConfig streamConfig = new StreamConfig(configuration);
    StreamMap<Integer, Integer> headOperator = streamConfig.getStreamOperator(Thread.currentThread().getContextClassLoader());
    try (MockEnvironment environment = createMockEnvironment(chainedVertex.getName())) {
        StreamTask<Integer, StreamMap<Integer, Integer>> mockTask = createMockTask(streamConfig, environment);
        OperatorChain<Integer, StreamMap<Integer, Integer>> operatorChain = createOperatorChain(streamConfig, environment, mockTask);
        headOperator.setup(mockTask, streamConfig, operatorChain.getMainOperatorOutput());
        operatorChain.initializeStateAndOpenOperators(null);
        headOperator.processElement(new StreamRecord<>(1));
        headOperator.processElement(new StreamRecord<>(2));
        headOperator.processElement(new StreamRecord<>(3));
        assertThat(sink1Results, contains("First: 1", "First: 2", "First: 3"));
        assertThat(sink2Results, contains("Second: 1", "Second: 2", "Second: 3"));
    }
}

Example 67 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class DataStreamTest method testErgonomicWatermarkStrategy.

/**
 * Ensure that WatermarkStrategy is easy to use in the API, without superfluous generics.
 */
@Test
public void testErgonomicWatermarkStrategy() {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<String> input = env.fromElements("bonjour");
    // as soon as you have a chain of methods the first call needs a generic
    input.assignTimestampsAndWatermarks(WatermarkStrategy.forBoundedOutOfOrderness(Duration.ofMillis(10)));
    // as soon as you have a chain of methods the first call needs to specify the generic type
    input.assignTimestampsAndWatermarks(WatermarkStrategy.<String>forBoundedOutOfOrderness(Duration.ofMillis(10)).withTimestampAssigner((event, timestamp) -> 42L));
}

Example 68 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class DataStreamTest method testFailedTranslationOnKeyed.

/**
 * Tests that with a {@link KeyedStream} we have to provide a {@link
 * KeyedBroadcastProcessFunction}.
 */
@Test
public void testFailedTranslationOnKeyed() {
    final MapStateDescriptor<Long, String> descriptor = new MapStateDescriptor<>("broadcast", BasicTypeInfo.LONG_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final DataStream<Long> srcOne = env.generateSequence(0L, 5L).assignTimestampsAndWatermarks(new CustomWmEmitter<Long>() {

        @Override
        public long extractTimestamp(Long element, long previousElementTimestamp) {
            return element;
        }
    }).keyBy((KeySelector<Long, Long>) value -> value);
    final DataStream<String> srcTwo = env.fromElements("Test:0", "Test:1", "Test:2", "Test:3", "Test:4", "Test:5").assignTimestampsAndWatermarks(new CustomWmEmitter<String>() {

        @Override
        public long extractTimestamp(String element, long previousElementTimestamp) {
            return Long.parseLong(element.split(":")[1]);
        }
    });
    BroadcastStream<String> broadcast = srcTwo.broadcast(descriptor);
    BroadcastConnectedStream<Long, String> bcStream = srcOne.connect(broadcast);
    expectedException.expect(IllegalArgumentException.class);
    bcStream.process(new BroadcastProcessFunction<Long, String, String>() {

        @Override
        public void processBroadcastElement(String value, Context ctx, Collector<String> out) throws Exception {
        // do nothing
        }

        @Override
        public void processElement(Long value, ReadOnlyContext ctx, Collector<String> out) throws Exception {
        // do nothing
        }
    });
}

Example 69 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class DataStreamTest method testWindowOperatorDescription.

/**
 * Tests that verifies window operator has different name and description.
 */
@Test
public void testWindowOperatorDescription() {
    // global window
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Long> dataStream1 = env.generateSequence(0, 0).windowAll(GlobalWindows.create()).trigger(PurgingTrigger.of(CountTrigger.of(10))).reduce(new ReduceFunction<Long>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Long reduce(Long value1, Long value2) throws Exception {
            return null;
        }
    });
    // name is simplified
    assertEquals("GlobalWindows", dataStream1.getTransformation().getName());
    // description contains detail of function:
    // TriggerWindow(GlobalWindows(), ReducingStateDescriptor{name=window-contents,
    // defaultValue=null,
    // serializer=org.apache.flink.api.common.typeutils.base.LongSerializer@6af9fcb2},
    // PurgingTrigger(CountTrigger(10)), AllWindowedStream.reduce(AllWindowedStream.java:229))
    assertTrue(dataStream1.getTransformation().getDescription().contains("PurgingTrigger"));
    // keyed window
    DataStream<Long> dataStream2 = env.generateSequence(0, 0).keyBy(value -> value).window(TumblingEventTimeWindows.of(Time.milliseconds(1000))).trigger(PurgingTrigger.of(CountTrigger.of(10))).reduce(new ReduceFunction<Long>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Long reduce(Long value1, Long value2) throws Exception {
            return null;
        }
    });
    // name is simplified
    assertEquals("TumblingEventTimeWindows", dataStream2.getTransformation().getName());
    // description contains detail of function:
    // Window(TumblingEventTimeWindows(1000), PurgingTrigger, ReduceFunction$36,
    // PassThroughWindowFunction)
    assertTrue(dataStream2.getTransformation().getDescription().contains("PurgingTrigger"));
}

Example 70 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class DataStreamTest method testEnumKeyRejection.

@Test
public void testEnumKeyRejection() {
    KeySelector<Tuple2<TestEnum, String>, TestEnum> keySelector = value -> value.f0;
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Tuple2<TestEnum, String>> input = env.fromElements(Tuple2.of(TestEnum.FOO, "Foo"), Tuple2.of(TestEnum.BAR, "Bar"));
    expectedException.expect(InvalidProgramException.class);
    expectedException.expectMessage(new StringStartsWith("Type " + EnumTypeInfo.of(TestEnum.class) + " cannot be used as key."));
    input.keyBy(keySelector);
}