Examples with DataStream org.apache.flink.streaming.api.datastream.DataStream used on opensource projects

Example 11 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class CEPITCase method testSimpleAfterMatchSkip.

@Test
public void testSimpleAfterMatchSkip() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(envConfiguration);
    DataStream<Tuple2<Integer, String>> input = env.fromElements(new Tuple2<>(1, "a"), new Tuple2<>(2, "a"), new Tuple2<>(3, "a"), new Tuple2<>(4, "a"));
    Pattern<Tuple2<Integer, String>, ?> pattern = Pattern.<Tuple2<Integer, String>>begin("start", AfterMatchSkipStrategy.skipPastLastEvent()).where(new SimpleCondition<Tuple2<Integer, String>>() {

        @Override
        public boolean filter(Tuple2<Integer, String> rec) throws Exception {
            return rec.f1.equals("a");
        }
    }).times(2);
    PatternStream<Tuple2<Integer, String>> pStream = CEP.pattern(input, pattern).inProcessingTime();
    DataStream<Tuple2<Integer, String>> result = pStream.select(new PatternSelectFunction<Tuple2<Integer, String>, Tuple2<Integer, String>>() {

        @Override
        public Tuple2<Integer, String> select(Map<String, List<Tuple2<Integer, String>>> pattern) throws Exception {
            return pattern.get("start").get(0);
        }
    });
    List<Tuple2<Integer, String>> resultList = new ArrayList<>();
    DataStreamUtils.collect(result).forEachRemaining(resultList::add);
    resultList.sort(Comparator.comparing(tuple2 -> tuple2.toString()));
    List<Tuple2<Integer, String>> expected = Arrays.asList(Tuple2.of(1, "a"), Tuple2.of(3, "a"));
    assertEquals(expected, resultList);
}

Example 12 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class SavepointWriterTest method testCustomStateBackend.

@Test(expected = CustomStateBackendFactory.ExpectedException.class)
public void testCustomStateBackend() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    Configuration configuration = new Configuration();
    configuration.set(StateBackendOptions.STATE_BACKEND, CustomStateBackendFactory.class.getCanonicalName());
    configuration.set(ExecutionOptions.RUNTIME_MODE, RuntimeExecutionMode.BATCH);
    env.configure(configuration);
    DataStream<String> input = env.fromElements("");
    StateBootstrapTransformation<String> transformation = OperatorTransformation.bootstrapWith(input).keyBy(element -> element).transform(new Bootstrapper());
    SavepointWriter.newSavepoint(128).withOperator("uid", transformation).write("file:///tmp/path");
    env.execute();
}

Example 13 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class SavepointWriterWindowITCase method testTumbleWindowWithEvictor.

@Test
public void testTumbleWindowWithEvictor() throws Exception {
    final String savepointPath = getTempDirPath(new AbstractID().toHexString());
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
    DataStream<Tuple2<String, Integer>> bootstrapData = env.fromCollection(WORDS).map(word -> Tuple2.of(word, 1)).returns(TUPLE_TYPE_INFO).assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String, Integer>>noWatermarks().withTimestampAssigner((record, ts) -> 2L));
    WindowedStateTransformation<Tuple2<String, Integer>, String, TimeWindow> transformation = OperatorTransformation.bootstrapWith(bootstrapData).keyBy(tuple -> tuple.f0, Types.STRING).window(TumblingEventTimeWindows.of(Time.milliseconds(5))).evictor(CountEvictor.of(1));
    SavepointWriter.newSavepoint(stateBackend, 128).withOperator(UID, windowBootstrap.bootstrap(transformation)).write(savepointPath);
    env.execute("write state");
    WindowedStream<Tuple2<String, Integer>, String, TimeWindow> stream = env.addSource(new MaxWatermarkSource<Tuple2<String, Integer>>()).returns(TUPLE_TYPE_INFO).keyBy(tuple -> tuple.f0).window(TumblingEventTimeWindows.of(Time.milliseconds(5))).evictor(CountEvictor.of(1));
    DataStream<Tuple2<String, Integer>> windowed = windowStream.window(stream).uid(UID);
    CompletableFuture<Collection<Tuple2<String, Integer>>> future = collector.collect(windowed);
    submitJob(savepointPath, env);
    Collection<Tuple2<String, Integer>> results = future.get();
    Assert.assertThat("Incorrect results from bootstrapped windows", results, EVICTOR_MATCHER);
}

Example 14 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class SavepointWriterWindowITCase method testSlideWindowWithEvictor.

@Test
public void testSlideWindowWithEvictor() throws Exception {
    final String savepointPath = getTempDirPath(new AbstractID().toHexString());
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
    DataStream<Tuple2<String, Integer>> bootstrapData = env.fromCollection(WORDS).map(word -> Tuple2.of(word, 1)).returns(TUPLE_TYPE_INFO).assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String, Integer>>noWatermarks().withTimestampAssigner((record, ts) -> 2L));
    WindowedStateTransformation<Tuple2<String, Integer>, String, TimeWindow> transformation = OperatorTransformation.bootstrapWith(bootstrapData).keyBy(tuple -> tuple.f0, Types.STRING).window(SlidingEventTimeWindows.of(Time.milliseconds(5), Time.milliseconds(1))).evictor(CountEvictor.of(1));
    SavepointWriter.newSavepoint(stateBackend, 128).withOperator(UID, windowBootstrap.bootstrap(transformation)).write(savepointPath);
    env.execute("write state");
    WindowedStream<Tuple2<String, Integer>, String, TimeWindow> stream = env.addSource(new MaxWatermarkSource<Tuple2<String, Integer>>()).returns(TUPLE_TYPE_INFO).keyBy(tuple -> tuple.f0).window(SlidingEventTimeWindows.of(Time.milliseconds(5), Time.milliseconds(1))).evictor(CountEvictor.of(1));
    DataStream<Tuple2<String, Integer>> windowed = windowStream.window(stream).uid(UID);
    CompletableFuture<Collection<Tuple2<String, Integer>>> future = collector.collect(windowed);
    submitJob(savepointPath, env);
    Collection<Tuple2<String, Integer>> results = future.get().stream().distinct().collect(Collectors.toList());
    Assert.assertThat("Incorrect results from bootstrapped windows", results, EVICTOR_MATCHER);
}

Example 15 with DataStream

use of org.apache.flink.streaming.api.datastream.DataStream in project flink by apache.

the class SavepointWriterWindowITCase method testTumbleWindow.

@Test
public void testTumbleWindow() throws Exception {
    final String savepointPath = getTempDirPath(new AbstractID().toHexString());
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStateBackend(stateBackend);
    env.setRuntimeMode(RuntimeExecutionMode.AUTOMATIC);
    DataStream<Tuple2<String, Integer>> bootstrapData = env.fromCollection(WORDS).map(word -> Tuple2.of(word, 1)).returns(TUPLE_TYPE_INFO).assignTimestampsAndWatermarks(WatermarkStrategy.<Tuple2<String, Integer>>noWatermarks().withTimestampAssigner((record, ts) -> 2L));
    WindowedStateTransformation<Tuple2<String, Integer>, String, TimeWindow> transformation = OperatorTransformation.bootstrapWith(bootstrapData).keyBy(tuple -> tuple.f0, Types.STRING).window(TumblingEventTimeWindows.of(Time.milliseconds(5)));
    SavepointWriter.newSavepoint(stateBackend, 128).withOperator(UID, windowBootstrap.bootstrap(transformation)).write(savepointPath);
    env.execute("write state");
    WindowedStream<Tuple2<String, Integer>, String, TimeWindow> stream = env.addSource(new MaxWatermarkSource<Tuple2<String, Integer>>()).returns(TUPLE_TYPE_INFO).keyBy(tuple -> tuple.f0).window(TumblingEventTimeWindows.of(Time.milliseconds(5)));
    DataStream<Tuple2<String, Integer>> windowed = windowStream.window(stream).uid(UID);
    CompletableFuture<Collection<Tuple2<String, Integer>>> future = collector.collect(windowed);
    submitJob(savepointPath, env);
    Collection<Tuple2<String, Integer>> results = future.get();
    Assert.assertThat("Incorrect results from bootstrapped windows", results, STANDARD_MATCHER);
}