Examples with TimeWindow org.apache.flink.streaming.api.windowing.windows.TimeWindow used on opensource projects

Example 66 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class AbstractEventTimeWindowCheckpointingITCase method testPreAggregatedSlidingTimeWindow.

@Test
public void testPreAggregatedSlidingTimeWindow() {
    final int NUM_ELEMENTS_PER_KEY = numElementsPerKey();
    final int WINDOW_SIZE = windowSize();
    final int WINDOW_SLIDE = windowSlide();
    final int NUM_KEYS = numKeys();
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.setStateBackend(this.stateBackend);
        env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)).rebalance().keyBy(0).timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS), Time.of(WINDOW_SLIDE, MILLISECONDS)).reduce(new ReduceFunction<Tuple2<Long, IntType>>() {

            @Override
            public Tuple2<Long, IntType> reduce(Tuple2<Long, IntType> a, Tuple2<Long, IntType> b) {
                // validate that the function has been opened properly
                return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
            }
        }, new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> input, Collector<Tuple4<Long, Long, Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                for (Tuple2<Long, IntType> in : input) {
                    out.collect(new Tuple4<>(in.f0, window.getStart(), window.getEnd(), in.f1));
                }
            }
        }).addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SLIDE)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 67 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class EventTimeAllWindowCheckpointingITCase method testSlidingTimeWindow.

@Test
public void testSlidingTimeWindow() {
    final int NUM_ELEMENTS_PER_KEY = 3000;
    final int WINDOW_SIZE = 1000;
    final int WINDOW_SLIDE = 100;
    final int NUM_KEYS = 1;
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0));
        env.getConfig().disableSysoutLogging();
        env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)).rebalance().timeWindowAll(Time.of(WINDOW_SIZE, MILLISECONDS), Time.of(WINDOW_SLIDE, MILLISECONDS)).apply(new RichAllWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(1, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(TimeWindow window, Iterable<Tuple2<Long, IntType>> values, Collector<Tuple4<Long, Long, Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                int sum = 0;
                long key = -1;
                for (Tuple2<Long, IntType> value : values) {
                    sum += value.f1.value;
                    key = value.f0;
                }
                out.collect(new Tuple4<>(key, window.getStart(), window.getEnd(), new IntType(sum)));
            }
        }).addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SLIDE)).setParallelism(1);
        tryExecute(env, "Sliding Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 68 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class RocksDBAggregatingStateTest method testMerging.

@Test
public void testMerging() throws Exception {
    final AggregatingStateDescriptor<Long, MutableLong, Long> stateDescr = new AggregatingStateDescriptor<>("my-state", new AddingFunction(), MutableLong.class);
    stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
    final TimeWindow win1 = new TimeWindow(1000, 2000);
    final TimeWindow win2 = new TimeWindow(2000, 3000);
    final TimeWindow win3 = new TimeWindow(3000, 4000);
    final Long expectedResult = 165L;
    final RocksDBStateBackend backend = new RocksDBStateBackend(tmp.newFolder().toURI());
    backend.setDbStoragePath(tmp.newFolder().getAbsolutePath());
    final RocksDBKeyedStateBackend<String> keyedBackend = createKeyedBackend(backend);
    try {
        InternalAggregatingState<TimeWindow, Long, Long> state = keyedBackend.createAggregatingState(new TimeWindow.Serializer(), stateDescr);
        // populate the different namespaces
        //  - abc spreads the values over three namespaces
        //  - def spreads teh values over two namespaces (one empty)
        //  - ghi is empty
        //  - jkl has all elements already in the target namespace
        //  - mno has all elements already in one source namespace
        keyedBackend.setCurrentKey("abc");
        state.setCurrentNamespace(win1);
        state.add(33L);
        state.add(55L);
        state.setCurrentNamespace(win2);
        state.add(22L);
        state.add(11L);
        state.setCurrentNamespace(win3);
        state.add(44L);
        keyedBackend.setCurrentKey("def");
        state.setCurrentNamespace(win1);
        state.add(11L);
        state.add(44L);
        state.setCurrentNamespace(win3);
        state.add(22L);
        state.add(55L);
        state.add(33L);
        keyedBackend.setCurrentKey("jkl");
        state.setCurrentNamespace(win1);
        state.add(11L);
        state.add(22L);
        state.add(33L);
        state.add(44L);
        state.add(55L);
        keyedBackend.setCurrentKey("mno");
        state.setCurrentNamespace(win3);
        state.add(11L);
        state.add(22L);
        state.add(33L);
        state.add(44L);
        state.add(55L);
        keyedBackend.setCurrentKey("abc");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        assertEquals(expectedResult, state.get());
        keyedBackend.setCurrentKey("def");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        assertEquals(expectedResult, state.get());
        keyedBackend.setCurrentKey("ghi");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        assertNull(state.get());
        keyedBackend.setCurrentKey("jkl");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        assertEquals(expectedResult, state.get());
        keyedBackend.setCurrentKey("mno");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        assertEquals(expectedResult, state.get());
    } finally {
        keyedBackend.close();
        keyedBackend.dispose();
    }
}

Example 69 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class RocksDBListStateTest method testMerging.

@Test
public void testMerging() throws Exception {
    final ListStateDescriptor<Long> stateDescr = new ListStateDescriptor<>("my-state", Long.class);
    stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
    final TimeWindow win1 = new TimeWindow(1000, 2000);
    final TimeWindow win2 = new TimeWindow(2000, 3000);
    final TimeWindow win3 = new TimeWindow(3000, 4000);
    final Set<Long> expectedResult = new HashSet<>(asList(11L, 22L, 33L, 44L, 55L));
    final RocksDBStateBackend backend = new RocksDBStateBackend(tmp.newFolder().toURI());
    backend.setDbStoragePath(tmp.newFolder().getAbsolutePath());
    final RocksDBKeyedStateBackend<String> keyedBackend = createKeyedBackend(backend);
    try {
        InternalListState<TimeWindow, Long> state = keyedBackend.createListState(new TimeWindow.Serializer(), stateDescr);
        // populate the different namespaces
        //  - abc spreads the values over three namespaces
        //  - def spreads teh values over two namespaces (one empty)
        //  - ghi is empty
        //  - jkl has all elements already in the target namespace
        //  - mno has all elements already in one source namespace
        keyedBackend.setCurrentKey("abc");
        state.setCurrentNamespace(win1);
        state.add(33L);
        state.add(55L);
        state.setCurrentNamespace(win2);
        state.add(22L);
        state.add(11L);
        state.setCurrentNamespace(win3);
        state.add(44L);
        keyedBackend.setCurrentKey("def");
        state.setCurrentNamespace(win1);
        state.add(11L);
        state.add(44L);
        state.setCurrentNamespace(win3);
        state.add(22L);
        state.add(55L);
        state.add(33L);
        keyedBackend.setCurrentKey("jkl");
        state.setCurrentNamespace(win1);
        state.add(11L);
        state.add(22L);
        state.add(33L);
        state.add(44L);
        state.add(55L);
        keyedBackend.setCurrentKey("mno");
        state.setCurrentNamespace(win3);
        state.add(11L);
        state.add(22L);
        state.add(33L);
        state.add(44L);
        state.add(55L);
        keyedBackend.setCurrentKey("abc");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        validateResult(state.get(), expectedResult);
        keyedBackend.setCurrentKey("def");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        validateResult(state.get(), expectedResult);
        keyedBackend.setCurrentKey("ghi");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        assertNull(state.get());
        keyedBackend.setCurrentKey("jkl");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        validateResult(state.get(), expectedResult);
        keyedBackend.setCurrentKey("mno");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        validateResult(state.get(), expectedResult);
    } finally {
        keyedBackend.close();
        keyedBackend.dispose();
    }
}

Example 70 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class RocksDBReducingStateTest method testMerging.

@Test
public void testMerging() throws Exception {
    final ReducingStateDescriptor<Long> stateDescr = new ReducingStateDescriptor<>("my-state", new AddingFunction(), Long.class);
    stateDescr.initializeSerializerUnlessSet(new ExecutionConfig());
    final TimeWindow win1 = new TimeWindow(1000, 2000);
    final TimeWindow win2 = new TimeWindow(2000, 3000);
    final TimeWindow win3 = new TimeWindow(3000, 4000);
    final Long expectedResult = 165L;
    final RocksDBStateBackend backend = new RocksDBStateBackend(tmp.newFolder().toURI());
    backend.setDbStoragePath(tmp.newFolder().getAbsolutePath());
    final RocksDBKeyedStateBackend<String> keyedBackend = createKeyedBackend(backend);
    try {
        final InternalReducingState<TimeWindow, Long> state = keyedBackend.createReducingState(new TimeWindow.Serializer(), stateDescr);
        // populate the different namespaces
        //  - abc spreads the values over three namespaces
        //  - def spreads teh values over two namespaces (one empty)
        //  - ghi is empty
        //  - jkl has all elements already in the target namespace
        //  - mno has all elements already in one source namespace
        keyedBackend.setCurrentKey("abc");
        state.setCurrentNamespace(win1);
        state.add(33L);
        state.add(55L);
        state.setCurrentNamespace(win2);
        state.add(22L);
        state.add(11L);
        state.setCurrentNamespace(win3);
        state.add(44L);
        keyedBackend.setCurrentKey("def");
        state.setCurrentNamespace(win1);
        state.add(11L);
        state.add(44L);
        state.setCurrentNamespace(win3);
        state.add(22L);
        state.add(55L);
        state.add(33L);
        keyedBackend.setCurrentKey("jkl");
        state.setCurrentNamespace(win1);
        state.add(11L);
        state.add(22L);
        state.add(33L);
        state.add(44L);
        state.add(55L);
        keyedBackend.setCurrentKey("mno");
        state.setCurrentNamespace(win3);
        state.add(11L);
        state.add(22L);
        state.add(33L);
        state.add(44L);
        state.add(55L);
        keyedBackend.setCurrentKey("abc");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        assertEquals(expectedResult, state.get());
        keyedBackend.setCurrentKey("def");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        assertEquals(expectedResult, state.get());
        keyedBackend.setCurrentKey("ghi");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        assertNull(state.get());
        keyedBackend.setCurrentKey("jkl");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        assertEquals(expectedResult, state.get());
        keyedBackend.setCurrentKey("mno");
        state.mergeNamespaces(win1, asList(win2, win3));
        state.setCurrentNamespace(win1);
        assertEquals(expectedResult, state.get());
    } finally {
        keyedBackend.close();
        keyedBackend.dispose();
    }
}