Examples with TimeWindow org.apache.flink.streaming.api.windowing.windows.TimeWindow used on opensource projects

Example 61 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class MergingWindowSetTest method testPersistOnlyIfHaveUpdates.

@Test
public void testPersistOnlyIfHaveUpdates() throws Exception {
    @SuppressWarnings("unchecked") ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);
    when(mockState.get()).thenReturn(Lists.newArrayList(new Tuple2<>(new TimeWindow(17, 42), new TimeWindow(42, 17)), new Tuple2<>(new TimeWindow(1, 2), new TimeWindow(3, 4))));
    MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);
    assertEquals(new TimeWindow(42, 17), windowSet.getStateWindow(new TimeWindow(17, 42)));
    assertEquals(new TimeWindow(3, 4), windowSet.getStateWindow(new TimeWindow(1, 2)));
    windowSet.persist();
    verify(mockState, times(0)).add(Matchers.<Tuple2<TimeWindow, TimeWindow>>anyObject());
}

Example 62 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class MergingWindowSetTest method testAddingIdenticalWindows.

/**
	 * Test adding a new window that is identical to an existing window. This should not cause
	 * a merge.
	 */
@Test
public void testAddingIdenticalWindows() throws Exception {
    @SuppressWarnings("unchecked") ListState<Tuple2<TimeWindow, TimeWindow>> mockState = mock(ListState.class);
    MergingWindowSet<TimeWindow> windowSet = new MergingWindowSet<>(EventTimeSessionWindows.withGap(Time.milliseconds(3)), mockState);
    TestingMergeFunction mergeFunction = new TestingMergeFunction();
    mergeFunction.reset();
    assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
    assertFalse(mergeFunction.hasMerged());
    assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));
    mergeFunction.reset();
    assertEquals(new TimeWindow(1, 2), windowSet.addWindow(new TimeWindow(1, 2), mergeFunction));
    assertFalse(mergeFunction.hasMerged());
    assertEquals(new TimeWindow(1, 2), windowSet.getStateWindow(new TimeWindow(1, 2)));
}

Example 63 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class AbstractEventTimeWindowCheckpointingITCase method testPreAggregatedTumblingTimeWindow.

@Test
public void testPreAggregatedTumblingTimeWindow() {
    final int NUM_ELEMENTS_PER_KEY = numElementsPerKey();
    final int WINDOW_SIZE = windowSize();
    final int NUM_KEYS = numKeys();
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.setStateBackend(this.stateBackend);
        env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)).rebalance().keyBy(0).timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS)).reduce(new ReduceFunction<Tuple2<Long, IntType>>() {

            @Override
            public Tuple2<Long, IntType> reduce(Tuple2<Long, IntType> a, Tuple2<Long, IntType> b) {
                return new Tuple2<>(a.f0, new IntType(a.f1.value + b.f1.value));
            }
        }, new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> input, Collector<Tuple4<Long, Long, Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                for (Tuple2<Long, IntType> in : input) {
                    out.collect(new Tuple4<>(in.f0, window.getStart(), window.getEnd(), in.f1));
                }
            }
        }).addSink(new ValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 64 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class WindowCheckpointingITCase method testTumblingProcessingTimeWindow.

// ------------------------------------------------------------------------
@Test
public void testTumblingProcessingTimeWindow() {
    final int NUM_ELEMENTS = 3000;
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setStreamTimeCharacteristic(timeCharacteristic);
        env.getConfig().setAutoWatermarkInterval(10);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.addSource(new FailingSource(NUM_ELEMENTS, NUM_ELEMENTS / 3)).rebalance().keyBy(0).timeWindow(Time.of(100, MILLISECONDS)).apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple2<Long, IntType>, Tuple, TimeWindow>() {

            private boolean open = false;

            @Override
            public void open(Configuration parameters) {
                assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
            }

            @Override
            public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> values, Collector<Tuple2<Long, IntType>> out) {
                // validate that the function has been opened properly
                assertTrue(open);
                for (Tuple2<Long, IntType> value : values) {
                    assertEquals(value.f0.intValue(), value.f1.value);
                    out.collect(new Tuple2<Long, IntType>(value.f0, new IntType(1)));
                }
            }
        }).addSink(new ValidatingSink(NUM_ELEMENTS, 1)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 65 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class AbstractEventTimeWindowCheckpointingITCase method doTestTumblingTimeWindowWithKVState.

public void doTestTumblingTimeWindowWithKVState(int maxParallelism) {
    final int NUM_ELEMENTS_PER_KEY = numElementsPerKey();
    final int WINDOW_SIZE = windowSize();
    final int NUM_KEYS = numKeys();
    FailingSource.reset();
    try {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", cluster.getLeaderRPCPort());
        env.setParallelism(PARALLELISM);
        env.setMaxParallelism(maxParallelism);
        env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
        env.enableCheckpointing(100);
        env.setRestartStrategy(RestartStrategies.fixedDelayRestart(3, 0));
        env.getConfig().disableSysoutLogging();
        env.setStateBackend(this.stateBackend);
        env.addSource(new FailingSource(NUM_KEYS, NUM_ELEMENTS_PER_KEY, NUM_ELEMENTS_PER_KEY / 3)).rebalance().keyBy(0).timeWindow(Time.of(WINDOW_SIZE, MILLISECONDS)).apply(new RichWindowFunction<Tuple2<Long, IntType>, Tuple4<Long, Long, Long, IntType>, Tuple, TimeWindow>() {

            private boolean open = false;

            private ValueState<Integer> count;

            @Override
            public void open(Configuration parameters) {
                assertEquals(PARALLELISM, getRuntimeContext().getNumberOfParallelSubtasks());
                open = true;
                count = getRuntimeContext().getState(new ValueStateDescriptor<>("count", Integer.class, 0));
            }

            @Override
            public void apply(Tuple tuple, TimeWindow window, Iterable<Tuple2<Long, IntType>> values, Collector<Tuple4<Long, Long, Long, IntType>> out) throws Exception {
                // different count results for each key
                if (count.value() == 0) {
                    count.update(tuple.<Long>getField(0).intValue());
                }
                // validate that the function has been opened properly
                assertTrue(open);
                count.update(count.value() + 1);
                out.collect(new Tuple4<>(tuple.<Long>getField(0), window.getStart(), window.getEnd(), new IntType(count.value())));
            }
        }).addSink(new CountValidatingSink(NUM_KEYS, NUM_ELEMENTS_PER_KEY / WINDOW_SIZE)).setParallelism(1);
        tryExecute(env, "Tumbling Window Test");
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}