Examples with TimeWindow org.apache.flink.streaming.api.windowing.windows.TimeWindow used on opensource projects

Example 51 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class EvictingWindowOperatorTest method testTumblingWindowWithApply.

@Test
@SuppressWarnings("unchecked")
public void testTumblingWindowWithApply() throws Exception {
    AtomicInteger closeCalled = new AtomicInteger(0);
    final int WINDOW_SIZE = 4;
    TypeInformation<Tuple2<String, Integer>> inputType = TypeInfoParser.parse("Tuple2<String, Integer>");
    @SuppressWarnings({ "unchecked", "rawtypes" }) TypeSerializer<StreamRecord<Tuple2<String, Integer>>> streamRecordSerializer = (TypeSerializer<StreamRecord<Tuple2<String, Integer>>>) new StreamElementSerializer(inputType.createSerializer(new ExecutionConfig()));
    ListStateDescriptor<StreamRecord<Tuple2<String, Integer>>> stateDesc = new ListStateDescriptor<>("window-contents", streamRecordSerializer);
    EvictingWindowOperator<String, Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow> operator = new EvictingWindowOperator<>(TumblingEventTimeWindows.of(Time.of(WINDOW_SIZE, TimeUnit.SECONDS)), new TimeWindow.Serializer(), new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO.createSerializer(new ExecutionConfig()), stateDesc, new InternalIterableWindowFunction<>(new RichSumReducer<TimeWindow>(closeCalled)), EventTimeTrigger.create(), CountEvictor.of(WINDOW_SIZE), 0, null);
    OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Tuple2<String, Integer>> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new TupleKeySelector(), BasicTypeInfo.STRING_TYPE_INFO);
    long initialTime = 0L;
    testHarness.open();
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 10));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 100));
    testHarness.processWatermark(new Watermark(1999));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 1997));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 1998));
    // not late but more than 4
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 2310));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key1", 1), initialTime + 2310));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2310));
    testHarness.processElement(new StreamRecord<>(new Tuple2<>("key2", 1), initialTime + 2310));
    // now is the evictor
    testHarness.processWatermark(new Watermark(3999));
    ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>();
    expectedOutput.add(new Watermark(1999));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key1", 4), 3999));
    expectedOutput.add(new StreamRecord<>(new Tuple2<>("key2", 2), 3999));
    expectedOutput.add(new Watermark(3999));
    TestHarnessUtil.assertOutputEqualsSorted("Output was not correct.", expectedOutput, testHarness.getOutput(), new EvictingWindowOperatorTest.ResultSortComparator());
    testHarness.close();
}

Example 52 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class AllWindowTranslationTest method testApplyEventTime.

// ------------------------------------------------------------------------
//  apply() translation tests
// ------------------------------------------------------------------------
@Test
@SuppressWarnings("rawtypes")
public void testApplyEventTime() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
    DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
    DataStream<Tuple2<String, Integer>> window1 = source.windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))).apply(new AllWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void apply(TimeWindow window, Iterable<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) throws Exception {
            for (Tuple2<String, Integer> in : values) {
                out.collect(in);
            }
        }
    });
    OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
    OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
    Assert.assertTrue(operator instanceof WindowOperator);
    WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
    Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
    Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
    Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);
    processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example 53 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class ContinuousEventTimeTriggerTest method testClear.

/**
	 * Verify that clear() does not leak across windows.
	 */
@Test
public void testClear() throws Exception {
    TriggerTestHarness<Object, TimeWindow> testHarness = new TriggerTestHarness<>(ContinuousEventTimeTrigger.<TimeWindow>of(Time.hours(1)), new TimeWindow.Serializer());
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2)));
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(2, 4)));
    assertEquals(2, testHarness.numStateEntries());
    assertEquals(0, testHarness.numProcessingTimeTimers());
    assertEquals(4, testHarness.numEventTimeTimers());
    assertEquals(2, testHarness.numEventTimeTimers(new TimeWindow(0, 2)));
    assertEquals(2, testHarness.numEventTimeTimers(new TimeWindow(2, 4)));
    testHarness.clearTriggerState(new TimeWindow(2, 4));
    assertEquals(1, testHarness.numStateEntries());
    assertEquals(0, testHarness.numProcessingTimeTimers());
    assertEquals(3, testHarness.numEventTimeTimers());
    assertEquals(2, testHarness.numEventTimeTimers(new TimeWindow(0, 2)));
    assertEquals(1, testHarness.numEventTimeTimers(new TimeWindow(2, 4)));
    testHarness.clearTriggerState(new TimeWindow(0, 2));
    assertEquals(0, testHarness.numStateEntries());
    assertEquals(0, testHarness.numProcessingTimeTimers());
    // doesn't clean up timers
    assertEquals(2, testHarness.numEventTimeTimers());
}

Example 54 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class CountTriggerTest method testMergingWindows.

@Test
public void testMergingWindows() throws Exception {
    TriggerTestHarness<Object, TimeWindow> testHarness = new TriggerTestHarness<>(CountTrigger.<TimeWindow>of(3), new TimeWindow.Serializer());
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2)));
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(2, 4)));
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(4, 6)));
    // shouldn't have any timers
    assertEquals(0, testHarness.numProcessingTimeTimers());
    assertEquals(0, testHarness.numEventTimeTimers());
    assertEquals(3, testHarness.numStateEntries());
    assertEquals(1, testHarness.numStateEntries(new TimeWindow(0, 2)));
    assertEquals(1, testHarness.numStateEntries(new TimeWindow(2, 4)));
    assertEquals(1, testHarness.numStateEntries(new TimeWindow(4, 6)));
    testHarness.mergeWindows(new TimeWindow(0, 4), Lists.newArrayList(new TimeWindow(0, 2), new TimeWindow(2, 4)));
    assertEquals(2, testHarness.numStateEntries());
    assertEquals(0, testHarness.numStateEntries(new TimeWindow(0, 2)));
    assertEquals(0, testHarness.numStateEntries(new TimeWindow(2, 4)));
    assertEquals(1, testHarness.numStateEntries(new TimeWindow(0, 4)));
    assertEquals(1, testHarness.numStateEntries(new TimeWindow(4, 6)));
    assertEquals(TriggerResult.FIRE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 4)));
    assertEquals(1, testHarness.numStateEntries());
    assertEquals(0, testHarness.numStateEntries(new TimeWindow(0, 4)));
    assertEquals(1, testHarness.numStateEntries(new TimeWindow(4, 6)));
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(4, 6)));
    assertEquals(TriggerResult.FIRE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(4, 6)));
    assertEquals(0, testHarness.numStateEntries());
}

Example 55 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class CountTriggerTest method testClear.

/**
	 * Verify that clear() does not leak across windows.
	 */
@Test
public void testClear() throws Exception {
    TriggerTestHarness<Object, TimeWindow> testHarness = new TriggerTestHarness<>(CountTrigger.<TimeWindow>of(3), new TimeWindow.Serializer());
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2)));
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(2, 4)));
    // shouldn't have any timers
    assertEquals(0, testHarness.numProcessingTimeTimers());
    assertEquals(0, testHarness.numEventTimeTimers());
    assertEquals(2, testHarness.numStateEntries());
    assertEquals(1, testHarness.numStateEntries(new TimeWindow(0, 2)));
    assertEquals(1, testHarness.numStateEntries(new TimeWindow(2, 4)));
    testHarness.clearTriggerState(new TimeWindow(2, 4));
    assertEquals(1, testHarness.numStateEntries());
    assertEquals(1, testHarness.numStateEntries(new TimeWindow(0, 2)));
    assertEquals(0, testHarness.numStateEntries(new TimeWindow(2, 4)));
    testHarness.clearTriggerState(new TimeWindow(0, 2));
    assertEquals(0, testHarness.numStateEntries());
    assertEquals(0, testHarness.numStateEntries(new TimeWindow(0, 2)));
    assertEquals(0, testHarness.numStateEntries(new TimeWindow(2, 4)));
}