Examples with TimeWindow org.apache.flink.streaming.api.windowing.windows.TimeWindow used on opensource projects

Example 81 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class AllWindowTranslationTest method testApplyWithEvictor.

@Test
@SuppressWarnings("rawtypes")
public void testApplyWithEvictor() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
    DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
    DataStream<Tuple2<String, Integer>> window1 = source.windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))).trigger(CountTrigger.of(1)).evictor(TimeEvictor.of(Time.of(100, TimeUnit.MILLISECONDS))).apply(new AllWindowFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void apply(TimeWindow window, Iterable<Tuple2<String, Integer>> values, Collector<Tuple2<String, Integer>> out) throws Exception {
            for (Tuple2<String, Integer> in : values) {
                out.collect(in);
            }
        }
    });
    OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window1.getTransformation();
    OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
    Assert.assertTrue(operator instanceof EvictingWindowOperator);
    EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator;
    Assert.assertTrue(winOperator.getTrigger() instanceof CountTrigger);
    Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
    Assert.assertTrue(winOperator.getEvictor() instanceof TimeEvictor);
    Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);
    processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example 82 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class AllWindowTranslationTest method testFoldWithEvictorAndProcessFunction.

@Test
@SuppressWarnings({ "rawtypes", "unchecked" })
public void testFoldWithEvictorAndProcessFunction() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
    DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
    DataStream<Tuple3<String, String, Integer>> window1 = source.windowAll(SlidingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS), Time.of(100, TimeUnit.MILLISECONDS))).evictor(CountEvictor.of(100)).fold(new Tuple3<>("", "", 1), new DummyFolder(), new ProcessAllWindowFunction<Tuple3<String, String, Integer>, Tuple3<String, String, Integer>, TimeWindow>() {

        @Override
        public void process(Context context, Iterable<Tuple3<String, String, Integer>> elements, Collector<Tuple3<String, String, Integer>> out) throws Exception {
            for (Tuple3<String, String, Integer> in : elements) {
                out.collect(in);
            }
        }
    });
    OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple3<String, String, Integer>>) window1.getTransformation();
    OneInputStreamOperator<Tuple2<String, Integer>, Tuple3<String, String, Integer>> operator = transform.getOperator();
    Assert.assertTrue(operator instanceof EvictingWindowOperator);
    EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?> winOperator = (EvictingWindowOperator<String, Tuple2<String, Integer>, ?, ?>) operator;
    Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
    Assert.assertTrue(winOperator.getEvictor() instanceof CountEvictor);
    Assert.assertTrue(winOperator.getWindowAssigner() instanceof SlidingEventTimeWindows);
    Assert.assertTrue(winOperator.getStateDescriptor() instanceof ListStateDescriptor);
    winOperator.setOutputType((TypeInformation) window1.getType(), new ExecutionConfig());
    processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example 83 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class AllWindowTranslationTest method testFoldWithWindowFunctionEventTime.

@Test
@SuppressWarnings("rawtypes")
public void testFoldWithWindowFunctionEventTime() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
    DataStream<Tuple2<String, Integer>> source = env.fromElements(Tuple2.of("hello", 1), Tuple2.of("hello", 2));
    DataStream<Tuple2<String, Integer>> window = source.windowAll(TumblingEventTimeWindows.of(Time.of(1, TimeUnit.SECONDS))).fold(new Tuple3<>("", "", 0), new DummyFolder(), new AllWindowFunction<Tuple3<String, String, Integer>, Tuple2<String, Integer>, TimeWindow>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void apply(TimeWindow window, Iterable<Tuple3<String, String, Integer>> values, Collector<Tuple2<String, Integer>> out) throws Exception {
            for (Tuple3<String, String, Integer> in : values) {
                out.collect(new Tuple2<>(in.f0, in.f2));
            }
        }
    });
    OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>> transform = (OneInputTransformation<Tuple2<String, Integer>, Tuple2<String, Integer>>) window.getTransformation();
    OneInputStreamOperator<Tuple2<String, Integer>, Tuple2<String, Integer>> operator = transform.getOperator();
    Assert.assertTrue(operator instanceof WindowOperator);
    WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?> winOperator = (WindowOperator<String, Tuple2<String, Integer>, ?, ?, ?>) operator;
    Assert.assertTrue(winOperator.getTrigger() instanceof EventTimeTrigger);
    Assert.assertTrue(winOperator.getWindowAssigner() instanceof TumblingEventTimeWindows);
    Assert.assertTrue(winOperator.getStateDescriptor() instanceof FoldingStateDescriptor);
    processElementAndEnsureOutput(winOperator, winOperator.getKeySelector(), BasicTypeInfo.STRING_TYPE_INFO, new Tuple2<>("hello", 1));
}

Example 84 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class ContinuousEventTimeTriggerTest method testWindowSeparationAndFiring.

/**
	 * Verify that state <TimeWindow>of separate windows does not leak into other windows.
	 */
@Test
public void testWindowSeparationAndFiring() throws Exception {
    TriggerTestHarness<Object, TimeWindow> testHarness = new TriggerTestHarness<>(ContinuousEventTimeTrigger.<TimeWindow>of(Time.hours(1)), new TimeWindow.Serializer());
    // inject several elements
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2)));
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2)));
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2)));
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(2, 4)));
    assertEquals(TriggerResult.CONTINUE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(2, 4)));
    assertEquals(2, testHarness.numStateEntries());
    assertEquals(0, testHarness.numProcessingTimeTimers());
    assertEquals(4, testHarness.numEventTimeTimers());
    assertEquals(2, testHarness.numEventTimeTimers(new TimeWindow(0, 2)));
    assertEquals(2, testHarness.numEventTimeTimers(new TimeWindow(2, 4)));
    Collection<Tuple2<TimeWindow, TriggerResult>> triggerResults = testHarness.advanceWatermark(2);
    boolean sawFiring = false;
    for (Tuple2<TimeWindow, TriggerResult> r : triggerResults) {
        if (r.f0.equals(new TimeWindow(0, 2))) {
            sawFiring = true;
            assertTrue(r.f1.equals(TriggerResult.FIRE));
        }
    }
    assertTrue(sawFiring);
    assertEquals(2, testHarness.numStateEntries());
    assertEquals(0, testHarness.numProcessingTimeTimers());
    assertEquals(3, testHarness.numEventTimeTimers());
    assertEquals(1, testHarness.numEventTimeTimers(new TimeWindow(0, 2)));
    assertEquals(2, testHarness.numEventTimeTimers(new TimeWindow(2, 4)));
    triggerResults = testHarness.advanceWatermark(4);
    sawFiring = false;
    for (Tuple2<TimeWindow, TriggerResult> r : triggerResults) {
        if (r.f0.equals(new TimeWindow(2, 4))) {
            sawFiring = true;
            assertTrue(r.f1.equals(TriggerResult.FIRE));
        }
    }
    assertTrue(sawFiring);
    assertEquals(2, testHarness.numStateEntries());
    assertEquals(0, testHarness.numProcessingTimeTimers());
    assertEquals(2, testHarness.numEventTimeTimers());
}

Example 85 with TimeWindow

use of org.apache.flink.streaming.api.windowing.windows.TimeWindow in project flink by apache.

the class ContinuousEventTimeTriggerTest method testTriggerHandlesAllOnTimerCalls.

/**
	 * Verify that the trigger doesn't fail with an NPE if we insert a timer firing when there is
	 * no trigger state.
	 */
@Test
public void testTriggerHandlesAllOnTimerCalls() throws Exception {
    TriggerTestHarness<Object, TimeWindow> testHarness = new TriggerTestHarness<>(ContinuousEventTimeTrigger.<TimeWindow>of(Time.milliseconds(5)), new TimeWindow.Serializer());
    assertEquals(0, testHarness.numStateEntries());
    assertEquals(0, testHarness.numProcessingTimeTimers());
    assertEquals(0, testHarness.numEventTimeTimers());
    // this will make the elements we now process fall into late windows, i.e. no trigger state
    // will be created
    testHarness.advanceWatermark(10);
    // late fires immediately
    assertEquals(TriggerResult.FIRE, testHarness.processElement(new StreamRecord<Object>(1), new TimeWindow(0, 2)));
    // simulate a GC timer firing
    testHarness.invokeOnEventTime(20, new TimeWindow(0, 2));
}