Examples with TimeTriggerPolicy com.twitter.heron.api.windowing.triggers.TimeTriggerPolicy used on opensource projects

Example 1 with TimeTriggerPolicy

use of com.twitter.heron.api.windowing.triggers.TimeTriggerPolicy in project incubator-heron by apache.

the class WindowManagerTest method testTimeBasedWindowExpiry.

@Test
public void testTimeBasedWindowExpiry() throws Exception {
    EvictionPolicy<Integer, ?> evictionPolicy = new TimeEvictionPolicy<Integer>(Duration.ofMillis(100).toMillis());
    windowManager.setEvictionPolicy(evictionPolicy);
    /*
         * Don't wait for Timetrigger to fire since this could lead to timing issues in unit tests.
         * Set it to a large value and trigger manually.
          */
    TriggerPolicy<Integer, ?> triggerPolicy = new TimeTriggerPolicy<Integer>(Duration.ofDays(1).toMillis());
    triggerPolicy.setTriggerHandler(windowManager);
    triggerPolicy.setEvictionPolicy(evictionPolicy);
    triggerPolicy.setTopologyConfig(new Config());
    triggerPolicy.start();
    windowManager.setTriggerPolicy(triggerPolicy);
    long now = System.currentTimeMillis();
    // add 10 events
    for (int i : seq(1, 10)) {
        windowManager.add(i);
    }
    // simulate the time trigger by setting the reference time and invoking onTrigger() manually
    evictionPolicy.setContext(new DefaultEvictionContext(now + 60));
    windowManager.onTrigger();
    assertEquals(seq(1, 10), listener.onActivationEvents);
    assertTrue(listener.onActivationExpiredEvents.isEmpty());
    listener.clear();
    // wait so all events expire
    evictionPolicy.setContext(new DefaultEvictionContext(now + 120));
    windowManager.onTrigger();
    assertEquals(seq(1, 10), listener.onExpiryEvents);
    assertTrue(listener.onActivationEvents.isEmpty());
    listener.clear();
    evictionPolicy.setContext(new DefaultEvictionContext(now + 180));
    windowManager.onTrigger();
    assertTrue(listener.onActivationExpiredEvents.isEmpty());
    assertTrue(listener.onActivationEvents.isEmpty());
}

Example 2 with TimeTriggerPolicy

use of com.twitter.heron.api.windowing.triggers.TimeTriggerPolicy in project incubator-heron by apache.

the class WindowManagerTest method testTimeBasedWindow.

@Test
public void testTimeBasedWindow() throws Exception {
    EvictionPolicy<Integer, ?> evictionPolicy = new TimeEvictionPolicy<Integer>(Duration.ofSeconds(1).toMillis());
    windowManager.setEvictionPolicy(evictionPolicy);
    /*
         * Don't wait for Timetrigger to fire since this could lead to timing issues in unit tests.
         * Set it to a large value and trigger manually.
          */
    TriggerPolicy<Integer, ?> triggerPolicy = new TimeTriggerPolicy<Integer>(Duration.ofDays(1).toMillis());
    triggerPolicy.setTriggerHandler(windowManager);
    triggerPolicy.setEvictionPolicy(evictionPolicy);
    triggerPolicy.setTopologyConfig(new Config());
    triggerPolicy.start();
    windowManager.setTriggerPolicy(triggerPolicy);
    long now = System.currentTimeMillis();
    // add with past ts
    for (int i : seq(1, 50)) {
        windowManager.add(i, now - 1000);
    }
    // add with current ts
    for (int i : seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD)) {
        windowManager.add(i, now);
    }
    // first 50 should have expired due to expire events threshold
    assertEquals(50, listener.onExpiryEvents.size());
    // add more events with past ts
    for (int i : seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100)) {
        windowManager.add(i, now - 1000);
    }
    // simulate the time trigger by setting the reference time and invoking onTrigger() manually
    evictionPolicy.setContext(new DefaultEvictionContext(now + 100));
    windowManager.onTrigger();
    // 100 events with past ts should expire
    assertEquals(100, listener.onExpiryEvents.size());
    assertEquals(seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100), listener.onExpiryEvents);
    List<Integer> activationsEvents = seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD);
    assertEquals(seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD), listener.onActivationEvents);
    assertEquals(seq(51, WindowManager.EXPIRE_EVENTS_THRESHOLD), listener.onActivationNewEvents);
    // activation expired list should contain even the ones expired due to EXPIRE_EVENTS_THRESHOLD
    List<Integer> expiredList = seq(1, 50);
    expiredList.addAll(seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 1, WindowManager.EXPIRE_EVENTS_THRESHOLD + 100));
    assertEquals(expiredList, listener.onActivationExpiredEvents);
    listener.clear();
    // add more events with current ts
    List<Integer> newEvents = seq(WindowManager.EXPIRE_EVENTS_THRESHOLD + 101, WindowManager.EXPIRE_EVENTS_THRESHOLD + 200);
    for (int i : newEvents) {
        windowManager.add(i, now);
    }
    activationsEvents.addAll(newEvents);
    // simulate the time trigger by setting the reference time and invoking onTrigger() manually
    evictionPolicy.setContext(new DefaultEvictionContext(now + 200));
    windowManager.onTrigger();
    assertTrue(listener.onExpiryEvents.isEmpty());
    assertEquals(activationsEvents, listener.onActivationEvents);
    assertEquals(newEvents, listener.onActivationNewEvents);
}

Example 3 with TimeTriggerPolicy

use of com.twitter.heron.api.windowing.triggers.TimeTriggerPolicy in project incubator-heron by apache.

the class WindowManagerTest method testExpireThreshold.

@Test
public void testExpireThreshold() throws Exception {
    int threshold = WindowManager.EXPIRE_EVENTS_THRESHOLD;
    int windowLength = 5;
    CountEvictionPolicy<Integer> countEvictionPolicy = new CountEvictionPolicy<Integer>(5);
    windowManager.setEvictionPolicy(countEvictionPolicy);
    TriggerPolicy<Integer, ?> triggerPolicy = new TimeTriggerPolicy<Integer>(Duration.ofHours(1).toMillis());
    triggerPolicy.setEvictionPolicy(countEvictionPolicy);
    triggerPolicy.setTriggerHandler(windowManager);
    triggerPolicy.setTopologyConfig(new Config());
    triggerPolicy.start();
    windowManager.setTriggerPolicy(triggerPolicy);
    for (int i : seq(1, 5)) {
        windowManager.add(i);
    }
    // nothing expired yet
    assertTrue(listener.onExpiryEvents.isEmpty());
    for (int i : seq(6, 10)) {
        windowManager.add(i);
    }
    for (int i : seq(11, threshold)) {
        windowManager.add(i);
    }
    // window should be compacted and events should be expired.
    assertEquals(seq(1, threshold - windowLength), listener.onExpiryEvents);
}