Examples with IntervalWindow org.apache.beam.sdk.transforms.windowing.IntervalWindow used on opensource projects

Example 46 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class OrFinallyStateMachineTest method testShouldFireAfterMerge.

/**
 * Tests that if the first trigger rewinds to be non-finished in the merged window, then it
 * becomes the currently active trigger again, with real triggers.
 */
@Test
public void testShouldFireAfterMerge() throws Exception {
    tester = TriggerStateMachineTester.forTrigger(AfterEachStateMachine.inOrder(AfterPaneStateMachine.elementCountAtLeast(5).orFinally(AfterWatermarkStateMachine.pastEndOfWindow()), RepeatedlyStateMachine.forever(AfterPaneStateMachine.elementCountAtLeast(1))), Sessions.withGapDuration(Duration.millis(10)));
    // Finished the orFinally in the first window
    tester.injectElements(1);
    IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11));
    assertFalse(tester.shouldFire(firstWindow));
    tester.advanceInputWatermark(new Instant(11));
    assertTrue(tester.shouldFire(firstWindow));
    tester.fireIfShouldFire(firstWindow);
    // Set up second window where it is not done
    tester.injectElements(5);
    IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15));
    assertFalse(tester.shouldFire(secondWindow));
    // Merge them, if the merged window were on the second trigger, it would be ready
    tester.mergeWindows();
    IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15));
    assertFalse(tester.shouldFire(mergedWindow));
    // Now adding 3 more makes the main trigger ready to fire
    tester.injectElements(1, 2, 3, 4, 5);
    tester.mergeWindows();
    assertTrue(tester.shouldFire(mergedWindow));
}

Example 47 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class AfterEachStateMachineTest method testAfterEachInSequence.

/**
 * Tests that the {@link AfterEachStateMachine} trigger fires and finishes the first trigger then
 * the second.
 */
@Test
public void testAfterEachInSequence() throws Exception {
    tester = TriggerStateMachineTester.forTrigger(AfterEachStateMachine.inOrder(RepeatedlyStateMachine.forever(AfterPaneStateMachine.elementCountAtLeast(2)).orFinally(AfterPaneStateMachine.elementCountAtLeast(3)), RepeatedlyStateMachine.forever(AfterPaneStateMachine.elementCountAtLeast(5)).orFinally(AfterWatermarkStateMachine.pastEndOfWindow())), FixedWindows.of(Duration.millis(10)));
    IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10));
    // AfterCount(2) not ready
    tester.injectElements(1);
    assertFalse(tester.shouldFire(window));
    // AfterCount(2) ready, not finished
    tester.injectElements(2);
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertFalse(tester.isMarkedFinished(window));
    // orFinally(AfterCount(3)) ready and will finish the first
    tester.injectElements(1, 2, 3);
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertFalse(tester.isMarkedFinished(window));
    // Now running as the second trigger
    assertFalse(tester.shouldFire(window));
    // This quantity of elements would fire and finish if it were erroneously still the first
    tester.injectElements(1, 2, 3, 4);
    assertFalse(tester.shouldFire(window));
    // Now fire once
    tester.injectElements(5);
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertFalse(tester.isMarkedFinished(window));
    // This time advance the watermark to finish the whole mess.
    tester.advanceInputWatermark(new Instant(10));
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertTrue(tester.isMarkedFinished(window));
}

Example 48 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class AfterPaneStateMachineTest method testAfterPaneElementCountSessions.

@Test
public void testAfterPaneElementCountSessions() throws Exception {
    tester = TriggerStateMachineTester.forTrigger(AfterPaneStateMachine.elementCountAtLeast(2), Sessions.withGapDuration(Duration.millis(10)));
    tester.injectElements(// in [1, 11)
    1, // in [2, 12)
    2);
    assertFalse(tester.shouldFire(new IntervalWindow(new Instant(1), new Instant(11))));
    assertFalse(tester.shouldFire(new IntervalWindow(new Instant(2), new Instant(12))));
    tester.mergeWindows();
    IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(12));
    assertTrue(tester.shouldFire(mergedWindow));
    tester.fireIfShouldFire(mergedWindow);
    assertTrue(tester.isMarkedFinished(mergedWindow));
    // Because we closed the previous window, we don't have it around to merge with. So there
    // will be a new FIRE_AND_FINISH result.
    tester.injectElements(// in [7, 17)
    7, // in [9, 19)
    9);
    tester.mergeWindows();
    IntervalWindow newMergedWindow = new IntervalWindow(new Instant(7), new Instant(19));
    assertTrue(tester.shouldFire(newMergedWindow));
    tester.fireIfShouldFire(newMergedWindow);
    assertTrue(tester.isMarkedFinished(newMergedWindow));
}

Example 49 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class DefaultTriggerStateMachineTest method testDefaultTriggerSlidingWindows.

@Test
public void testDefaultTriggerSlidingWindows() throws Exception {
    tester = TriggerStateMachineTester.forTrigger(DefaultTriggerStateMachine.of(), SlidingWindows.of(Duration.millis(100)).every(Duration.millis(50)));
    tester.injectElements(// [-50, 50), [0, 100)
    1, // [0, 100), [50, 150)
    50);
    IntervalWindow firstWindow = new IntervalWindow(new Instant(-50), new Instant(50));
    IntervalWindow secondWindow = new IntervalWindow(new Instant(0), new Instant(100));
    IntervalWindow thirdWindow = new IntervalWindow(new Instant(50), new Instant(150));
    assertFalse(tester.shouldFire(firstWindow));
    assertFalse(tester.shouldFire(secondWindow));
    assertFalse(tester.shouldFire(thirdWindow));
    // At 50, the first becomes ready; it stays ready after firing
    tester.advanceInputWatermark(new Instant(50));
    assertTrue(tester.shouldFire(firstWindow));
    assertFalse(tester.shouldFire(secondWindow));
    assertFalse(tester.shouldFire(thirdWindow));
    tester.fireIfShouldFire(firstWindow);
    assertTrue(tester.shouldFire(firstWindow));
    assertFalse(tester.shouldFire(secondWindow));
    assertFalse(tester.shouldFire(thirdWindow));
    // At 99, the first is still the only one ready
    tester.advanceInputWatermark(new Instant(99));
    assertTrue(tester.shouldFire(firstWindow));
    assertFalse(tester.shouldFire(secondWindow));
    assertFalse(tester.shouldFire(thirdWindow));
    // At 100, the first and second are ready
    tester.advanceInputWatermark(new Instant(100));
    assertTrue(tester.shouldFire(firstWindow));
    assertTrue(tester.shouldFire(secondWindow));
    assertFalse(tester.shouldFire(thirdWindow));
    tester.fireIfShouldFire(firstWindow);
    assertFalse(tester.isMarkedFinished(firstWindow));
    assertFalse(tester.isMarkedFinished(secondWindow));
    assertFalse(tester.isMarkedFinished(thirdWindow));
}

Example 50 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class GroupAlsoByWindowProperties method groupsElementsInMergedSessionsWithLatestTimestamp.

/**
 * Tests that the given GABW implementation correctly groups elements into merged sessions with
 * output timestamps at the end of the merged window.
 */
public static void groupsElementsInMergedSessionsWithLatestTimestamp(GroupAlsoByWindowDoFnFactory<String, String, Iterable<String>> gabwFactory) throws Exception {
    WindowingStrategy<?, IntervalWindow> windowingStrategy = WindowingStrategy.of(Sessions.withGapDuration(Duration.millis(10))).withTimestampCombiner(TimestampCombiner.LATEST);
    BoundedWindow unmergedWindow = window(15, 25);
    List<WindowedValue<KV<String, Iterable<String>>>> result = runGABW(gabwFactory, windowingStrategy, "k", WindowedValue.of("v1", new Instant(0), Arrays.asList(window(0, 10)), PaneInfo.NO_FIRING), WindowedValue.of("v2", new Instant(5), Arrays.asList(window(5, 15)), PaneInfo.NO_FIRING), WindowedValue.of("v3", new Instant(15), Arrays.asList(unmergedWindow), PaneInfo.NO_FIRING));
    assertThat(result, hasSize(2));
    BoundedWindow mergedWindow = window(0, 15);
    TimestampedValue<KV<String, Iterable<String>>> item0 = getOnlyElementInWindow(result, mergedWindow);
    assertThat(item0.getValue().getValue(), containsInAnyOrder("v1", "v2"));
    assertThat(item0.getTimestamp(), equalTo(new Instant(5)));
    TimestampedValue<KV<String, Iterable<String>>> item1 = getOnlyElementInWindow(result, unmergedWindow);
    assertThat(item1.getValue().getValue(), contains("v3"));
    assertThat(item1.getTimestamp(), equalTo(new Instant(15)));
}