Examples with IntervalWindow org.apache.beam.sdk.transforms.windowing.IntervalWindow used on opensource projects

Example 91 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testPaneInfoAllStates.

@Test
public void testPaneInfoAllStates() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY);
    tester.advanceInputWatermark(new Instant(0));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 1);
    assertThat(tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY))));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 2);
    assertThat(tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.EARLY, 1, -1))));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false);
    tester.setAutoAdvanceOutputWatermark(false);
    tester.advanceInputWatermark(new Instant(15));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 3);
    assertThat(tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.ON_TIME, 2, 0))));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.setAutoAdvanceOutputWatermark(true);
    injectElement(tester, 4);
    assertThat(tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.LATE, 3, 1))));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    injectElement(tester, 5);
    assertThat(tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.LATE, 4, 2))));
}

Example 92 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testPaneInfoAllStatesAfterWatermark.

@Test
public void testPaneInfoAllStatesAfterWatermark() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(FixedWindows.of(Duration.millis(10))).withTrigger(Repeatedly.forever(AfterFirst.of(AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))).withMode(AccumulationMode.DISCARDING_FIRED_PANES).withAllowedLateness(Duration.millis(100)).withTimestampCombiner(TimestampCombiner.EARLIEST).withClosingBehavior(ClosingBehavior.FIRE_ALWAYS));
    tester.advanceInputWatermark(new Instant(0));
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1))));
    assertThat(output, contains(isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10)));
    tester.advanceInputWatermark(new Instant(50));
    // We should get the ON_TIME pane even though it is empty,
    // because we have an AfterWatermark.pastEndOfWindow() trigger.
    output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.ON_TIME, 1, 0))));
    assertThat(output, contains(isSingleWindowedValue(emptyIterable(), 9, 0, 10)));
    // We should get the final pane even though it is empty.
    tester.advanceInputWatermark(new Instant(150));
    output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.LATE, 2, 1))));
    assertThat(output, contains(isSingleWindowedValue(emptyIterable(), 9, 0, 10)));
}

Example 93 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testMergingWithClosedRepresentative.

/**
 * When a merged window's trigger is closed we record that state using the merged window rather
 * than the original windows.
 */
@Test
public void testMergingWithClosedRepresentative() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(Sessions.withGapDuration(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(50), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // 2 elements into merged session window.
    // Close the trigger, but the garbage collection timer is still pending.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    tester.injectElements(// in [1, 11), gc at 21.
    TimestampedValue.of(1, new Instant(1)), // in [8, 18), gc at 28.
    TimestampedValue.of(8, new Instant(8)));
    // More elements into the same merged session window.
    // It has not yet been gced.
    // Should be discarded with 'window closed'.
    tester.injectElements(// in [1, 11), gc at 21.
    TimestampedValue.of(1, new Instant(1)), // in [2, 12), gc at 22.
    TimestampedValue.of(2, new Instant(2)), // in [8, 18), gc at 28.
    TimestampedValue.of(8, new Instant(8)));
    // Now the garbage collection timer will fire, finding the trigger already closed.
    tester.advanceInputWatermark(new Instant(100));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output.size(), equalTo(1));
    assertThat(output.get(0), isSingleWindowedValue(containsInAnyOrder(1, 8), // timestamp
    1, // window start
    1, // window end
    18));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(true, true, Timing.EARLY, 0, 0)));
}

Example 94 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class SideInputHandlerTest method testMultipleSideInputs.

@Test
public void testMultipleSideInputs() {
    long windowSize = 100;
    PCollectionView<Iterable<String>> view1;
    PCollectionView<Iterable<String>> view2;
    PCollection<String> pc = Pipeline.create().apply(Create.of("1"));
    view1 = pc.apply(Window.into(FixedWindows.of(Duration.millis(windowSize)))).apply(View.asIterable());
    view2 = pc.apply(Window.into(FixedWindows.of(Duration.millis(windowSize)))).apply(View.asIterable());
    SideInputHandler sideInputHandler = new SideInputHandler(ImmutableList.of(view1, view2), InMemoryStateInternals.<Void>forKey(null));
    // two windows that we'll later use for adding elements/retrieving side input
    IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(windowSize));
    // add value for view1 in the first window
    sideInputHandler.addSideInputValue(view1, valuesInWindow(materializeValuesFor(view1.getPipeline().getOptions(), View.asIterable(), "Hello"), new Instant(0), window));
    assertThat(sideInputHandler.get(view1, window), contains("Hello"));
    // view2 should not have any data
    assertFalse(sideInputHandler.isReady(view2, window));
    // also add some data for view2
    sideInputHandler.addSideInputValue(view2, valuesInWindow(materializeValuesFor(view2.getPipeline().getOptions(), View.asIterable(), "Salut"), new Instant(0), window));
    assertTrue(sideInputHandler.isReady(view2, window));
    assertThat(sideInputHandler.get(view2, window), contains("Salut"));
    // view1 should not be affected by that
    assertThat(sideInputHandler.get(view1, window), contains("Hello"));
}

Example 95 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class SideInputHandlerTest method testIsReady.

@Test
public void testIsReady() {
    long view1WindowSize = 100;
    PCollection<String> pc = Pipeline.create().apply(Create.of("1"));
    PCollectionView<Iterable<String>> view1 = pc.apply(Window.into(FixedWindows.of(Duration.millis(view1WindowSize)))).apply(View.asIterable());
    // Unused, just to have a non-trivial handler set up
    PCollectionView<Iterable<String>> view2 = pc.apply(View.asIterable());
    SideInputHandler sideInputHandler = new SideInputHandler(ImmutableList.of(view1, view2), InMemoryStateInternals.<Void>forKey(null));
    // Adjacent fixed windows
    IntervalWindow firstWindow = new IntervalWindow(new Instant(0), new Instant(view1WindowSize));
    IntervalWindow secondWindow = new IntervalWindow(new Instant(view1WindowSize), new Instant(view1WindowSize * 2));
    // side input should not yet be ready in first window
    assertFalse(sideInputHandler.isReady(view1, firstWindow));
    // add a value for view1
    sideInputHandler.addSideInputValue(view1, valuesInWindow(materializeValuesFor(view1.getPipeline().getOptions(), View.asIterable(), "Hello"), new Instant(0), firstWindow));
    // now side input should be ready in first window
    assertTrue(sideInputHandler.isReady(view1, firstWindow));
    // second window input should still not be ready
    assertFalse(sideInputHandler.isReady(view1, secondWindow));
}