Examples with IntervalWindow org.apache.beam.sdk.transforms.windowing.IntervalWindow used on opensource projects

Example 76 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testOnlyOneOnTimePane.

/**
 * Tests that with the default trigger we will not produce two ON_TIME panes, even if there are
 * two outputs that are both candidates.
 */
@Test
public void testOnlyOneOnTimePane() throws Exception {
    WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10))).withTrigger(DefaultTrigger.of()).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.millis(100));
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
    tester.advanceInputWatermark(new Instant(0));
    int value1 = 1;
    int value2 = 3;
    // A single element that should be in the ON_TIME output
    tester.injectElements(TimestampedValue.of(value1, new Instant(1)));
    // Should fire ON_TIME
    tester.advanceInputWatermark(new Instant(10));
    // The DefaultTrigger should cause output labeled LATE, even though it does not have to be
    // labeled as such.
    tester.injectElements(TimestampedValue.of(value2, new Instant(3)));
    List<WindowedValue<Integer>> output = tester.extractOutput();
    assertEquals(2, output.size());
    assertThat(output.get(0), isWindowedValue(equalTo(value1)));
    assertThat(output.get(1), isWindowedValue(equalTo(value1 + value2)));
    assertThat(output.get(0), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.ON_TIME, 0, 0)));
    assertThat(output.get(1), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.LATE, 1, 1)));
}

Example 77 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testEmptyOnTimeFromOrFinally.

/**
 * Test that we receive an empty on-time pane when an or-finally waiting for the watermark fires.
 * Specifically, verify the proper triggerings and pane-info of a typical speculative/on-time/late
 * when the on-time pane is empty.
 */
@Test
public void testEmptyOnTimeFromOrFinally() throws Exception {
    WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10))).withTimestampCombiner(TimestampCombiner.EARLIEST).withTrigger(AfterEach.inOrder(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(5))).orFinally(AfterWatermark.pastEndOfWindow()), Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(25))))).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.millis(100));
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
    tester.advanceInputWatermark(new Instant(0));
    tester.advanceProcessingTime(new Instant(0));
    // Processing time timer for 5
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(1, new Instant(3)), TimestampedValue.of(1, new Instant(7)), TimestampedValue.of(1, new Instant(5)));
    // Should fire early pane
    tester.advanceProcessingTime(new Instant(6));
    // Should fire empty on time pane
    tester.advanceInputWatermark(new Instant(11));
    List<WindowedValue<Integer>> output = tester.extractOutput();
    assertEquals(2, output.size());
    assertThat(output.get(0), isSingleWindowedValue(4, 1, 0, 10));
    assertThat(output.get(1), isSingleWindowedValue(4, 9, 0, 10));
    assertThat(output.get(0), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
    assertThat(output.get(1), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.ON_TIME, 1, 0)));
}

Example 78 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testCombiningAccumulatingProcessingTimeSeparateBundles.

/**
 * Tests that when a processing time timers comes in after a window is expired and GC'd it does
 * not cause a spurious output.
 */
@Test
public void testCombiningAccumulatingProcessingTimeSeparateBundles() throws Exception {
    WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100))).withTimestampCombiner(TimestampCombiner.EARLIEST).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.ZERO).withTrigger(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10))));
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
    tester.advanceProcessingTime(new Instant(5000));
    // processing timer @ 5000 + 10; EOW timer @ 100
    injectElement(tester, 2);
    injectElement(tester, 5);
    tester.advanceInputWatermark(new Instant(100));
    tester.advanceProcessingTime(new Instant(5011));
    assertThat(tester.extractOutput(), contains(isSingleWindowedValue(equalTo(7), 2, 0, 100, PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0))));
}

Example 79 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testMergingWatermarkHoldLateNewWindowMerged.

@Test
public void testMergingWatermarkHoldLateNewWindowMerged() throws Exception {
    Duration allowedLateness = Duration.standardMinutes(1);
    Duration gapDuration = Duration.millis(10);
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(Sessions.withGapDuration(gapDuration)).withMode(AccumulationMode.DISCARDING_FIRED_PANES).withTrigger(Repeatedly.forever(AfterWatermark.pastEndOfWindow().withLateFirings(AfterPane.elementCountAtLeast(1)))).withAllowedLateness(allowedLateness));
    tester.setAutoAdvanceOutputWatermark(false);
    assertEquals(null, tester.getWatermarkHold());
    assertEquals(null, tester.getOutputWatermark());
    tester.advanceInputWatermark(new Instant(24));
    injectElements(tester, 1);
    assertThat(tester.getWatermarkHold(), nullValue());
    injectElements(tester, 14);
    assertThat(tester.getWatermarkHold(), nullValue());
    injectElements(tester, 6, 16);
    // There should now be a watermark hold since the window has extended past the input watermark.
    // The hold should be for the end of the window (last element + gapDuration - 1).
    assertEquals(tester.getWatermarkHold(), new Instant(25));
    injectElements(tester, 6, 21);
    // The hold should be extended with the window.
    assertEquals(tester.getWatermarkHold(), new Instant(30));
    // Advancing the watermark should remove the hold.
    tester.advanceInputWatermark(new Instant(31));
    assertThat(tester.getWatermarkHold(), nullValue());
    // Late elements added to the window should not generate a hold.
    injectElements(tester, 0);
    assertThat(tester.getWatermarkHold(), nullValue());
    // Generate a new window that is ontime.
    injectElements(tester, 32, 40);
    assertEquals(tester.getWatermarkHold(), new Instant(49));
    // Join the closed window with the new window.
    injectElements(tester, 24);
    assertEquals(tester.getWatermarkHold(), new Instant(49));
    tester.advanceInputWatermark(new Instant(50));
    assertThat(tester.getWatermarkHold(), nullValue());
}

Example 80 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testSessionEowAndGcTogether.

/**
 * When the watermark passes the end-of-window and window expiration time in a single update, this
 * tests that it does not crash.
 */
@Test
public void testSessionEowAndGcTogether() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(Sessions.withGapDuration(Duration.millis(10)), DefaultTriggerStateMachine.of(), AccumulationMode.ACCUMULATING_FIRED_PANES, Duration.millis(50), ClosingBehavior.FIRE_ALWAYS);
    tester.setAutoAdvanceOutputWatermark(true);
    tester.advanceInputWatermark(new Instant(0));
    injectElement(tester, 1);
    tester.advanceInputWatermark(new Instant(100));
    assertThat(tester.extractOutput(), contains(isSingleWindowedValue(contains(1), 1, 1, 11, PaneInfo.createPane(true, true, Timing.ON_TIME))));
}