Examples with WindowMatchers.isSingleWindowedValue org.apache.beam.runners.core.WindowMatchers.isSingleWindowedValue used on opensource projects

Example 1 with WindowMatchers.isSingleWindowedValue

use of org.apache.beam.runners.core.WindowMatchers.isSingleWindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testProcessingTime.

/**
   * Tests for processing time firings after the watermark passes the end of the window.
   * Specifically, verify the proper triggerings and pane-info of a typical speculative/on-time/late
   * when the on-time pane is non-empty.
   */
@Test
public void testProcessingTime() throws Exception {
    WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10))).withTimestampCombiner(TimestampCombiner.EARLIEST).withTrigger(AfterEach.<IntervalWindow>inOrder(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(new Duration(5))).orFinally(AfterWatermark.pastEndOfWindow()), Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(new Duration(25))))).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.millis(100));
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
    tester.advanceInputWatermark(new Instant(0));
    tester.advanceProcessingTime(new Instant(0));
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(1, new Instant(3)), TimestampedValue.of(1, new Instant(7)), TimestampedValue.of(1, new Instant(5)));
    // 4 elements all at processing time 0
    // fire [1,3,7,5] since 6 > 0 + 5
    tester.advanceProcessingTime(new Instant(6));
    tester.injectElements(TimestampedValue.of(1, new Instant(8)), TimestampedValue.of(1, new Instant(4)));
    // 6 elements
    // fire [1,3,7,5,8,4] since 11 > 9
    tester.advanceInputWatermark(new Instant(11));
    tester.injectElements(TimestampedValue.of(1, new Instant(8)), TimestampedValue.of(1, new Instant(4)), TimestampedValue.of(1, new Instant(5)));
    // 9 elements
    tester.advanceInputWatermark(new Instant(12));
    tester.injectElements(TimestampedValue.of(1, new Instant(3)));
    // 10 elements
    tester.advanceProcessingTime(new Instant(15));
    tester.injectElements(TimestampedValue.of(1, new Instant(5)));
    // 11 elements
    // fire since 32 > 6 + 25
    tester.advanceProcessingTime(new Instant(32));
    tester.injectElements(TimestampedValue.of(1, new Instant(3)));
    // 12 elements
    // fire [1,3,7,5,8,4,8,4,5,3,5,3] since 125 > 6 + 25
    tester.advanceInputWatermark(new Instant(125));
    List<WindowedValue<Integer>> output = tester.extractOutput();
    assertEquals(4, output.size());
    assertThat(output.get(0), WindowMatchers.isSingleWindowedValue(4, 1, 0, 10));
    assertThat(output.get(1), WindowMatchers.isSingleWindowedValue(6, 4, 0, 10));
    assertThat(output.get(2), WindowMatchers.isSingleWindowedValue(11, 9, 0, 10));
    assertThat(output.get(3), WindowMatchers.isSingleWindowedValue(12, 9, 0, 10));
    assertThat(output.get(0), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
    assertThat(output.get(1), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.ON_TIME, 1, 0)));
    assertThat(output.get(2), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.LATE, 2, 1)));
    assertThat(output.get(3), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.LATE, 3, 2)));
}

Example 2 with WindowMatchers.isSingleWindowedValue

use of org.apache.beam.runners.core.WindowMatchers.isSingleWindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testPaneInfoAllStatesAfterWatermarkAccumulating.

@Test
public void testPaneInfoAllStatesAfterWatermarkAccumulating() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(FixedWindows.of(Duration.millis(10))).withTrigger(Repeatedly.forever(AfterFirst.of(AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.millis(100)).withTimestampCombiner(TimestampCombiner.EARLIEST).withClosingBehavior(ClosingBehavior.FIRE_ALWAYS));
    tester.advanceInputWatermark(new Instant(0));
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1))));
    assertThat(output, contains(WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10)));
    tester.advanceInputWatermark(new Instant(50));
    // We should get the ON_TIME pane even though it is empty,
    // because we have an AfterWatermark.pastEndOfWindow() trigger.
    output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.ON_TIME, 1, 0))));
    assertThat(output, contains(WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10)));
    // We should get the final pane even though it is empty.
    tester.advanceInputWatermark(new Instant(150));
    output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.LATE, 2, 1))));
    assertThat(output, contains(WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10)));
}

Example 3 with WindowMatchers.isSingleWindowedValue

use of org.apache.beam.runners.core.WindowMatchers.isSingleWindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testEmptyOnTimeFromOrFinally.

/**
   * Test that we receive an empty on-time pane when an or-finally waiting for the watermark fires.
   * Specifically, verify the proper triggerings and pane-info of a typical speculative/on-time/late
   * when the on-time pane is empty.
   */
@Test
public void testEmptyOnTimeFromOrFinally() throws Exception {
    WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10))).withTimestampCombiner(TimestampCombiner.EARLIEST).withTrigger(AfterEach.<IntervalWindow>inOrder(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(new Duration(5))).orFinally(AfterWatermark.pastEndOfWindow()), Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(new Duration(25))))).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.millis(100));
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
    tester.advanceInputWatermark(new Instant(0));
    tester.advanceProcessingTime(new Instant(0));
    // Processing time timer for 5
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(1, new Instant(3)), TimestampedValue.of(1, new Instant(7)), TimestampedValue.of(1, new Instant(5)));
    // Should fire early pane
    tester.advanceProcessingTime(new Instant(6));
    // Should fire empty on time pane
    tester.advanceInputWatermark(new Instant(11));
    List<WindowedValue<Integer>> output = tester.extractOutput();
    assertEquals(2, output.size());
    assertThat(output.get(0), WindowMatchers.isSingleWindowedValue(4, 1, 0, 10));
    assertThat(output.get(1), WindowMatchers.isSingleWindowedValue(4, 9, 0, 10));
    assertThat(output.get(0), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
    assertThat(output.get(1), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.ON_TIME, 1, 0)));
}

Example 4 with WindowMatchers.isSingleWindowedValue

use of org.apache.beam.runners.core.WindowMatchers.isSingleWindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testPaneInfoAllStatesAfterWatermark.

@Test
public void testPaneInfoAllStatesAfterWatermark() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(FixedWindows.of(Duration.millis(10))).withTrigger(Repeatedly.forever(AfterFirst.of(AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))).withMode(AccumulationMode.DISCARDING_FIRED_PANES).withAllowedLateness(Duration.millis(100)).withTimestampCombiner(TimestampCombiner.EARLIEST).withClosingBehavior(ClosingBehavior.FIRE_ALWAYS));
    tester.advanceInputWatermark(new Instant(0));
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1))));
    assertThat(output, contains(WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10)));
    tester.advanceInputWatermark(new Instant(50));
    // We should get the ON_TIME pane even though it is empty,
    // because we have an AfterWatermark.pastEndOfWindow() trigger.
    output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.ON_TIME, 1, 0))));
    assertThat(output, contains(WindowMatchers.isSingleWindowedValue(emptyIterable(), 9, 0, 10)));
    // We should get the final pane even though it is empty.
    tester.advanceInputWatermark(new Instant(150));
    output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.LATE, 2, 1))));
    assertThat(output, contains(WindowMatchers.isSingleWindowedValue(emptyIterable(), 9, 0, 10)));
}

Example 5 with WindowMatchers.isSingleWindowedValue

use of org.apache.beam.runners.core.WindowMatchers.isSingleWindowedValue in project beam by apache.

the class ReduceFnRunnerTest method noEmptyPanesFinalIfNonEmpty.

@Test
public void noEmptyPanesFinalIfNonEmpty() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(FixedWindows.of(Duration.millis(10))).withTrigger(Repeatedly.<IntervalWindow>forever(AfterFirst.<IntervalWindow>of(AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.millis(100)).withTimestampCombiner(TimestampCombiner.EARLIEST).withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY));
    tester.advanceInputWatermark(new Instant(0));
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
    tester.advanceInputWatermark(new Instant(20));
    tester.advanceInputWatermark(new Instant(250));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output, contains(// Trigger with 2 elements
    WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), // Trigger for the empty on time pane
    WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10)));
}