Examples with WindowedValue org.apache.beam.sdk.util.WindowedValue used on opensource projects

Example 16 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class SimplePushbackSideInputDoFnRunnerTest method processElementSideInputNotReady.

@Test
public void processElementSideInputNotReady() {
    when(reader.isReady(Mockito.eq(singletonView), Mockito.any(BoundedWindow.class))).thenReturn(false);
    SimplePushbackSideInputDoFnRunner<Integer, Integer> runner = createRunner(ImmutableList.<PCollectionView<?>>of(singletonView));
    WindowedValue<Integer> oneWindow = WindowedValue.of(2, new Instant(-2), new IntervalWindow(new Instant(-500L), new Instant(0L)), PaneInfo.ON_TIME_AND_ONLY_FIRING);
    Iterable<WindowedValue<Integer>> oneWindowPushback = runner.processElementInReadyWindows(oneWindow);
    assertThat(oneWindowPushback, containsInAnyOrder(oneWindow));
    assertThat(underlying.inputElems, Matchers.<WindowedValue<Integer>>emptyIterable());
}

Example 17 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testWatermarkHoldAndLateData.

@Test
public void testWatermarkHoldAndLateData() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    // Test handling of late data. Specifically, ensure the watermark hold is correct.
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.ACCUMULATING_FIRED_PANES, Duration.millis(10), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // Input watermark -> null
    assertEquals(null, tester.getWatermarkHold());
    assertEquals(null, tester.getOutputWatermark());
    // All on time data, verify watermark hold.
    injectElement(tester, 1);
    injectElement(tester, 3);
    assertEquals(new Instant(1), tester.getWatermarkHold());
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 2);
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output, contains(isSingleWindowedValue(containsInAnyOrder(1, 2, 3), // timestamp
    1, // window start
    0, // window end
    10)));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
    // Holding for the end-of-window transition.
    assertEquals(new Instant(9), tester.getWatermarkHold());
    // Nothing dropped.
    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative().longValue();
    assertEquals(0, droppedElements);
    // Input watermark -> 4, output watermark should advance that far as well
    tester.advanceInputWatermark(new Instant(4));
    assertEquals(new Instant(4), tester.getOutputWatermark());
    // Some late, some on time. Verify that we only hold to the minimum of on-time.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false);
    tester.advanceInputWatermark(new Instant(4));
    injectElement(tester, 2);
    injectElement(tester, 3);
    assertEquals(new Instant(9), tester.getWatermarkHold());
    injectElement(tester, 5);
    assertEquals(new Instant(5), tester.getWatermarkHold());
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 4);
    output = tester.extractOutput();
    assertThat(output, contains(isSingleWindowedValue(containsInAnyOrder(// earlier firing
    1, // earlier firing
    2, // earlier firing
    3, 2, 3, 4, // new elements
    5), // timestamp
    4, // window start
    0, // window end
    10)));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(false, false, Timing.EARLY, 1, -1)));
    // All late -- output at end of window timestamp.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false);
    tester.advanceInputWatermark(new Instant(8));
    injectElement(tester, 6);
    injectElement(tester, 5);
    assertEquals(new Instant(9), tester.getWatermarkHold());
    injectElement(tester, 4);
    // Fire the ON_TIME pane
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.advanceInputWatermark(new Instant(10));
    // Output time is end of the window, because all the new data was late, but the pane
    // is the ON_TIME pane.
    output = tester.extractOutput();
    assertThat(output, contains(isSingleWindowedValue(containsInAnyOrder(// earlier firing
    1, // earlier firing
    2, // earlier firing
    3, // earlier firing
    2, // earlier firing
    3, // earlier firing
    4, // earlier firing
    5, 4, 5, // new elements
    6), // timestamp
    9, // window start
    0, // window end
    10)));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(false, false, Timing.ON_TIME, 2, 0)));
    // This is "pending" at the time the watermark makes it way-late.
    // Because we're about to expire the window, we output it.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false);
    injectElement(tester, 8);
    droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative().longValue();
    assertEquals(0, droppedElements);
    // Exceed the GC limit, triggering the last pane to be fired
    tester.advanceInputWatermark(new Instant(50));
    output = tester.extractOutput();
    // Output time is still end of the window, because the new data (8) was behind
    // the output watermark.
    assertThat(output, contains(isSingleWindowedValue(containsInAnyOrder(// earlier firing
    1, // earlier firing
    2, // earlier firing
    3, // earlier firing
    2, // earlier firing
    3, // earlier firing
    4, // earlier firing
    5, // earlier firing
    4, // earlier firing
    5, // earlier firing
    6, // new element prior to window becoming expired
    8), // timestamp
    9, // window start
    0, // window end
    10)));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(false, true, Timing.LATE, 3, 1)));
    assertEquals(new Instant(50), tester.getOutputWatermark());
    assertEquals(null, tester.getWatermarkHold());
    // Late timers are ignored
    tester.fireTimer(new IntervalWindow(new Instant(0), new Instant(10)), new Instant(12), TimeDomain.EVENT_TIME);
    // And because we're past the end of window + allowed lateness, everything should be cleaned up.
    assertFalse(tester.isMarkedFinished(firstWindow));
    tester.assertHasOnlyGlobalAndFinishedSetsFor();
}

Example 18 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testIdempotentEmptyPanesDiscarding.

@Test
public void testIdempotentEmptyPanesDiscarding() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    // Test uninteresting (empty) panes don't increment the index or otherwise
    // modify PaneInfo.
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // Inject a couple of on-time elements and fire at the window end.
    injectElement(tester, 1);
    injectElement(tester, 2);
    tester.advanceInputWatermark(new Instant(12));
    // Fire the on-time pane
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.fireTimer(firstWindow, new Instant(9), TimeDomain.EVENT_TIME);
    // Fire another timer (with no data, so it's an uninteresting pane that should not be output).
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.fireTimer(firstWindow, new Instant(9), TimeDomain.EVENT_TIME);
    // Finish it off with another datum.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    injectElement(tester, 3);
    // The intermediate trigger firing shouldn't result in any output.
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output.size(), equalTo(2));
    // The on-time pane is as expected.
    assertThat(output.get(0), isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10));
    // The late pane has the correct indices.
    assertThat(output.get(1).getValue(), contains(3));
    assertThat(output.get(1).getPane(), equalTo(PaneInfo.createPane(false, true, Timing.LATE, 1, 1)));
    assertTrue(tester.isMarkedFinished(firstWindow));
    tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow);
    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative().longValue();
    assertEquals(0, droppedElements);
}

Example 19 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testEmptyOnTimeFromOrFinally.

/**
   * Test that we receive an empty on-time pane when an or-finally waiting for the watermark fires.
   * Specifically, verify the proper triggerings and pane-info of a typical speculative/on-time/late
   * when the on-time pane is empty.
   */
@Test
public void testEmptyOnTimeFromOrFinally() throws Exception {
    WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10))).withTimestampCombiner(TimestampCombiner.EARLIEST).withTrigger(AfterEach.<IntervalWindow>inOrder(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(new Duration(5))).orFinally(AfterWatermark.pastEndOfWindow()), Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(new Duration(25))))).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.millis(100));
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
    tester.advanceInputWatermark(new Instant(0));
    tester.advanceProcessingTime(new Instant(0));
    // Processing time timer for 5
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(1, new Instant(3)), TimestampedValue.of(1, new Instant(7)), TimestampedValue.of(1, new Instant(5)));
    // Should fire early pane
    tester.advanceProcessingTime(new Instant(6));
    // Should fire empty on time pane
    tester.advanceInputWatermark(new Instant(11));
    List<WindowedValue<Integer>> output = tester.extractOutput();
    assertEquals(2, output.size());
    assertThat(output.get(0), WindowMatchers.isSingleWindowedValue(4, 1, 0, 10));
    assertThat(output.get(1), WindowMatchers.isSingleWindowedValue(4, 9, 0, 10));
    assertThat(output.get(0), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
    assertThat(output.get(1), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.ON_TIME, 1, 0)));
}

Example 20 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method setGarbageCollectionHoldOnLateElements.

/**
   * Late elements should still have a garbage collection hold set so that they
   * can make a late pane rather than be dropped due to lateness.
   */
@Test
public void setGarbageCollectionHoldOnLateElements() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(FixedWindows.of(Duration.millis(10))).withTrigger(AfterWatermark.pastEndOfWindow().withLateFirings(AfterPane.elementCountAtLeast(2))).withMode(AccumulationMode.DISCARDING_FIRED_PANES).withAllowedLateness(Duration.millis(100)).withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY));
    tester.advanceInputWatermark(new Instant(0));
    tester.advanceOutputWatermark(new Instant(0));
    tester.injectElements(TimestampedValue.of(1, new Instant(1)));
    // Fire ON_TIME pane @ 9 with 1
    tester.advanceInputWatermark(new Instant(109));
    tester.advanceOutputWatermark(new Instant(109));
    tester.injectElements(TimestampedValue.of(2, new Instant(2)));
    // We should have set a garbage collection hold for the final pane.
    Instant hold = tester.getWatermarkHold();
    assertEquals(new Instant(109), hold);
    tester.advanceInputWatermark(new Instant(110));
    tester.advanceOutputWatermark(new Instant(110));
    // Fire final LATE pane @ 9 with 2
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertEquals(2, output.size());
}