Examples with WindowedValue org.apache.beam.sdk.util.WindowedValue used on opensource projects

Example 51 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class LateDataDroppingDoFnRunnerTest method testLateDataFilter.

@Test
public void testLateDataFilter() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    when(mockTimerInternals.currentInputWatermarkTime()).thenReturn(new Instant(15L));
    LateDataFilter lateDataFilter = new LateDataFilter(WindowingStrategy.of(WINDOW_FN), mockTimerInternals);
    Iterable<WindowedValue<Integer>> actual = lateDataFilter.filter("a", ImmutableList.of(createDatum(13, 13L), // late element, earlier than 4L.
    createDatum(5, 5L), createDatum(16, 16L), createDatum(18, 18L)));
    Iterable<WindowedValue<Integer>> expected = ImmutableList.of(createDatum(13, 13L), createDatum(16, 16L), createDatum(18, 18L));
    assertThat(expected, containsInAnyOrder(Iterables.toArray(actual, WindowedValue.class)));
    long droppedValues = container.getCounter(MetricName.named(LateDataDroppingDoFnRunner.class, LateDataDroppingDoFnRunner.DROPPED_DUE_TO_LATENESS)).getCumulative().longValue();
    assertEquals(1, droppedValues);
    // Ensure that reiterating returns the same results and doesn't increment the counter again.
    assertThat(expected, containsInAnyOrder(Iterables.toArray(actual, WindowedValue.class)));
    droppedValues = container.getCounter(MetricName.named(LateDataDroppingDoFnRunner.class, LateDataDroppingDoFnRunner.DROPPED_DUE_TO_LATENESS)).getCumulative().longValue();
    assertEquals(1, droppedValues);
}

Example 52 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testIdempotentEmptyPanesAccumulating.

@Test
public void testIdempotentEmptyPanesAccumulating() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    // Test uninteresting (empty) panes don't increment the index or otherwise
    // modify PaneInfo.
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.ACCUMULATING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // Inject a couple of on-time elements and fire at the window end.
    injectElement(tester, 1);
    injectElement(tester, 2);
    tester.advanceInputWatermark(new Instant(12));
    // Trigger the on-time pane
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.fireTimer(firstWindow, new Instant(9), TimeDomain.EVENT_TIME);
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output.size(), equalTo(1));
    assertThat(output.get(0), isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(true, false, Timing.ON_TIME, 0, 0)));
    // Fire another timer with no data; the empty pane should not be output even though the
    // trigger is ready to fire
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.fireTimer(firstWindow, new Instant(9), TimeDomain.EVENT_TIME);
    assertThat(tester.extractOutput().size(), equalTo(0));
    // Finish it off with another datum, which is late
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    injectElement(tester, 3);
    output = tester.extractOutput();
    assertThat(output.size(), equalTo(1));
    // The late pane has the correct indices.
    assertThat(output.get(0).getValue(), containsInAnyOrder(1, 2, 3));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(false, true, Timing.LATE, 1, 1)));
    assertTrue(tester.isMarkedFinished(firstWindow));
    tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow);
    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative().longValue();
    assertEquals(0, droppedElements);
}

Example 53 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testMergingWithClosedDoesNotPoison.

/**
   * If an element for a closed session window ends up being merged into other still-open
   * session windows, the resulting session window is not 'poisoned'.
   */
@Test
public void testMergingWithClosedDoesNotPoison() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(Sessions.withGapDuration(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(50), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // 1 element, force its trigger to close.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    tester.injectElements(TimestampedValue.of(2, new Instant(2)));
    // 3 elements, one already closed.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false);
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)), TimestampedValue.of(3, new Instant(3)));
    tester.advanceInputWatermark(new Instant(100));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output.size(), equalTo(2));
    assertThat(output.get(0), isSingleWindowedValue(containsInAnyOrder(2), // timestamp
    2, // window start
    2, // window end
    12));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(true, true, Timing.EARLY, 0, 0)));
    assertThat(output.get(1), isSingleWindowedValue(containsInAnyOrder(1, 2, 3), // timestamp
    1, // window start
    1, // window end
    13));
    assertThat(output.get(1).getPane(), equalTo(PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0)));
}

Example 54 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method fireEmptyOnDrainInGlobalWindowIfRequested.

/**
   * We should fire an empty ON_TIME pane in the GlobalWindow when the watermark moves to
   * end-of-time.
   */
@Test
public void fireEmptyOnDrainInGlobalWindowIfRequested() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, GlobalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(new GlobalWindows()).withTrigger(Repeatedly.<GlobalWindow>forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(new Duration(3)))).withMode(AccumulationMode.DISCARDING_FIRED_PANES));
    final int n = 20;
    for (int i = 0; i < n; i++) {
        tester.advanceProcessingTime(new Instant(i));
        tester.injectElements(TimestampedValue.of(i, new Instant(i)));
    }
    tester.advanceProcessingTime(new Instant(n + 4));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertEquals((n + 3) / 4, output.size());
    for (int i = 0; i < output.size(); i++) {
        assertEquals(Timing.EARLY, output.get(i).getPane().getTiming());
        assertEquals(i, output.get(i).getPane().getIndex());
        assertEquals(4, Iterables.size(output.get(i).getValue()));
    }
    tester.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
    output = tester.extractOutput();
    assertEquals(1, output.size());
    assertEquals(Timing.ON_TIME, output.get(0).getPane().getTiming());
    assertEquals((n + 3) / 4, output.get(0).getPane().getIndex());
    assertEquals(0, Iterables.size(output.get(0).getValue()));
}

Example 55 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testMergingWithCloseBeforeGC.

/**
   * It is possible for a session window's trigger to be closed at the point at which
   * the (merged) session window is garbage collected. Make sure we don't accidentally
   * assume the window is still active.
   */
@Test
public void testMergingWithCloseBeforeGC() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(Sessions.withGapDuration(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(50), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // Two elements in two overlapping session windows.
    tester.injectElements(// in [1, 11)
    TimestampedValue.of(1, new Instant(1)), // in [10, 20)
    TimestampedValue.of(10, new Instant(10)));
    // Close the trigger, but the gargbage collection timer is still pending.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    tester.advanceInputWatermark(new Instant(30));
    // Now the garbage collection timer will fire, finding the trigger already closed.
    tester.advanceInputWatermark(new Instant(100));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output.size(), equalTo(1));
    assertThat(output.get(0), isSingleWindowedValue(containsInAnyOrder(1, 10), // timestamp
    1, // window start
    1, // window end
    20));
    assertThat(output.get(0).getPane(), equalTo(PaneInfo.createPane(true, true, Timing.ON_TIME, 0, 0)));
}