Examples with IntervalWindow org.apache.beam.sdk.transforms.windowing.IntervalWindow used on opensource projects

Example 86 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testDropDataMultipleWindowsFinishedTrigger.

/**
 * Tests that when data is assigned to multiple windows but some of those windows have had their
 * triggers finish, then the data is dropped and counted accurately.
 */
@Test
public void testDropDataMultipleWindowsFinishedTrigger() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(WindowingStrategy.of(SlidingWindows.of(Duration.millis(100)).every(Duration.millis(30))).withTrigger(AfterWatermark.pastEndOfWindow()).withAllowedLateness(Duration.millis(1000)), Sum.ofIntegers(), VarIntCoder.of());
    tester.injectElements(// assigned to [-60, 40), [-30, 70), [0, 100)
    TimestampedValue.of(10, new Instant(23)), // assigned to [-30, 70), [0, 100), [30, 130)
    TimestampedValue.of(12, new Instant(40)));
    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(0, droppedElements);
    tester.advanceInputWatermark(new Instant(70));
    tester.injectElements(// but [-30, 70) is closed by the trigger
    TimestampedValue.of(14, new Instant(60)));
    droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(1, droppedElements);
    tester.advanceInputWatermark(new Instant(130));
    // assigned to [-30, 70), [0, 100), [30, 130)
    // but they are all closed
    tester.injectElements(TimestampedValue.of(16, new Instant(40)));
    droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(4, droppedElements);
}

Example 87 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method setGarbageCollectionHoldOnLateElements.

/**
 * Late elements should still have a garbage collection hold set so that they can make a late pane
 * rather than be dropped due to lateness.
 */
@Test
public void setGarbageCollectionHoldOnLateElements() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(FixedWindows.of(Duration.millis(10))).withTrigger(AfterWatermark.pastEndOfWindow().withLateFirings(AfterPane.elementCountAtLeast(2))).withMode(AccumulationMode.DISCARDING_FIRED_PANES).withAllowedLateness(Duration.millis(100)).withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY));
    tester.advanceInputWatermark(new Instant(0));
    tester.advanceOutputWatermark(new Instant(0));
    tester.injectElements(TimestampedValue.of(1, new Instant(1)));
    // Fire ON_TIME pane @ 9 with 1
    tester.advanceInputWatermark(new Instant(109));
    tester.advanceOutputWatermark(new Instant(109));
    tester.injectElements(TimestampedValue.of(2, new Instant(2)));
    // We should have set a garbage collection hold for the final pane.
    Instant hold = tester.getWatermarkHold();
    assertEquals(new Instant(109), hold);
    tester.advanceInputWatermark(new Instant(110));
    tester.advanceOutputWatermark(new Instant(110));
    // Fire final LATE pane @ 9 with 2
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertEquals(2, output.size());
}

Example 88 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testEmptyOnTimeWithOnTimeBehaviorFireIfNonEmpty.

/**
 * Test that it won't fire an empty on-time pane when OnTimeBehavior is FIRE_IF_NON_EMPTY.
 */
@Test
public void testEmptyOnTimeWithOnTimeBehaviorFireIfNonEmpty() throws Exception {
    WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10))).withTimestampCombiner(TimestampCombiner.EARLIEST).withTrigger(AfterEach.inOrder(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(5))).orFinally(AfterWatermark.pastEndOfWindow()), Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(25))))).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.millis(100)).withClosingBehavior(ClosingBehavior.FIRE_ALWAYS).withOnTimeBehavior(Window.OnTimeBehavior.FIRE_IF_NON_EMPTY);
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
    tester.advanceInputWatermark(new Instant(0));
    tester.advanceProcessingTime(new Instant(0));
    // Processing time timer for 5
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(1, new Instant(3)), TimestampedValue.of(1, new Instant(7)), TimestampedValue.of(1, new Instant(5)));
    // Should fire early pane
    tester.advanceProcessingTime(new Instant(6));
    // Should not fire empty on time pane
    tester.advanceInputWatermark(new Instant(11));
    // Should fire final GC pane
    tester.advanceInputWatermark(new Instant(10 + 100));
    List<WindowedValue<Integer>> output = tester.extractOutput();
    assertEquals(2, output.size());
    assertThat(output.get(0), isSingleWindowedValue(4, 1, 0, 10));
    assertThat(output.get(1), isSingleWindowedValue(4, 9, 0, 10));
    assertThat(output.get(0), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
    assertThat(output.get(1), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.LATE, 1, 0)));
}

Example 89 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testIdempotentEmptyPanesDiscarding.

@Test
public void testIdempotentEmptyPanesDiscarding() throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    // Test uninteresting (empty) panes don't increment the index or otherwise
    // modify PaneInfo.
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY);
    // Inject a couple of on-time elements and fire at the window end.
    injectElement(tester, 1);
    injectElement(tester, 2);
    tester.advanceInputWatermark(new Instant(12));
    // Fire the on-time pane
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.fireTimer(firstWindow, new Instant(9), TimeDomain.EVENT_TIME);
    // Fire another timer (with no data, so it's an uninteresting pane that should not be output).
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    tester.fireTimer(firstWindow, new Instant(9), TimeDomain.EVENT_TIME);
    // Finish it off with another datum.
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    injectElement(tester, 3);
    // The intermediate trigger firing shouldn't result in any output.
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output.size(), equalTo(2));
    // The on-time pane is as expected.
    assertThat(output.get(0), isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10));
    // The late pane has the correct indices.
    assertThat(output.get(1).getValue(), contains(3));
    assertThat(output.get(1).getPane(), equalTo(PaneInfo.createPane(false, true, Timing.LATE, 1, 1)));
    assertTrue(tester.isMarkedFinished(firstWindow));
    tester.assertHasOnlyGlobalAndFinishedSetsFor(firstWindow);
    long droppedElements = container.getCounter(MetricName.named(ReduceFnRunner.class, ReduceFnRunner.DROPPED_DUE_TO_CLOSED_WINDOW)).getCumulative();
    assertEquals(0, droppedElements);
}

Example 90 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testLateProcessingTimeTimer.

/**
 * Tests that when a processing time timer comes in after a window is expired it is just ignored.
 */
@Test
public void testLateProcessingTimeTimer() throws Exception {
    WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(100))).withTimestampCombiner(TimestampCombiner.EARLIEST).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.ZERO).withTrigger(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(10))));
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
    tester.advanceProcessingTime(new Instant(5000));
    // processing timer @ 5000 + 10; EOW timer @ 100
    injectElement(tester, 2);
    injectElement(tester, 5);
    // After this advancement, the window is expired and only the GC process
    // should be allowed to touch it
    tester.advanceInputWatermarkNoTimers(new Instant(100));
    // This should not output
    tester.advanceProcessingTime(new Instant(6000));
    assertThat(tester.extractOutput(), emptyIterable());
}