Examples with IntervalWindow org.apache.beam.sdk.transforms.windowing.IntervalWindow used on opensource projects

Example 71 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class LeaderBoardTest method testTeamScoresDroppablyLate.

/**
 * A test where elements arrive beyond the maximum allowed lateness. These elements are dropped
 * within {@link CalculateTeamScores} and do not impact the final result.
 */
@Test
public void testTeamScoresDroppablyLate() {
    BoundedWindow window = new IntervalWindow(baseTime, TEAM_WINDOW_DURATION);
    TestStream<GameActionInfo> infos = TestStream.create(AvroCoder.of(GameActionInfo.class)).addElements(event(TestUser.BLUE_ONE, 12, Duration.ZERO), event(TestUser.RED_ONE, 3, Duration.ZERO)).advanceWatermarkTo(window.maxTimestamp()).addElements(event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_TWO, 3, Duration.ZERO), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(3))).advanceWatermarkTo(baseTime.plus(TEAM_WINDOW_DURATION)).advanceWatermarkTo(baseTime.plus(ALLOWED_LATENESS).plus(TEAM_WINDOW_DURATION).plus(Duration.standardMinutes(1))).addElements(event(TestUser.BLUE_TWO, 3, TEAM_WINDOW_DURATION.minus(Duration.standardSeconds(5))), event(TestUser.RED_ONE, 7, Duration.standardMinutes(4))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(infos).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    // Only one on-time pane and no late panes should be emitted
    PAssert.that(teamScores).inWindow(window).containsInAnyOrder(KV.of(redTeam, 7), KV.of(blueTeam, 18));
    // No elements are added before the watermark passes the end of the window plus the allowed
    // lateness, so no refinement should be emitted
    PAssert.that(teamScores).inFinalPane(window).empty();
    p.run().waitUntilFinish();
}

Example 72 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class LeaderBoardTest method testTeamScoresObservablyLate.

/**
 * A test where elements arrive behind the watermark (late data) after the watermark passes the
 * end of the window, but before the maximum allowed lateness. These elements are emitted in a
 * late pane.
 */
@Test
public void testTeamScoresObservablyLate() {
    Instant firstWindowCloses = baseTime.plus(ALLOWED_LATENESS).plus(TEAM_WINDOW_DURATION);
    TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)).advanceWatermarkTo(baseTime).addElements(event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(8))).advanceProcessingTime(Duration.standardMinutes(10)).advanceWatermarkTo(baseTime.plus(Duration.standardMinutes(3))).addElements(event(TestUser.RED_ONE, 3, Duration.standardMinutes(1)), event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(5))).advanceWatermarkTo(firstWindowCloses.minus(Duration.standardMinutes(1))).addElements(event(TestUser.RED_TWO, 2, Duration.ZERO), event(TestUser.RED_TWO, 5, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 3, Duration.standardMinutes(3))).advanceProcessingTime(Duration.standardMinutes(12)).addElements(event(TestUser.RED_TWO, 9, Duration.standardMinutes(1)), event(TestUser.RED_TWO, 1, Duration.standardMinutes(3))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    BoundedWindow window = new IntervalWindow(baseTime, TEAM_WINDOW_DURATION);
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    PAssert.that(teamScores).inWindow(window).satisfies(input -> {
        // The final sums need not exist in the same pane, but must appear in the output
        // PCollection
        assertThat(input, hasItem(KV.of(blueTeam, 11)));
        assertThat(input, hasItem(KV.of(redTeam, 27)));
        return null;
    });
    PAssert.thatMap(teamScores).inOnTimePane(window).isEqualTo(ImmutableMap.<String, Integer>builder().put(redTeam, 7).put(blueTeam, 11).build());
    // No final pane is emitted for the blue team, as all of their updates have been taken into
    // account in earlier panes
    PAssert.that(teamScores).inFinalPane(window).containsInAnyOrder(KV.of(redTeam, 27));
    p.run().waitUntilFinish();
}

Example 73 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class SortingFlinkCombineRunner method mergeWindow.

/**
 * Merge windows. This assumes that the list of elements is sorted by window-end timestamp. This
 * replaces windows in the input list.
 */
private void mergeWindow(List<WindowedValue<KV<K, InputT>>> elements) {
    int currentStart = 0;
    IntervalWindow currentWindow = (IntervalWindow) Iterables.getOnlyElement(elements.get(0).getWindows());
    for (int i = 1; i < elements.size(); i++) {
        WindowedValue<KV<K, InputT>> nextValue = elements.get(i);
        IntervalWindow nextWindow = (IntervalWindow) Iterables.getOnlyElement(nextValue.getWindows());
        if (currentWindow.intersects(nextWindow)) {
            // we continue
            currentWindow = currentWindow.span(nextWindow);
        } else {
            // retrofit the merged window to all windows up to "currentStart"
            for (int j = i - 1; j >= currentStart; j--) {
                WindowedValue<KV<K, InputT>> value = elements.get(j);
                elements.set(j, WindowedValue.of(value.getValue(), value.getTimestamp(), currentWindow, value.getPane()));
            }
            currentStart = i;
            currentWindow = nextWindow;
        }
    }
    if (currentStart < elements.size() - 1) {
        // we have to retrofit the last batch
        for (int j = elements.size() - 1; j >= currentStart; j--) {
            WindowedValue<KV<K, InputT>> value = elements.get(j);
            elements.set(j, WindowedValue.of(value.getValue(), value.getTimestamp(), currentWindow, value.getPane()));
        }
    }
}

Example 74 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testProcessingTime.

/**
 * Tests for processing time firings after the watermark passes the end of the window.
 * Specifically, verify the proper triggerings and pane-info of a typical speculative/on-time/late
 * when the on-time pane is non-empty.
 */
@Test
public void testProcessingTime() throws Exception {
    WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) FixedWindows.of(Duration.millis(10))).withTimestampCombiner(TimestampCombiner.EARLIEST).withTrigger(AfterEach.inOrder(Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(5))).orFinally(AfterWatermark.pastEndOfWindow()), Repeatedly.forever(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.millis(25))))).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.millis(100));
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
    tester.advanceInputWatermark(new Instant(0));
    tester.advanceProcessingTime(new Instant(0));
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(1, new Instant(3)), TimestampedValue.of(1, new Instant(7)), TimestampedValue.of(1, new Instant(5)));
    // 4 elements all at processing time 0
    // fire [1,3,7,5] since 6 > 0 + 5
    tester.advanceProcessingTime(new Instant(6));
    tester.injectElements(TimestampedValue.of(1, new Instant(8)), TimestampedValue.of(1, new Instant(4)));
    // 6 elements
    // fire [1,3,7,5,8,4] since 11 > 9
    tester.advanceInputWatermark(new Instant(11));
    tester.injectElements(TimestampedValue.of(1, new Instant(8)), TimestampedValue.of(1, new Instant(4)), TimestampedValue.of(1, new Instant(5)));
    // 9 elements
    tester.advanceInputWatermark(new Instant(12));
    tester.injectElements(TimestampedValue.of(1, new Instant(3)));
    // 10 elements
    tester.advanceProcessingTime(new Instant(15));
    tester.injectElements(TimestampedValue.of(1, new Instant(5)));
    // 11 elements
    // fire since 32 > 6 + 25
    tester.advanceProcessingTime(new Instant(32));
    tester.injectElements(TimestampedValue.of(1, new Instant(3)));
    // 12 elements
    // fire [1,3,7,5,8,4,8,4,5,3,5,3] since 125 > 6 + 25
    tester.advanceInputWatermark(new Instant(125));
    List<WindowedValue<Integer>> output = tester.extractOutput();
    assertEquals(4, output.size());
    assertThat(output.get(0), isSingleWindowedValue(4, 1, 0, 10));
    assertThat(output.get(1), isSingleWindowedValue(6, 4, 0, 10));
    assertThat(output.get(2), isSingleWindowedValue(11, 9, 0, 10));
    assertThat(output.get(3), isSingleWindowedValue(12, 9, 0, 10));
    assertThat(output.get(0), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1)));
    assertThat(output.get(1), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.ON_TIME, 1, 0)));
    assertThat(output.get(2), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.LATE, 2, 1)));
    assertThat(output.get(3), WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.LATE, 3, 2)));
}

Example 75 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class ReduceFnRunnerTest method testFixedWindowEndOfTimeGarbageCollection.

/**
 * Tests that the garbage collection time for a fixed window does not overflow the end of time.
 */
@Test
public void testFixedWindowEndOfTimeGarbageCollection() throws Exception {
    Duration allowedLateness = Duration.standardDays(365);
    Duration windowSize = Duration.millis(10);
    WindowFn<Object, IntervalWindow> windowFn = FixedWindows.of(windowSize);
    // This timestamp falls into a window where the end of the window is before the end of the
    // global window - the "end of time" - yet its expiration time is after.
    final Instant elementTimestamp = GlobalWindow.INSTANCE.maxTimestamp().minus(allowedLateness).plus(Duration.millis(1));
    IntervalWindow window = Iterables.getOnlyElement(windowFn.assignWindows(windowFn.new AssignContext() {

        @Override
        public Object element() {
            throw new UnsupportedOperationException();
        }

        @Override
        public Instant timestamp() {
            return elementTimestamp;
        }

        @Override
        public BoundedWindow window() {
            throw new UnsupportedOperationException();
        }
    }));
    assertTrue(window.maxTimestamp().isBefore(GlobalWindow.INSTANCE.maxTimestamp()));
    assertTrue(window.maxTimestamp().plus(allowedLateness).isAfter(GlobalWindow.INSTANCE.maxTimestamp()));
    // Test basic execution of a trigger using a non-combining window set and accumulating mode.
    WindowingStrategy<?, IntervalWindow> strategy = WindowingStrategy.of((WindowFn<?, IntervalWindow>) windowFn).withTimestampCombiner(TimestampCombiner.EARLIEST).withTrigger(AfterWatermark.pastEndOfWindow().withLateFirings(Never.ever())).withMode(AccumulationMode.DISCARDING_FIRED_PANES).withAllowedLateness(allowedLateness);
    ReduceFnTester<Integer, Integer, IntervalWindow> tester = ReduceFnTester.combining(strategy, Sum.ofIntegers(), VarIntCoder.of());
    tester.injectElements(TimestampedValue.of(13, elementTimestamp));
    // Should fire ON_TIME pane and there will be a checkState that the cleanup time
    // is prior to timestamp max value
    tester.advanceInputWatermark(window.maxTimestamp());
    // Nothing in the ON_TIME pane (not governed by triggers, but by ReduceFnRunner)
    assertThat(tester.extractOutput(), emptyIterable());
    tester.injectElements(TimestampedValue.of(42, elementTimestamp));
    // Now the final pane should fire, demonstrating that the GC time was truncated
    tester.advanceInputWatermark(GlobalWindow.INSTANCE.maxTimestamp());
    assertThat(tester.extractOutput(), contains(isWindowedValue(equalTo(55))));
}