Examples with BoundedWindow org.apache.beam.sdk.transforms.windowing.BoundedWindow used on opensource projects

Example 46 with BoundedWindow

use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.

the class WindowedValueTest method testExplodeWindowsInOneWindowEquals.

@Test
public void testExplodeWindowsInOneWindowEquals() {
    Instant now = Instant.now();
    BoundedWindow window = new IntervalWindow(now.minus(1000L), now.plus(1000L));
    WindowedValue<String> value = WindowedValue.of("foo", now, window, PaneInfo.ON_TIME_AND_ONLY_FIRING);
    assertThat(Iterables.getOnlyElement(value.explodeWindows()), equalTo(value));
}

Example 47 with BoundedWindow

use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.

the class LeaderBoardTest method testTeamScoresUnobservablyLate.

/**
   * A test where elements arrive behind the watermark (late data), but before the end of the
   * window. These elements are emitted on time.
   */
@Test
public void testTeamScoresUnobservablyLate() {
    BoundedWindow window = new IntervalWindow(baseTime, TEAM_WINDOW_DURATION);
    TestStream<GameActionInfo> createEvents = TestStream.create(AvroCoder.of(GameActionInfo.class)).advanceWatermarkTo(baseTime).addElements(event(TestUser.BLUE_ONE, 3, Duration.standardSeconds(3)), event(TestUser.BLUE_TWO, 5, Duration.standardMinutes(8)), event(TestUser.RED_ONE, 4, Duration.standardMinutes(2)), event(TestUser.BLUE_ONE, 3, Duration.standardMinutes(5))).advanceWatermarkTo(baseTime.plus(TEAM_WINDOW_DURATION).minus(Duration.standardMinutes(1))).addElements(event(TestUser.RED_TWO, 2, Duration.ZERO), event(TestUser.RED_TWO, 5, Duration.standardMinutes(1)), event(TestUser.BLUE_TWO, 2, Duration.standardSeconds(90)), event(TestUser.RED_TWO, 3, Duration.standardMinutes(3))).advanceWatermarkTo(baseTime.plus(TEAM_WINDOW_DURATION).plus(Duration.standardMinutes(1))).advanceWatermarkToInfinity();
    PCollection<KV<String, Integer>> teamScores = p.apply(createEvents).apply(new CalculateTeamScores(TEAM_WINDOW_DURATION, ALLOWED_LATENESS));
    String blueTeam = TestUser.BLUE_ONE.getTeam();
    String redTeam = TestUser.RED_ONE.getTeam();
    // The On Time pane contains the late elements that arrived before the end of the window
    PAssert.that(teamScores).inOnTimePane(window).containsInAnyOrder(KV.of(redTeam, 14), KV.of(blueTeam, 13));
    p.run().waitUntilFinish();
}

Example 48 with BoundedWindow

use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.

the class ParDoTest method testTimerReceivedInOriginalWindow.

@Test
@Category({ ValidatesRunner.class, UsesTimersInParDo.class })
public void testTimerReceivedInOriginalWindow() throws Exception {
    final String timerId = "foo";
    DoFn<KV<String, Integer>, BoundedWindow> fn = new DoFn<KV<String, Integer>, BoundedWindow>() {

        @TimerId(timerId)
        private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);

        @ProcessElement
        public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
            timer.offset(Duration.standardSeconds(1)).setRelative();
        }

        @OnTimer(timerId)
        public void onTimer(OnTimerContext context, BoundedWindow window) {
            context.output(context.window());
        }

        public TypeDescriptor<BoundedWindow> getOutputTypeDescriptor() {
            return (TypeDescriptor) TypeDescriptor.of(IntervalWindow.class);
        }
    };
    SlidingWindows windowing = SlidingWindows.of(Duration.standardMinutes(3)).every(Duration.standardMinutes(1));
    PCollection<BoundedWindow> output = pipeline.apply(Create.timestamped(TimestampedValue.of(KV.of("hello", 24), new Instant(0L)))).apply(Window.<KV<String, Integer>>into(windowing)).apply(ParDo.of(fn));
    PAssert.that(output).containsInAnyOrder(new IntervalWindow(new Instant(0), Duration.standardMinutes(3)), new IntervalWindow(new Instant(0).minus(Duration.standardMinutes(1)), Duration.standardMinutes(3)), new IntervalWindow(new Instant(0).minus(Duration.standardMinutes(2)), Duration.standardMinutes(3)));
    pipeline.run();
}

Example 49 with BoundedWindow

use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.

the class ParDoTest method testMultipleWindowSubtypesOK.

/**
   * Tests that it is OK to use different window types in the parameter lists to different
   * {@link DoFn} functions, as long as they are all subtypes of the actual window type
   * of the input.
   *
   * <p>Today, the only method other than {@link ProcessElement @ProcessElement} that can accept
   * extended parameters is {@link OnTimer @OnTimer}, which is rejected before it reaches window
   * type validation. Rather than delay validation, this test is temporarily disabled.
   */
@Ignore("ParDo rejects this on account of it using timers")
@Test
public void testMultipleWindowSubtypesOK() {
    final String timerId = "gobbledegook";
    pipeline.apply(Create.of(1, 2, 3)).apply(Window.<Integer>into(FixedWindows.of(Duration.standardSeconds(10)))).apply(ParDo.of(new DoFn<Integer, Integer>() {

        @TimerId(timerId)
        private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);

        @ProcessElement
        public void process(ProcessContext c, IntervalWindow w) {
        }

        @OnTimer(timerId)
        public void onTimer(BoundedWindow w) {
        }
    }));
// If it doesn't crash, we made it!
}

Example 50 with BoundedWindow

use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.

the class ParDoTest method testEventTimeTimerAbsolute.

/**
   * Tests that an event time timer set absolutely for the last possible moment fires and results in
   * supplementary output. The test is otherwise identical to {@link #testEventTimeTimerBounded()}.
   */
@Test
@Category({ ValidatesRunner.class, UsesTimersInParDo.class })
public void testEventTimeTimerAbsolute() throws Exception {
    final String timerId = "foo";
    DoFn<KV<String, Integer>, Integer> fn = new DoFn<KV<String, Integer>, Integer>() {

        @TimerId(timerId)
        private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);

        @ProcessElement
        public void processElement(ProcessContext context, @TimerId(timerId) Timer timer, BoundedWindow window) {
            timer.set(window.maxTimestamp());
            context.output(3);
        }

        @OnTimer(timerId)
        public void onTimer(OnTimerContext context) {
            context.output(42);
        }
    };
    PCollection<Integer> output = pipeline.apply(Create.of(KV.of("hello", 37))).apply(ParDo.of(fn));
    PAssert.that(output).containsInAnyOrder(3, 42);
    pipeline.run();
}