Examples with TimerSpec org.apache.beam.sdk.state.TimerSpec used on opensource projects

Example 11 with TimerSpec

use of org.apache.beam.sdk.state.TimerSpec in project beam by apache.

the class ParDoTest method testEventTimeTimerAlignBounded.

@Test
@Category({ ValidatesRunner.class, UsesTimersInParDo.class })
public void testEventTimeTimerAlignBounded() throws Exception {
    final String timerId = "foo";
    DoFn<KV<String, Integer>, KV<Integer, Instant>> fn = new DoFn<KV<String, Integer>, KV<Integer, Instant>>() {

        @TimerId(timerId)
        private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);

        @ProcessElement
        public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
            timer.align(Duration.standardSeconds(1)).offset(Duration.millis(1)).setRelative();
            context.output(KV.of(3, context.timestamp()));
        }

        @OnTimer(timerId)
        public void onTimer(OnTimerContext context) {
            context.output(KV.of(42, context.timestamp()));
        }
    };
    PCollection<KV<Integer, Instant>> output = pipeline.apply(Create.of(KV.of("hello", 37))).apply(ParDo.of(fn));
    PAssert.that(output).containsInAnyOrder(KV.of(3, BoundedWindow.TIMESTAMP_MIN_VALUE), KV.of(42, BoundedWindow.TIMESTAMP_MIN_VALUE.plus(1774)));
    pipeline.run();
}

Example 12 with TimerSpec

use of org.apache.beam.sdk.state.TimerSpec in project beam by apache.

the class ParDoTest method testEventTimeTimerAlignAfterGcTimeUnbounded.

@Test
@Category({ NeedsRunner.class, UsesTimersInParDo.class, UsesTestStream.class })
public void testEventTimeTimerAlignAfterGcTimeUnbounded() throws Exception {
    final String timerId = "foo";
    DoFn<KV<String, Integer>, KV<Integer, Instant>> fn = new DoFn<KV<String, Integer>, KV<Integer, Instant>>() {

        @TimerId(timerId)
        private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);

        @ProcessElement
        public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
            // This aligned time will exceed the END_OF_GLOBAL_WINDOW
            timer.align(Duration.standardDays(1)).setRelative();
            context.output(KV.of(3, context.timestamp()));
        }

        @OnTimer(timerId)
        public void onTimer(OnTimerContext context) {
            context.output(KV.of(42, context.timestamp()));
        }
    };
    TestStream<KV<String, Integer>> stream = TestStream.create(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())).advanceWatermarkTo(BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(1))).addElements(KV.of("hello", 37)).advanceWatermarkToInfinity();
    PCollection<KV<Integer, Instant>> output = pipeline.apply(stream).apply(ParDo.of(fn));
    PAssert.that(output).containsInAnyOrder(KV.of(3, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(1))), KV.of(42, BoundedWindow.TIMESTAMP_MAX_VALUE.minus(Duration.standardDays(1))));
    pipeline.run();
}

Example 13 with TimerSpec

use of org.apache.beam.sdk.state.TimerSpec in project beam by apache.

the class ParDoTest method testEventTimeTimerAlignUnbounded.

@Test
@Category({ NeedsRunner.class, UsesTimersInParDo.class, UsesTestStream.class })
public void testEventTimeTimerAlignUnbounded() throws Exception {
    final String timerId = "foo";
    DoFn<KV<String, Integer>, KV<Integer, Instant>> fn = new DoFn<KV<String, Integer>, KV<Integer, Instant>>() {

        @TimerId(timerId)
        private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);

        @ProcessElement
        public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
            timer.align(Duration.standardSeconds(1)).offset(Duration.millis(1)).setRelative();
            context.output(KV.of(3, context.timestamp()));
        }

        @OnTimer(timerId)
        public void onTimer(OnTimerContext context) {
            context.output(KV.of(42, context.timestamp()));
        }
    };
    TestStream<KV<String, Integer>> stream = TestStream.create(KvCoder.of(StringUtf8Coder.of(), VarIntCoder.of())).advanceWatermarkTo(new Instant(5)).addElements(KV.of("hello", 37)).advanceWatermarkTo(new Instant(0).plus(Duration.standardSeconds(1).plus(1))).advanceWatermarkToInfinity();
    PCollection<KV<Integer, Instant>> output = pipeline.apply(stream).apply(ParDo.of(fn));
    PAssert.that(output).containsInAnyOrder(KV.of(3, new Instant(5)), KV.of(42, new Instant(Duration.standardSeconds(1).minus(1).getMillis())));
    pipeline.run();
}

Example 14 with TimerSpec

use of org.apache.beam.sdk.state.TimerSpec in project beam by apache.

the class ParDoTest method testTimerReceivedInOriginalWindow.

@Test
@Category({ ValidatesRunner.class, UsesTimersInParDo.class })
public void testTimerReceivedInOriginalWindow() throws Exception {
    final String timerId = "foo";
    DoFn<KV<String, Integer>, BoundedWindow> fn = new DoFn<KV<String, Integer>, BoundedWindow>() {

        @TimerId(timerId)
        private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);

        @ProcessElement
        public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
            timer.offset(Duration.standardSeconds(1)).setRelative();
        }

        @OnTimer(timerId)
        public void onTimer(OnTimerContext context, BoundedWindow window) {
            context.output(context.window());
        }

        public TypeDescriptor<BoundedWindow> getOutputTypeDescriptor() {
            return (TypeDescriptor) TypeDescriptor.of(IntervalWindow.class);
        }
    };
    SlidingWindows windowing = SlidingWindows.of(Duration.standardMinutes(3)).every(Duration.standardMinutes(1));
    PCollection<BoundedWindow> output = pipeline.apply(Create.timestamped(TimestampedValue.of(KV.of("hello", 24), new Instant(0L)))).apply(Window.<KV<String, Integer>>into(windowing)).apply(ParDo.of(fn));
    PAssert.that(output).containsInAnyOrder(new IntervalWindow(new Instant(0), Duration.standardMinutes(3)), new IntervalWindow(new Instant(0).minus(Duration.standardMinutes(1)), Duration.standardMinutes(3)), new IntervalWindow(new Instant(0).minus(Duration.standardMinutes(2)), Duration.standardMinutes(3)));
    pipeline.run();
}

Example 15 with TimerSpec

use of org.apache.beam.sdk.state.TimerSpec in project beam by apache.

the class ParDoTest method testMultipleWindowSubtypesOK.

/**
   * Tests that it is OK to use different window types in the parameter lists to different
   * {@link DoFn} functions, as long as they are all subtypes of the actual window type
   * of the input.
   *
   * <p>Today, the only method other than {@link ProcessElement @ProcessElement} that can accept
   * extended parameters is {@link OnTimer @OnTimer}, which is rejected before it reaches window
   * type validation. Rather than delay validation, this test is temporarily disabled.
   */
@Ignore("ParDo rejects this on account of it using timers")
@Test
public void testMultipleWindowSubtypesOK() {
    final String timerId = "gobbledegook";
    pipeline.apply(Create.of(1, 2, 3)).apply(Window.<Integer>into(FixedWindows.of(Duration.standardSeconds(10)))).apply(ParDo.of(new DoFn<Integer, Integer>() {

        @TimerId(timerId)
        private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);

        @ProcessElement
        public void process(ProcessContext c, IntervalWindow w) {
        }

        @OnTimer(timerId)
        public void onTimer(BoundedWindow w) {
        }
    }));
// If it doesn't crash, we made it!
}