Examples with SlidingWindows org.apache.beam.sdk.transforms.windowing.SlidingWindows used on opensource projects

Example 1 with SlidingWindows

use of org.apache.beam.sdk.transforms.windowing.SlidingWindows in project beam by apache.

the class BeamAggregationRel method explainTerms.

@Override
public RelWriter explainTerms(RelWriter pw) {
    super.explainTerms(pw);
    if (this.windowFn != null) {
        WindowFn windowFn = this.windowFn;
        String window = windowFn.getClass().getSimpleName() + "($" + String.valueOf(windowFieldIndex);
        if (windowFn instanceof FixedWindows) {
            FixedWindows fn = (FixedWindows) windowFn;
            window = window + ", " + fn.getSize().toString() + ", " + fn.getOffset().toString();
        } else if (windowFn instanceof SlidingWindows) {
            SlidingWindows fn = (SlidingWindows) windowFn;
            window = window + ", " + fn.getPeriod().toString() + ", " + fn.getSize().toString() + ", " + fn.getOffset().toString();
        } else if (windowFn instanceof Sessions) {
            Sessions fn = (Sessions) windowFn;
            window = window + ", " + fn.getGapDuration().toString();
        } else {
            throw new UnsupportedOperationException("Unknown window function " + windowFn.getClass().getSimpleName());
        }
        window = window + ")";
        pw.item("window", window);
    }
    return pw;
}

Example 2 with SlidingWindows

use of org.apache.beam.sdk.transforms.windowing.SlidingWindows in project beam by apache.

the class SplittableDoFnTest method testPairWithIndexWindowedTimestamped.

private void testPairWithIndexWindowedTimestamped(IsBounded bounded) {
    // Tests that Splittable DoFn correctly propagates windowing strategy, windows and timestamps
    // of elements in the input collection.
    MutableDateTime mutableNow = Instant.now().toMutableDateTime();
    mutableNow.setMillisOfSecond(0);
    Instant now = mutableNow.toInstant();
    Instant nowP1 = now.plus(Duration.standardSeconds(1));
    Instant nowP2 = now.plus(Duration.standardSeconds(2));
    SlidingWindows windowFn = SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
    PCollection<KV<String, Integer>> res = p.apply(Create.timestamped(TimestampedValue.of("a", now), TimestampedValue.of("bb", nowP1), TimestampedValue.of("ccccc", nowP2))).apply(Window.into(windowFn)).apply(ParDo.of(pairStringWithIndexToLengthFn(bounded))).setCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()));
    assertEquals(windowFn, res.getWindowingStrategy().getWindowFn());
    PCollection<TimestampedValue<KV<String, Integer>>> timestamped = res.apply(Reify.timestamps());
    for (int i = 0; i < 4; ++i) {
        Instant base = now.minus(Duration.standardSeconds(i));
        IntervalWindow window = new IntervalWindow(base, base.plus(Duration.standardSeconds(5)));
        List<TimestampedValue<KV<String, Integer>>> expectedUnfiltered = Arrays.asList(TimestampedValue.of(KV.of("a", 0), now), TimestampedValue.of(KV.of("bb", 0), nowP1), TimestampedValue.of(KV.of("bb", 1), nowP1), TimestampedValue.of(KV.of("ccccc", 0), nowP2), TimestampedValue.of(KV.of("ccccc", 1), nowP2), TimestampedValue.of(KV.of("ccccc", 2), nowP2), TimestampedValue.of(KV.of("ccccc", 3), nowP2), TimestampedValue.of(KV.of("ccccc", 4), nowP2));
        List<TimestampedValue<KV<String, Integer>>> expected = new ArrayList<>();
        for (TimestampedValue<KV<String, Integer>> tv : expectedUnfiltered) {
            if (!window.start().isAfter(tv.getTimestamp()) && !tv.getTimestamp().isAfter(window.maxTimestamp())) {
                expected.add(tv);
            }
        }
        assertFalse(expected.isEmpty());
        PAssert.that(timestamped).inWindow(window).containsInAnyOrder(expected);
    }
    p.run();
}

Example 3 with SlidingWindows

use of org.apache.beam.sdk.transforms.windowing.SlidingWindows in project beam by apache.

the class MultiStepCombineTest method testMultiStepCombineWindowed.

@Test
public void testMultiStepCombineWindowed() {
    SlidingWindows windowFn = SlidingWindows.of(Duration.millis(6L)).every(Duration.millis(3L));
    PCollection<KV<String, Long>> combined = pipeline.apply(Create.timestamped(TimestampedValue.of(KV.of("foo", 1L), new Instant(1L)), TimestampedValue.of(KV.of("bar", 2L), new Instant(2L)), TimestampedValue.of(KV.of("bizzle", 3L), new Instant(3L)), TimestampedValue.of(KV.of("bar", 4L), new Instant(4L)), TimestampedValue.of(KV.of("bizzle", 11L), new Instant(11L)))).apply(Window.into(windowFn)).apply(Combine.perKey(new MultiStepCombineFn()));
    PAssert.that("Windows should combine only elements in their windows", combined).inWindow(new IntervalWindow(new Instant(0L), Duration.millis(6L))).containsInAnyOrder(KV.of("foo", 1L), KV.of("bar", 6L), KV.of("bizzle", 3L));
    PAssert.that("Elements should appear in all the windows they are assigned to", combined).inWindow(new IntervalWindow(new Instant(-3L), Duration.millis(6L))).containsInAnyOrder(KV.of("foo", 1L), KV.of("bar", 2L));
    PAssert.that(combined).inWindow(new IntervalWindow(new Instant(6L), Duration.millis(6L))).containsInAnyOrder(KV.of("bizzle", 11L));
    PAssert.that(combined).containsInAnyOrder(KV.of("foo", 1L), KV.of("foo", 1L), KV.of("bar", 6L), KV.of("bar", 2L), KV.of("bar", 4L), KV.of("bizzle", 11L), KV.of("bizzle", 11L), KV.of("bizzle", 3L), KV.of("bizzle", 3L));
    pipeline.run();
}

Example 4 with SlidingWindows

use of org.apache.beam.sdk.transforms.windowing.SlidingWindows in project beam by apache.

the class ParDoTest method testSideInputsWithMultipleWindows.

@Test
@Category(ValidatesRunner.class)
public void testSideInputsWithMultipleWindows() {
    // Tests that the runner can safely run a DoFn that uses side inputs
    // on an input where the element is in multiple windows. The complication is
    // that side inputs are per-window, so the runner has to make sure
    // to process each window individually.
    MutableDateTime mutableNow = Instant.now().toMutableDateTime();
    mutableNow.setMillisOfSecond(0);
    Instant now = mutableNow.toInstant();
    SlidingWindows windowFn = SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
    PCollectionView<Integer> view = pipeline.apply(Create.of(1)).apply(View.<Integer>asSingleton());
    PCollection<String> res = pipeline.apply(Create.timestamped(TimestampedValue.of("a", now))).apply(Window.<String>into(windowFn)).apply(ParDo.of(new FnWithSideInputs(view)).withSideInputs(view));
    for (int i = 0; i < 4; ++i) {
        Instant base = now.minus(Duration.standardSeconds(i));
        IntervalWindow window = new IntervalWindow(base, base.plus(Duration.standardSeconds(5)));
        PAssert.that(res).inWindow(window).containsInAnyOrder("a:1");
    }
    pipeline.run();
}

Example 5 with SlidingWindows

use of org.apache.beam.sdk.transforms.windowing.SlidingWindows in project beam by apache.

the class ParDoTest method testTimerReceivedInOriginalWindow.

@Test
@Category({ ValidatesRunner.class, UsesTimersInParDo.class })
public void testTimerReceivedInOriginalWindow() throws Exception {
    final String timerId = "foo";
    DoFn<KV<String, Integer>, BoundedWindow> fn = new DoFn<KV<String, Integer>, BoundedWindow>() {

        @TimerId(timerId)
        private final TimerSpec spec = TimerSpecs.timer(TimeDomain.EVENT_TIME);

        @ProcessElement
        public void processElement(ProcessContext context, @TimerId(timerId) Timer timer) {
            timer.offset(Duration.standardSeconds(1)).setRelative();
        }

        @OnTimer(timerId)
        public void onTimer(OnTimerContext context, BoundedWindow window) {
            context.output(context.window());
        }

        public TypeDescriptor<BoundedWindow> getOutputTypeDescriptor() {
            return (TypeDescriptor) TypeDescriptor.of(IntervalWindow.class);
        }
    };
    SlidingWindows windowing = SlidingWindows.of(Duration.standardMinutes(3)).every(Duration.standardMinutes(1));
    PCollection<BoundedWindow> output = pipeline.apply(Create.timestamped(TimestampedValue.of(KV.of("hello", 24), new Instant(0L)))).apply(Window.<KV<String, Integer>>into(windowing)).apply(ParDo.of(fn));
    PAssert.that(output).containsInAnyOrder(new IntervalWindow(new Instant(0), Duration.standardMinutes(3)), new IntervalWindow(new Instant(0).minus(Duration.standardMinutes(1)), Duration.standardMinutes(3)), new IntervalWindow(new Instant(0).minus(Duration.standardMinutes(2)), Duration.standardMinutes(3)));
    pipeline.run();
}