Examples with FixedWindows org.apache.beam.sdk.transforms.windowing.FixedWindows used on opensource projects

Example 6 with FixedWindows

use of org.apache.beam.sdk.transforms.windowing.FixedWindows in project beam by apache.

the class CreateStreamTest method testDiscardingMode.

@Test
public void testDiscardingMode() throws IOException {
    CreateStream<String> source = CreateStream.of(StringUtf8Coder.of(), batchDuration()).nextBatch(TimestampedValue.of("firstPane", new Instant(100)), TimestampedValue.of("alsoFirstPane", new Instant(200))).advanceWatermarkForNextBatch(new Instant(1001L)).nextBatch(TimestampedValue.of("onTimePane", new Instant(500))).advanceNextBatchWatermarkToInfinity().nextBatch(TimestampedValue.of("finalLatePane", new Instant(750)), TimestampedValue.of("alsoFinalLatePane", new Instant(250)));
    FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
    Duration allowedLateness = Duration.millis(5000L);
    PCollection<String> values = p.apply(source).apply(Window.<String>into(windowFn).triggering(AfterWatermark.pastEndOfWindow().withEarlyFirings(AfterPane.elementCountAtLeast(2)).withLateFirings(Never.ever())).discardingFiredPanes().withAllowedLateness(allowedLateness)).apply(WithKeys.<Integer, String>of(1)).apply(GroupByKey.<Integer, String>create()).apply(Values.<Iterable<String>>create()).apply(Flatten.<String>iterables());
    IntervalWindow window = windowFn.assignWindow(new Instant(100));
    PAssert.that(values).inWindow(window).containsInAnyOrder("firstPane", "alsoFirstPane", "onTimePane", "finalLatePane", "alsoFinalLatePane");
    PAssert.that(values).inCombinedNonLatePanes(window).containsInAnyOrder("firstPane", "alsoFirstPane", "onTimePane");
    PAssert.that(values).inOnTimePane(window).containsInAnyOrder("onTimePane");
    PAssert.that(values).inFinalPane(window).containsInAnyOrder("finalLatePane", "alsoFinalLatePane");
    p.run();
}

Example 7 with FixedWindows

use of org.apache.beam.sdk.transforms.windowing.FixedWindows in project beam by apache.

the class ParDoTest method testWindowingInStartAndFinishBundle.

@Test
@Category(ValidatesRunner.class)
public void testWindowingInStartAndFinishBundle() {
    final FixedWindows windowFn = FixedWindows.of(Duration.millis(1));
    PCollection<String> output = pipeline.apply(Create.timestamped(TimestampedValue.of("elem", new Instant(1)))).apply(Window.<String>into(windowFn)).apply(ParDo.of(new DoFn<String, String>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            c.output(c.element());
            System.out.println("Process: " + c.element() + ":" + c.timestamp().getMillis());
        }

        @FinishBundle
        public void finishBundle(FinishBundleContext c) {
            Instant ts = new Instant(3);
            c.output("finish", ts, windowFn.assignWindow(ts));
            System.out.println("Finish: 3");
        }
    })).apply(ParDo.of(new PrintingDoFn()));
    PAssert.that(output).satisfies(new Checker());
    pipeline.run();
}

Example 8 with FixedWindows

use of org.apache.beam.sdk.transforms.windowing.FixedWindows in project beam by apache.

the class LateDataUtilsTest method garbageCollectionTimeAfterEndOfGlobalWindowWithLateness.

@Test
public void garbageCollectionTimeAfterEndOfGlobalWindowWithLateness() {
    FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(5));
    Duration allowedLateness = Duration.millis(Long.MAX_VALUE);
    WindowingStrategy<?, ?> strategy = WindowingStrategy.globalDefault().withWindowFn(windowFn).withAllowedLateness(allowedLateness);
    IntervalWindow window = windowFn.assignWindow(new Instant(-100));
    assertThat(window.maxTimestamp().plus(allowedLateness), Matchers.<ReadableInstant>greaterThan(GlobalWindow.INSTANCE.maxTimestamp()));
    assertThat(LateDataUtils.garbageCollectionTime(window, strategy), equalTo(GlobalWindow.INSTANCE.maxTimestamp()));
}

Example 9 with FixedWindows

use of org.apache.beam.sdk.transforms.windowing.FixedWindows in project beam by apache.

the class CombineTest method testWindowedCombineGloballyAsSingletonView.

@Test
@Category(ValidatesRunner.class)
public void testWindowedCombineGloballyAsSingletonView() {
    FixedWindows windowFn = FixedWindows.of(Duration.standardMinutes(1));
    final PCollectionView<Integer> view = pipeline.apply("CreateSideInput", Create.timestamped(TimestampedValue.of(1, new Instant(100)), TimestampedValue.of(3, new Instant(100)))).apply("WindowSideInput", Window.<Integer>into(windowFn)).apply("CombineSideInput", Sum.integersGlobally().asSingletonView());
    TimestampedValue<Void> nonEmptyElement = TimestampedValue.of(null, new Instant(100));
    TimestampedValue<Void> emptyElement = TimestampedValue.atMinimumTimestamp(null);
    PCollection<Integer> output = pipeline.apply("CreateMainInput", Create.<Void>timestamped(nonEmptyElement, emptyElement).withCoder(VoidCoder.of())).apply("WindowMainInput", Window.<Void>into(windowFn)).apply("OutputSideInput", ParDo.of(new DoFn<Void, Integer>() {

        @ProcessElement
        public void processElement(ProcessContext c) {
            c.output(c.sideInput(view));
        }
    }).withSideInputs(view));
    PAssert.that(output).containsInAnyOrder(4, 0);
    PAssert.that(output).inWindow(windowFn.assignWindow(nonEmptyElement.getTimestamp())).containsInAnyOrder(4);
    PAssert.that(output).inWindow(windowFn.assignWindow(emptyElement.getTimestamp())).containsInAnyOrder(0);
    pipeline.run();
}

Example 10 with FixedWindows

use of org.apache.beam.sdk.transforms.windowing.FixedWindows in project beam by apache.

the class TestStreamTest method testDiscardingMode.

@Test
@Category({ NeedsRunner.class, UsesTestStream.class })
public void testDiscardingMode() {
    TestStream<String> stream = TestStream.create(StringUtf8Coder.of()).advanceWatermarkTo(new Instant(0)).addElements(TimestampedValue.of("firstPane", new Instant(100)), TimestampedValue.of("alsoFirstPane", new Instant(200))).addElements(TimestampedValue.of("onTimePane", new Instant(500))).advanceWatermarkTo(new Instant(1001L)).addElements(TimestampedValue.of("finalLatePane", new Instant(750)), TimestampedValue.of("alsoFinalLatePane", new Instant(250))).advanceWatermarkToInfinity();
    FixedWindows windowFn = FixedWindows.of(Duration.millis(1000L));
    Duration allowedLateness = Duration.millis(5000L);
    PCollection<String> values = p.apply(stream).apply(Window.<String>into(windowFn).triggering(AfterWatermark.pastEndOfWindow().withEarlyFirings(AfterPane.elementCountAtLeast(2)).withLateFirings(Never.ever())).discardingFiredPanes().withAllowedLateness(allowedLateness)).apply(WithKeys.<Integer, String>of(1)).apply(GroupByKey.<Integer, String>create()).apply(Values.<Iterable<String>>create()).apply(Flatten.<String>iterables());
    IntervalWindow window = windowFn.assignWindow(new Instant(100));
    PAssert.that(values).inWindow(window).containsInAnyOrder("firstPane", "alsoFirstPane", "onTimePane", "finalLatePane", "alsoFinalLatePane");
    PAssert.that(values).inCombinedNonLatePanes(window).containsInAnyOrder("firstPane", "alsoFirstPane", "onTimePane");
    PAssert.that(values).inOnTimePane(window).containsInAnyOrder("onTimePane");
    PAssert.that(values).inFinalPane(window).containsInAnyOrder("finalLatePane", "alsoFinalLatePane");
    p.run();
}