Examples with IntervalWindow org.apache.beam.sdk.transforms.windowing.IntervalWindow used on opensource projects

Example 31 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class CreateStreamTest method testLateDataAccumulating.

@Test
public void testLateDataAccumulating() throws IOException {
    Instant instant = new Instant(0);
    CreateStream<Integer> source = CreateStream.of(VarIntCoder.of(), batchDuration()).emptyBatch().advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(6))).nextBatch(TimestampedValue.of(1, instant), TimestampedValue.of(2, instant), TimestampedValue.of(3, instant)).advanceWatermarkForNextBatch(instant.plus(Duration.standardMinutes(20))).nextBatch(TimestampedValue.of(4, instant), TimestampedValue.of(5, instant)).advanceNextBatchWatermarkToInfinity().nextBatch(TimestampedValue.of(-1, instant), TimestampedValue.of(-2, instant), TimestampedValue.of(-3, instant));
    PCollection<Integer> windowed = p.apply(source).apply(Window.<Integer>into(FixedWindows.of(Duration.standardMinutes(5))).triggering(AfterWatermark.pastEndOfWindow().withEarlyFirings(AfterProcessingTime.pastFirstElementInPane().plusDelayOf(Duration.standardMinutes(2))).withLateFirings(AfterPane.elementCountAtLeast(1))).accumulatingFiredPanes().withAllowedLateness(Duration.standardMinutes(5), Window.ClosingBehavior.FIRE_ALWAYS));
    PCollection<Integer> triggered = windowed.apply(WithKeys.of(1)).apply(GroupByKey.create()).apply(Values.create()).apply(Flatten.iterables());
    PCollection<Long> count = windowed.apply(Combine.globally(Count.<Integer>combineFn()).withoutDefaults());
    PCollection<Integer> sum = windowed.apply(Sum.integersGlobally().withoutDefaults());
    IntervalWindow window = new IntervalWindow(instant, instant.plus(Duration.standardMinutes(5L)));
    PAssert.that(triggered).inFinalPane(window).containsInAnyOrder(1, 2, 3, 4, 5);
    PAssert.that(triggered).inOnTimePane(window).containsInAnyOrder(1, 2, 3);
    PAssert.that(count).inWindow(window).satisfies(input -> {
        for (Long count1 : input) {
            assertThat(count1, allOf(greaterThanOrEqualTo(3L), lessThanOrEqualTo(5L)));
        }
        return null;
    });
    PAssert.that(sum).inWindow(window).satisfies(input -> {
        for (Integer sum1 : input) {
            assertThat(sum1, allOf(greaterThanOrEqualTo(6), lessThanOrEqualTo(15)));
        }
        return null;
    });
    p.run();
}

Example 32 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class SplittableDoFnTest method testPairWithIndexWindowedTimestamped.

private void testPairWithIndexWindowedTimestamped(IsBounded bounded) {
    // Tests that Splittable DoFn correctly propagates windowing strategy, windows and timestamps
    // of elements in the input collection.
    MutableDateTime mutableNow = Instant.now().toMutableDateTime();
    mutableNow.setMillisOfSecond(0);
    Instant now = mutableNow.toInstant();
    Instant nowP1 = now.plus(Duration.standardSeconds(1));
    Instant nowP2 = now.plus(Duration.standardSeconds(2));
    SlidingWindows windowFn = SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
    PCollection<KV<String, Integer>> res = p.apply(Create.timestamped(TimestampedValue.of("a", now), TimestampedValue.of("bb", nowP1), TimestampedValue.of("ccccc", nowP2))).apply(Window.into(windowFn)).apply(ParDo.of(pairStringWithIndexToLengthFn(bounded))).setCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()));
    assertEquals(windowFn, res.getWindowingStrategy().getWindowFn());
    PCollection<TimestampedValue<KV<String, Integer>>> timestamped = res.apply(Reify.timestamps());
    for (int i = 0; i < 4; ++i) {
        Instant base = now.minus(Duration.standardSeconds(i));
        IntervalWindow window = new IntervalWindow(base, base.plus(Duration.standardSeconds(5)));
        List<TimestampedValue<KV<String, Integer>>> expectedUnfiltered = Arrays.asList(TimestampedValue.of(KV.of("a", 0), now), TimestampedValue.of(KV.of("bb", 0), nowP1), TimestampedValue.of(KV.of("bb", 1), nowP1), TimestampedValue.of(KV.of("ccccc", 0), nowP2), TimestampedValue.of(KV.of("ccccc", 1), nowP2), TimestampedValue.of(KV.of("ccccc", 2), nowP2), TimestampedValue.of(KV.of("ccccc", 3), nowP2), TimestampedValue.of(KV.of("ccccc", 4), nowP2));
        List<TimestampedValue<KV<String, Integer>>> expected = new ArrayList<>();
        for (TimestampedValue<KV<String, Integer>> tv : expectedUnfiltered) {
            if (!window.start().isAfter(tv.getTimestamp()) && !tv.getTimestamp().isAfter(window.maxTimestamp())) {
                expected.add(tv);
            }
        }
        assertFalse(expected.isEmpty());
        PAssert.that(timestamped).inWindow(window).containsInAnyOrder(expected);
    }
    p.run();
}

Example 33 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class DoFnInvokersTest method testOnTimerWithWindow.

@Test
public void testOnTimerWithWindow() throws Exception {
    final String timerId = "my-timer-id";
    final IntervalWindow testWindow = new IntervalWindow(new Instant(0), new Instant(15));
    when(mockArgumentProvider.window()).thenReturn(testWindow);
    class SimpleTimerDoFn extends DoFn<String, String> {

        public IntervalWindow window = null;

        @TimerId(timerId)
        private final TimerSpec myTimer = TimerSpecs.timer(TimeDomain.PROCESSING_TIME);

        @ProcessElement
        public void process(ProcessContext c) {
        }

        @OnTimer(timerId)
        public void onMyTimer(IntervalWindow w) {
            window = w;
        }
    }
    SimpleTimerDoFn fn = new SimpleTimerDoFn();
    DoFnInvoker<String, String> invoker = DoFnInvokers.invokerFor(fn);
    invoker.invokeOnTimer(TimerDeclaration.PREFIX + timerId, "", mockArgumentProvider);
    assertThat(fn.window, equalTo(testWindow));
}

Example 34 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class RepeatedlyStateMachineTest method testShouldFireAfterMerge.

@Test
public void testShouldFireAfterMerge() throws Exception {
    tester = TriggerStateMachineTester.forTrigger(RepeatedlyStateMachine.forever(AfterPaneStateMachine.elementCountAtLeast(2)), Sessions.withGapDuration(Duration.millis(10)));
    tester.injectElements(1);
    IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11));
    assertFalse(tester.shouldFire(firstWindow));
    tester.injectElements(5);
    IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15));
    assertFalse(tester.shouldFire(secondWindow));
    // Merge them, if the merged window were on the second trigger, it would be ready
    tester.mergeWindows();
    IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15));
    assertTrue(tester.shouldFire(mergedWindow));
}

Example 35 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class StatefulDoFnRunnerTest method testLateDropping.

private void testLateDropping(boolean ordered) throws Exception {
    MetricsContainerImpl container = new MetricsContainerImpl("any");
    MetricsEnvironment.setCurrentContainer(container);
    timerInternals.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
    timerInternals.advanceOutputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
    MyDoFn fn = MyDoFn.create(ordered);
    DoFnRunner<KV<String, Integer>, Integer> runner = createStatefulDoFnRunner(fn);
    runner.startBundle();
    IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(0L + WINDOW_SIZE));
    Instant timestamp = new Instant(0);
    runner.processElement(WindowedValue.of(KV.of("hello", 1), timestamp, window, PaneInfo.NO_FIRING));
    long droppedValues = container.getCounter(MetricName.named(StatefulDoFnRunner.class, StatefulDoFnRunner.DROPPED_DUE_TO_LATENESS_COUNTER)).getCumulative();
    assertEquals(1L, droppedValues);
    runner.finishBundle();
}