Examples with SlidingWindows org.apache.beam.sdk.transforms.windowing.SlidingWindows used on opensource projects

Example 6 with SlidingWindows

use of org.apache.beam.sdk.transforms.windowing.SlidingWindows in project beam by apache.

the class SplittableDoFnTest method testPairWithIndexWindowedTimestamped.

@Test
@Category({ ValidatesRunner.class, UsesSplittableParDo.class })
public void testPairWithIndexWindowedTimestamped() {
    // Tests that Splittable DoFn correctly propagates windowing strategy, windows and timestamps
    // of elements in the input collection.
    MutableDateTime mutableNow = Instant.now().toMutableDateTime();
    mutableNow.setMillisOfSecond(0);
    Instant now = mutableNow.toInstant();
    Instant nowP1 = now.plus(Duration.standardSeconds(1));
    Instant nowP2 = now.plus(Duration.standardSeconds(2));
    SlidingWindows windowFn = SlidingWindows.of(Duration.standardSeconds(5)).every(Duration.standardSeconds(1));
    PCollection<KV<String, Integer>> res = p.apply(Create.timestamped(TimestampedValue.of("a", now), TimestampedValue.of("bb", nowP1), TimestampedValue.of("ccccc", nowP2))).apply(Window.<String>into(windowFn)).apply(ParDo.of(new PairStringWithIndexToLength())).setCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()));
    assertEquals(windowFn, res.getWindowingStrategy().getWindowFn());
    PCollection<TimestampedValue<KV<String, Integer>>> timestamped = res.apply("Reify timestamps", ParDo.of(new ReifyTimestampsFn<KV<String, Integer>>()));
    for (int i = 0; i < 4; ++i) {
        Instant base = now.minus(Duration.standardSeconds(i));
        IntervalWindow window = new IntervalWindow(base, base.plus(Duration.standardSeconds(5)));
        List<TimestampedValue<KV<String, Integer>>> expectedUnfiltered = Arrays.asList(TimestampedValue.of(KV.of("a", 0), now), TimestampedValue.of(KV.of("bb", 0), nowP1), TimestampedValue.of(KV.of("bb", 1), nowP1), TimestampedValue.of(KV.of("ccccc", 0), nowP2), TimestampedValue.of(KV.of("ccccc", 1), nowP2), TimestampedValue.of(KV.of("ccccc", 2), nowP2), TimestampedValue.of(KV.of("ccccc", 3), nowP2), TimestampedValue.of(KV.of("ccccc", 4), nowP2));
        List<TimestampedValue<KV<String, Integer>>> expected = new ArrayList<>();
        for (TimestampedValue<KV<String, Integer>> tv : expectedUnfiltered) {
            if (!window.start().isAfter(tv.getTimestamp()) && !tv.getTimestamp().isAfter(window.maxTimestamp())) {
                expected.add(tv);
            }
        }
        assertFalse(expected.isEmpty());
        PAssert.that(timestamped).inWindow(window).containsInAnyOrder(expected);
    }
    p.run();
}

Example 7 with SlidingWindows

use of org.apache.beam.sdk.transforms.windowing.SlidingWindows in project beam by apache.

the class BeamAggregationRel method computeWindowingCostEffect.

private NodeStats computeWindowingCostEffect(NodeStats inputStat) {
    if (windowFn == null) {
        return inputStat;
    }
    WindowFn w = windowFn;
    double multiplicationFactor = 1;
    // tuples repeat in multiple windows).
    if (w instanceof SlidingWindows) {
        multiplicationFactor = ((double) ((SlidingWindows) w).getSize().getStandardSeconds()) / ((SlidingWindows) w).getPeriod().getStandardSeconds();
    }
    return NodeStats.create(inputStat.getRowCount() * multiplicationFactor, inputStat.getRate() * multiplicationFactor, BeamIOSourceRel.CONSTANT_WINDOW_SIZE);
}