Examples with IntervalWindow org.apache.beam.sdk.transforms.windowing.IntervalWindow used on opensource projects

Example 26 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class DoFnTesterTest method peekValuesInWindow.

@Test
public void peekValuesInWindow() throws Exception {
    try (DoFnTester<Long, String> tester = DoFnTester.of(new CounterDoFn())) {
        tester.startBundle();
        tester.processElement(1L);
        tester.processElement(2L);
        tester.finishBundle();
        assertThat(tester.peekOutputElementsInWindow(GlobalWindow.INSTANCE), containsInAnyOrder(TimestampedValue.of("1", new Instant(1000L)), TimestampedValue.of("2", new Instant(2000L))));
        assertThat(tester.peekOutputElementsInWindow(new IntervalWindow(new Instant(0L), new Instant(10L))), Matchers.emptyIterable());
    }
}

Example 27 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class WindowedValueTest method testWindowedValueCoder.

@Test
public void testWindowedValueCoder() throws CoderException {
    Instant timestamp = new Instant(1234);
    WindowedValue<String> value = WindowedValue.of("abc", new Instant(1234), Arrays.asList(new IntervalWindow(timestamp, timestamp.plus(Duration.millis(1000))), new IntervalWindow(timestamp.plus(Duration.millis(1000)), timestamp.plus(Duration.millis(2000)))), PaneInfo.NO_FIRING);
    Coder<WindowedValue<String>> windowedValueCoder = WindowedValue.getFullCoder(StringUtf8Coder.of(), IntervalWindow.getCoder());
    byte[] encodedValue = CoderUtils.encodeToByteArray(windowedValueCoder, value);
    WindowedValue<String> decodedValue = CoderUtils.decodeFromByteArray(windowedValueCoder, encodedValue);
    Assert.assertEquals(value.getValue(), decodedValue.getValue());
    Assert.assertEquals(value.getTimestamp(), decodedValue.getTimestamp());
    Assert.assertArrayEquals(value.getWindows().toArray(), decodedValue.getWindows().toArray());
}

Example 28 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class WindowedValueTest method testExplodeWindowsManyWindowsMultipleWindowedValues.

@Test
public void testExplodeWindowsManyWindowsMultipleWindowedValues() {
    Instant now = Instant.now();
    BoundedWindow centerWindow = new IntervalWindow(now.minus(Duration.millis(1000L)), now.plus(Duration.millis(1000L)));
    BoundedWindow pastWindow = new IntervalWindow(now.minus(Duration.millis(1500L)), now.plus(Duration.millis(500L)));
    BoundedWindow futureWindow = new IntervalWindow(now.minus(Duration.millis(500L)), now.plus(Duration.millis(1500L)));
    BoundedWindow futureFutureWindow = new IntervalWindow(now, now.plus(Duration.millis(2000L)));
    PaneInfo pane = PaneInfo.createPane(false, false, Timing.ON_TIME, 3L, 0L);
    WindowedValue<String> value = WindowedValue.of("foo", now, ImmutableList.of(pastWindow, centerWindow, futureWindow, futureFutureWindow), pane);
    assertThat(value.explodeWindows(), containsInAnyOrder(WindowedValue.of("foo", now, futureFutureWindow, pane), WindowedValue.of("foo", now, futureWindow, pane), WindowedValue.of("foo", now, centerWindow, pane), WindowedValue.of("foo", now, pastWindow, pane)));
    assertThat(value.isSingleWindowedValue(), equalTo(false));
}

Example 29 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class SparkCombineFnTest method testSlidingCombineFnNonMerging.

@Test
public void testSlidingCombineFnNonMerging() throws Exception {
    WindowingStrategy<Object, IntervalWindow> strategy = WindowingStrategy.of(SlidingWindows.of(Duration.millis(3000)).every(Duration.millis(1000)));
    SparkCombineFn<KV<String, Integer>, Integer, Long, Long> sparkCombineFn = SparkCombineFn.keyed(combineFn, opts, Collections.emptyMap(), strategy, SparkCombineFn.WindowedAccumulator.Type.NON_MERGING);
    Instant now = Instant.ofEpochMilli(0);
    WindowedValue<KV<String, Integer>> first = input("key", 1, now.plus(Duration.millis(5000)), strategy.getWindowFn());
    WindowedValue<KV<String, Integer>> second = input("key", 2, now.plus(Duration.millis(1500)), strategy.getWindowFn());
    WindowedValue<KV<String, Integer>> third = input("key", 3, now.plus(Duration.millis(500)), strategy.getWindowFn());
    SparkCombineFn.WindowedAccumulator<KV<String, Integer>, Integer, Long, ?> c1 = sparkCombineFn.createCombiner(first);
    SparkCombineFn.WindowedAccumulator<KV<String, Integer>, Integer, Long, ?> c2 = sparkCombineFn.createCombiner(third);
    sparkCombineFn.mergeValue(c1, second);
    SparkCombineFn.WindowedAccumulator<KV<String, Integer>, Integer, Long, ?> c3 = sparkCombineFn.mergeCombiners(c1, c2);
    Iterable<WindowedValue<Long>> output = sparkCombineFn.extractOutput(c3);
    assertEquals(7, Iterables.size(output));
    List<String> format = StreamSupport.stream(output.spliterator(), false).map(val -> val.getValue() + ":" + val.getTimestamp().getMillis()).collect(Collectors.toList());
    assertUnorderedEquals(Lists.newArrayList("3:999", "5:1999", "5:2999", "2:3999", "1:5999", "1:6999", "1:7999"), format);
}

Example 30 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class SparkCombineFnTest method testSlidingCombineFnExplode.

@Test
public void testSlidingCombineFnExplode() throws Exception {
    WindowingStrategy<Object, IntervalWindow> strategy = WindowingStrategy.of(SlidingWindows.of(Duration.millis(3000)).every(Duration.millis(1000)));
    SparkCombineFn<KV<String, Integer>, Integer, Long, Long> sparkCombineFn = SparkCombineFn.keyed(combineFn, opts, Collections.emptyMap(), strategy, SparkCombineFn.WindowedAccumulator.Type.EXPLODE_WINDOWS);
    Instant now = Instant.ofEpochMilli(0);
    WindowedValue<KV<String, Integer>> first = input("key", 1, now.plus(Duration.millis(5000)), strategy.getWindowFn());
    WindowedValue<KV<String, Integer>> second = input("key", 2, now.plus(Duration.millis(1500)), strategy.getWindowFn());
    WindowedValue<KV<String, Integer>> third = input("key", 3, now.plus(Duration.millis(500)), strategy.getWindowFn());
    Map<KV<String, BoundedWindow>, List<WindowedValue<KV<String, Integer>>>> groupByKeyAndWindow;
    groupByKeyAndWindow = Stream.of(first, second, third).flatMap(e -> StreamSupport.stream(e.explodeWindows().spliterator(), false)).collect(Collectors.groupingBy(e -> KV.of(e.getValue().getKey(), Iterables.getOnlyElement(e.getWindows()))));
    List<String> result = new ArrayList<>();
    for (Map.Entry<KV<String, BoundedWindow>, List<WindowedValue<KV<String, Integer>>>> e : groupByKeyAndWindow.entrySet()) {
        SparkCombineFn.WindowedAccumulator<KV<String, Integer>, Integer, Long, ?> combiner = null;
        for (WindowedValue<KV<String, Integer>> v : e.getValue()) {
            if (combiner == null) {
                combiner = sparkCombineFn.createCombiner(v);
            } else {
                combiner.add(v, sparkCombineFn);
            }
        }
        WindowedValue<Long> combined = Iterables.getOnlyElement(combiner.extractOutput());
        result.add(combined.getValue() + ":" + combined.getTimestamp().getMillis());
    }
    assertUnorderedEquals(Lists.newArrayList("3:999", "5:1999", "5:2999", "2:3999", "1:5999", "1:6999", "1:7999"), result);
}