Examples with IntervalWindow org.apache.beam.sdk.transforms.windowing.IntervalWindow used on opensource projects

Example 66 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class WindowEvaluatorFactoryTest method singleWindowFnSucceeds.

@Test
public void singleWindowFnSucceeds() throws Exception {
    Duration windowDuration = Duration.standardDays(7);
    Window<Long> transform = Window.into(FixedWindows.of(windowDuration));
    PCollection<Long> windowed = input.apply(transform);
    CommittedBundle<Long> inputBundle = createInputBundle();
    UncommittedBundle<Long> outputBundle = createOutputBundle(windowed);
    BoundedWindow firstSecondWindow = new IntervalWindow(EPOCH, EPOCH.plus(windowDuration));
    BoundedWindow thirdWindow = new IntervalWindow(EPOCH.minus(windowDuration), EPOCH);
    TransformResult<Long> result = runEvaluator(windowed, inputBundle);
    assertThat(Iterables.getOnlyElement(result.getOutputBundles()), Matchers.equalTo(outputBundle));
    CommittedBundle<Long> committed = outputBundle.commit(Instant.now());
    assertThat(committed.getElements(), containsInAnyOrder(// value in global window
    isSingleWindowedValue(3L, new Instant(2L), firstSecondWindow, NO_FIRING), // value in just interval window
    isSingleWindowedValue(2L, new Instant(-10L), thirdWindow, intervalWindowPane), // value in global window and two interval windows
    isSingleWindowedValue(1L, EPOCH.plus(Duration.standardDays(3)), firstSecondWindow, multiWindowPane), isSingleWindowedValue(1L, EPOCH.plus(Duration.standardDays(3)), firstSecondWindow, multiWindowPane), isSingleWindowedValue(1L, EPOCH.plus(Duration.standardDays(3)), firstSecondWindow, multiWindowPane)));
}

Example 67 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class WindowEvaluatorFactoryTest method multipleWindowsWindowFnSucceeds.

@Test
public void multipleWindowsWindowFnSucceeds() throws Exception {
    Duration windowDuration = Duration.standardDays(6);
    Duration slidingBy = Duration.standardDays(3);
    Window<Long> transform = Window.into(SlidingWindows.of(windowDuration).every(slidingBy));
    PCollection<Long> windowed = input.apply(transform);
    CommittedBundle<Long> inputBundle = createInputBundle();
    UncommittedBundle<Long> outputBundle = createOutputBundle(windowed);
    TransformResult<Long> result = runEvaluator(windowed, inputBundle);
    assertThat(Iterables.getOnlyElement(result.getOutputBundles()), Matchers.equalTo(outputBundle));
    CommittedBundle<Long> committed = outputBundle.commit(Instant.now());
    BoundedWindow w1 = new IntervalWindow(EPOCH, EPOCH.plus(windowDuration));
    BoundedWindow w2 = new IntervalWindow(EPOCH.plus(slidingBy), EPOCH.plus(slidingBy).plus(windowDuration));
    BoundedWindow wMinus1 = new IntervalWindow(EPOCH.minus(windowDuration), EPOCH);
    BoundedWindow wMinusSlide = new IntervalWindow(EPOCH.minus(windowDuration).plus(slidingBy), EPOCH.plus(slidingBy));
    assertThat(committed.getElements(), containsInAnyOrder(// Value in global window mapped to one windowed value in multiple windows
    isWindowedValue(valueInGlobalWindow.getValue(), valueInGlobalWindow.getTimestamp(), ImmutableSet.of(w1, wMinusSlide), NO_FIRING), // Value in interval window mapped to one windowed value in multiple windows
    isWindowedValue(valueInIntervalWindow.getValue(), valueInIntervalWindow.getTimestamp(), ImmutableSet.of(wMinus1, wMinusSlide), valueInIntervalWindow.getPane()), // Value in three windows mapped to three windowed values in the same multiple windows
    isWindowedValue(valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), ImmutableSet.of(w1, w2), valueInGlobalAndTwoIntervalWindows.getPane()), isWindowedValue(valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), ImmutableSet.of(w1, w2), valueInGlobalAndTwoIntervalWindows.getPane()), isWindowedValue(valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), ImmutableSet.of(w1, w2), valueInGlobalAndTwoIntervalWindows.getPane())));
}

Example 68 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class WindowEvaluatorFactoryTest method referencesEarlierWindowsSucceeds.

@Test
public void referencesEarlierWindowsSucceeds() throws Exception {
    Window<Long> transform = Window.into(new EvaluatorTestWindowFn());
    PCollection<Long> windowed = input.apply(transform);
    CommittedBundle<Long> inputBundle = createInputBundle();
    UncommittedBundle<Long> outputBundle = createOutputBundle(windowed);
    TransformResult<Long> result = runEvaluator(windowed, inputBundle);
    assertThat(Iterables.getOnlyElement(result.getOutputBundles()), Matchers.equalTo(outputBundle));
    CommittedBundle<Long> committed = outputBundle.commit(Instant.now());
    assertThat(committed.getElements(), containsInAnyOrder(// Value in global window mapped to [timestamp, timestamp+1)
    isSingleWindowedValue(valueInGlobalWindow.getValue(), valueInGlobalWindow.getTimestamp(), new IntervalWindow(valueInGlobalWindow.getTimestamp(), valueInGlobalWindow.getTimestamp().plus(Duration.millis(1L))), valueInGlobalWindow.getPane()), // Value in interval window mapped to the same window
    isWindowedValue(valueInIntervalWindow.getValue(), valueInIntervalWindow.getTimestamp(), valueInIntervalWindow.getWindows(), valueInIntervalWindow.getPane()), // Value in global window and two interval windows exploded and mapped in both ways
    isSingleWindowedValue(valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), new IntervalWindow(valueInGlobalAndTwoIntervalWindows.getTimestamp(), valueInGlobalAndTwoIntervalWindows.getTimestamp().plus(Duration.millis(1L))), valueInGlobalAndTwoIntervalWindows.getPane()), isSingleWindowedValue(valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), intervalWindow1, valueInGlobalAndTwoIntervalWindows.getPane()), isSingleWindowedValue(valueInGlobalAndTwoIntervalWindows.getValue(), valueInGlobalAndTwoIntervalWindows.getTimestamp(), intervalWindow2, valueInGlobalAndTwoIntervalWindows.getPane())));
}

Example 69 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class FnApiWindowMappingFnTest method testWindowMapping.

@Test
public void testWindowMapping() throws Exception {
    TestSdkHarness testSdkHarness = new TestSdkHarness(GlobalWindow.INSTANCE);
    FnApiWindowMappingFn windowMappingFn = new FnApiWindowMappingFn(IdGenerators.decrementingLongs(), testSdkHarness, DATA_SERVICE, testSdkHarness, WINDOW_MAPPING_SPEC, IntervalWindowCoder.of(), GlobalWindow.Coder.INSTANCE);
    // Check mapping an element returns the expected result.
    BoundedWindow inputWindow = new IntervalWindow(Instant.now(), Duration.standardMinutes(1));
    assertEquals(GlobalWindow.INSTANCE, windowMappingFn.getSideInputWindow(inputWindow));
    assertEquals(inputWindow, ((KV) testSdkHarness.getInputValues().get(0).getValue()).getValue());
    // Check mapping a different element returns the expected result.
    BoundedWindow inputWindow2 = new IntervalWindow(Instant.now(), Duration.standardMinutes(2));
    assertEquals(GlobalWindow.INSTANCE, windowMappingFn.getSideInputWindow(inputWindow2));
    assertEquals(inputWindow2, ((KV) testSdkHarness.getInputValues().get(1).getValue()).getValue());
    // Check that mapping the same element returns a cached result.
    assertEquals(GlobalWindow.INSTANCE, windowMappingFn.getSideInputWindow(inputWindow));
    assertEquals(2, testSdkHarness.getInputValues().size());
}

Example 70 with IntervalWindow

use of org.apache.beam.sdk.transforms.windowing.IntervalWindow in project beam by apache.

the class WindowUtilsTest method testGetWindowStrategy.

@Test
public void testGetWindowStrategy() throws IOException {
    SdkComponents components = SdkComponents.create();
    String environmentId = components.registerEnvironment(Environments.createDockerEnvironment("java"));
    WindowingStrategy<Object, IntervalWindow> expected = WindowingStrategy.of(FixedWindows.of(Duration.standardMinutes(1))).withMode(WindowingStrategy.AccumulationMode.DISCARDING_FIRED_PANES).withTimestampCombiner(TimestampCombiner.END_OF_WINDOW).withAllowedLateness(Duration.ZERO).withEnvironmentId(environmentId);
    components.registerWindowingStrategy(expected);
    String collectionId = components.registerPCollection(PCollection.createPrimitiveOutputInternal(Pipeline.create(), expected, PCollection.IsBounded.BOUNDED, VoidCoder.of()).setName("name"));
    WindowingStrategy<?, ?> actual = WindowUtils.getWindowStrategy(collectionId, components.toComponents());
    assertEquals(expected, actual);
}