Examples with WindowedValue org.apache.beam.sdk.util.WindowedValue used on opensource projects

Example 11 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method testPaneInfoAllStatesAfterWatermark.

@Test
public void testPaneInfoAllStatesAfterWatermark() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(FixedWindows.of(Duration.millis(10))).withTrigger(Repeatedly.forever(AfterFirst.of(AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))).withMode(AccumulationMode.DISCARDING_FIRED_PANES).withAllowedLateness(Duration.millis(100)).withTimestampCombiner(TimestampCombiner.EARLIEST).withClosingBehavior(ClosingBehavior.FIRE_ALWAYS));
    tester.advanceInputWatermark(new Instant(0));
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY, 0, -1))));
    assertThat(output, contains(WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10)));
    tester.advanceInputWatermark(new Instant(50));
    // We should get the ON_TIME pane even though it is empty,
    // because we have an AfterWatermark.pastEndOfWindow() trigger.
    output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.ON_TIME, 1, 0))));
    assertThat(output, contains(WindowMatchers.isSingleWindowedValue(emptyIterable(), 9, 0, 10)));
    // We should get the final pane even though it is empty.
    tester.advanceInputWatermark(new Instant(150));
    output = tester.extractOutput();
    assertThat(output, contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.LATE, 2, 1))));
    assertThat(output, contains(WindowMatchers.isSingleWindowedValue(emptyIterable(), 9, 0, 10)));
}

Example 12 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method fireNonEmptyOnDrainInGlobalWindow.

/**
   * We should fire a non-empty ON_TIME pane in the GlobalWindow when the watermark moves to
   * end-of-time.
   */
@Test
public void fireNonEmptyOnDrainInGlobalWindow() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, GlobalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(new GlobalWindows()).withTrigger(Repeatedly.<GlobalWindow>forever(AfterPane.elementCountAtLeast(3))).withMode(AccumulationMode.DISCARDING_FIRED_PANES));
    tester.advanceInputWatermark(new Instant(0));
    final int n = 20;
    for (int i = 0; i < n; i++) {
        tester.injectElements(TimestampedValue.of(i, new Instant(i)));
    }
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertEquals(n / 3, output.size());
    for (int i = 0; i < output.size(); i++) {
        assertEquals(Timing.EARLY, output.get(i).getPane().getTiming());
        assertEquals(i, output.get(i).getPane().getIndex());
        assertEquals(3, Iterables.size(output.get(i).getValue()));
    }
    tester.advanceInputWatermark(BoundedWindow.TIMESTAMP_MAX_VALUE);
    output = tester.extractOutput();
    assertEquals(1, output.size());
    assertEquals(Timing.ON_TIME, output.get(0).getPane().getTiming());
    assertEquals(n / 3, output.get(0).getPane().getIndex());
    assertEquals(n - ((n / 3) * 3), Iterables.size(output.get(0).getValue()));
}

Example 13 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class ReduceFnRunnerTest method noEmptyPanesFinalIfNonEmpty.

@Test
public void noEmptyPanesFinalIfNonEmpty() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(WindowingStrategy.of(FixedWindows.of(Duration.millis(10))).withTrigger(Repeatedly.<IntervalWindow>forever(AfterFirst.<IntervalWindow>of(AfterPane.elementCountAtLeast(2), AfterWatermark.pastEndOfWindow()))).withMode(AccumulationMode.ACCUMULATING_FIRED_PANES).withAllowedLateness(Duration.millis(100)).withTimestampCombiner(TimestampCombiner.EARLIEST).withClosingBehavior(ClosingBehavior.FIRE_IF_NON_EMPTY));
    tester.advanceInputWatermark(new Instant(0));
    tester.injectElements(TimestampedValue.of(1, new Instant(1)), TimestampedValue.of(2, new Instant(2)));
    tester.advanceInputWatermark(new Instant(20));
    tester.advanceInputWatermark(new Instant(250));
    List<WindowedValue<Iterable<Integer>>> output = tester.extractOutput();
    assertThat(output, contains(// Trigger with 2 elements
    WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 1, 0, 10), // Trigger for the empty on time pane
    WindowMatchers.isSingleWindowedValue(containsInAnyOrder(1, 2), 9, 0, 10)));
}

Example 14 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class SideInputHandler method addSideInputValue.

/**
   * Add the given value to the internal side-input store of the given side input. This
   * might change the result of {@link #isReady(PCollectionView, BoundedWindow)} for that side
   * input.
   */
public void addSideInputValue(PCollectionView<?> sideInput, WindowedValue<Iterable<?>> value) {
    @SuppressWarnings("unchecked") Coder<BoundedWindow> windowCoder = (Coder<BoundedWindow>) sideInput.getWindowingStrategyInternal().getWindowFn().windowCoder();
    // reify the WindowedValue
    List<WindowedValue<?>> inputWithReifiedWindows = new ArrayList<>();
    for (Object e : value.getValue()) {
        inputWithReifiedWindows.add(value.withValue(e));
    }
    StateTag<ValueState<Iterable<WindowedValue<?>>>> stateTag = sideInputContentsTags.get(sideInput);
    for (BoundedWindow window : value.getWindows()) {
        stateInternals.state(StateNamespaces.window(windowCoder, window), stateTag).write(inputWithReifiedWindows);
        stateInternals.state(StateNamespaces.global(), availableWindowsTags.get(sideInput)).add(window);
    }
}

Example 15 with WindowedValue

use of org.apache.beam.sdk.util.WindowedValue in project beam by apache.

the class SimplePushbackSideInputDoFnRunner method processElementInReadyWindows.

@Override
public Iterable<WindowedValue<InputT>> processElementInReadyWindows(WindowedValue<InputT> elem) {
    if (views.isEmpty()) {
        // When there are no side inputs, we can preserve the compressed representation.
        underlying.processElement(elem);
        return Collections.emptyList();
    }
    ImmutableList.Builder<WindowedValue<InputT>> pushedBack = ImmutableList.builder();
    for (WindowedValue<InputT> windowElem : elem.explodeWindows()) {
        BoundedWindow mainInputWindow = Iterables.getOnlyElement(windowElem.getWindows());
        if (isReady(mainInputWindow)) {
            // When there are any side inputs, we have to process the element in each window
            // individually, to disambiguate access to per-window side inputs.
            underlying.processElement(windowElem);
        } else {
            notReadyWindows.add(mainInputWindow);
            pushedBack.add(windowElem);
        }
    }
    return pushedBack.build();
}