Examples with BoundedWindow org.apache.beam.sdk.transforms.windowing.BoundedWindow used on opensource projects

Example 6 with BoundedWindow

use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.

the class SideInputHandler method addSideInputValue.

/**
   * Add the given value to the internal side-input store of the given side input. This
   * might change the result of {@link #isReady(PCollectionView, BoundedWindow)} for that side
   * input.
   */
public void addSideInputValue(PCollectionView<?> sideInput, WindowedValue<Iterable<?>> value) {
    @SuppressWarnings("unchecked") Coder<BoundedWindow> windowCoder = (Coder<BoundedWindow>) sideInput.getWindowingStrategyInternal().getWindowFn().windowCoder();
    // reify the WindowedValue
    List<WindowedValue<?>> inputWithReifiedWindows = new ArrayList<>();
    for (Object e : value.getValue()) {
        inputWithReifiedWindows.add(value.withValue(e));
    }
    StateTag<ValueState<Iterable<WindowedValue<?>>>> stateTag = sideInputContentsTags.get(sideInput);
    for (BoundedWindow window : value.getWindows()) {
        stateInternals.state(StateNamespaces.window(windowCoder, window), stateTag).write(inputWithReifiedWindows);
        stateInternals.state(StateNamespaces.global(), availableWindowsTags.get(sideInput)).add(window);
    }
}

Example 7 with BoundedWindow

use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.

the class SimplePushbackSideInputDoFnRunner method processElementInReadyWindows.

@Override
public Iterable<WindowedValue<InputT>> processElementInReadyWindows(WindowedValue<InputT> elem) {
    if (views.isEmpty()) {
        // When there are no side inputs, we can preserve the compressed representation.
        underlying.processElement(elem);
        return Collections.emptyList();
    }
    ImmutableList.Builder<WindowedValue<InputT>> pushedBack = ImmutableList.builder();
    for (WindowedValue<InputT> windowElem : elem.explodeWindows()) {
        BoundedWindow mainInputWindow = Iterables.getOnlyElement(windowElem.getWindows());
        if (isReady(mainInputWindow)) {
            // When there are any side inputs, we have to process the element in each window
            // individually, to disambiguate access to per-window side inputs.
            underlying.processElement(windowElem);
        } else {
            notReadyWindows.add(mainInputWindow);
            pushedBack.add(windowElem);
        }
    }
    return pushedBack.build();
}

Example 8 with BoundedWindow

use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.

the class ProcessFnRunner method processElementInReadyWindows.

@Override
public Iterable<WindowedValue<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>>> processElementInReadyWindows(WindowedValue<KeyedWorkItem<String, ElementAndRestriction<InputT, RestrictionT>>> windowedKWI) {
    checkTrivialOuterWindows(windowedKWI);
    BoundedWindow window = getUnderlyingWindow(windowedKWI.getValue());
    if (!isReady(window)) {
        return Collections.singletonList(windowedKWI);
    }
    underlying.processElement(windowedKWI);
    return Collections.emptyList();
}

Example 9 with BoundedWindow

use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.

the class ProcessFnRunner method checkTrivialOuterWindows.

private static <T> void checkTrivialOuterWindows(WindowedValue<KeyedWorkItem<String, T>> windowedKWI) {
    // In practice it will be in 0 or 1 windows (ValueInEmptyWindows or ValueInGlobalWindow)
    Collection<? extends BoundedWindow> outerWindows = windowedKWI.getWindows();
    if (!outerWindows.isEmpty()) {
        checkArgument(outerWindows.size() == 1, "The KeyedWorkItem itself must not be in multiple windows, but was in: %s", outerWindows);
        BoundedWindow onlyWindow = Iterables.getOnlyElement(outerWindows);
        checkArgument(onlyWindow instanceof GlobalWindow, "KeyedWorkItem must be in the Global window, but was in: %s", onlyWindow);
    }
}

Example 10 with BoundedWindow

use of org.apache.beam.sdk.transforms.windowing.BoundedWindow in project beam by apache.

the class StatefulDoFnRunner method processElement.

@Override
public void processElement(WindowedValue<InputT> input) {
    // StatefulDoFnRunner always observes windows, so we need to explode
    for (WindowedValue<InputT> value : input.explodeWindows()) {
        BoundedWindow window = value.getWindows().iterator().next();
        if (isLate(window)) {
            // The element is too late for this window.
            droppedDueToLateness.inc();
            WindowTracing.debug("StatefulDoFnRunner.processElement: Dropping element at {}; window:{} " + "since too far behind inputWatermark:{}", input.getTimestamp(), window, cleanupTimer.currentInputWatermarkTime());
        } else {
            cleanupTimer.setForWindow(window);
            doFnRunner.processElement(value);
        }
    }
}