Examples with WindowNamespace org.apache.beam.runners.core.StateNamespaces.WindowNamespace used on opensource projects

Example 1 with WindowNamespace

use of org.apache.beam.runners.core.StateNamespaces.WindowNamespace in project beam by apache.

the class DoFnOperator method fireTimer.

// allow overriding this in WindowDoFnOperator
public void fireTimer(InternalTimer<?, TimerData> timer) {
    TimerInternals.TimerData timerData = timer.getNamespace();
    StateNamespace namespace = timerData.getNamespace();
    // This is a user timer, so namespace must be WindowNamespace
    checkArgument(namespace instanceof WindowNamespace);
    BoundedWindow window = ((WindowNamespace) namespace).getWindow();
    pushbackDoFnRunner.onTimer(timerData.getTimerId(), window, timerData.getTimestamp(), timerData.getDomain());
}

Example 2 with WindowNamespace

use of org.apache.beam.runners.core.StateNamespaces.WindowNamespace in project beam by apache.

the class ReduceFnRunner method onTimers.

public void onTimers(Iterable<TimerData> timers) throws Exception {
    if (!timers.iterator().hasNext()) {
        return;
    }
    // Create a reusable context for each window and begin prefetching necessary
    // state.
    Map<BoundedWindow, WindowActivation> windowActivations = new HashMap();
    for (TimerData timer : timers) {
        checkArgument(timer.getNamespace() instanceof WindowNamespace, "Expected timer to be in WindowNamespace, but was in %s", timer.getNamespace());
        @SuppressWarnings("unchecked") WindowNamespace<W> windowNamespace = (WindowNamespace<W>) timer.getNamespace();
        W window = windowNamespace.getWindow();
        WindowTracing.debug("ReduceFnRunner: Received timer key:{}; window:{}; data:{} with " + "inputWatermark:{}; outputWatermark:{}", key, window, timer, timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
        // that show up too late. Window GC is management by an event time timer
        if (TimeDomain.EVENT_TIME != timer.getDomain() && windowIsExpired(window)) {
            continue;
        }
        // the final pane.
        if (windowActivations.containsKey(window)) {
            continue;
        }
        ReduceFn<K, InputT, OutputT, W>.Context directContext = contextFactory.base(window, StateStyle.DIRECT);
        ReduceFn<K, InputT, OutputT, W>.Context renamedContext = contextFactory.base(window, StateStyle.RENAMED);
        WindowActivation windowActivation = new WindowActivation(directContext, renamedContext);
        windowActivations.put(window, windowActivation);
        // Perform prefetching of state to determine if the trigger should fire.
        if (windowActivation.isGarbageCollection) {
            triggerRunner.prefetchIsClosed(directContext.state());
        } else {
            triggerRunner.prefetchShouldFire(directContext.window(), directContext.state());
        }
    }
    // For those windows that are active and open, prefetch the triggering or emitting state.
    for (WindowActivation timer : windowActivations.values()) {
        if (timer.windowIsActiveAndOpen()) {
            ReduceFn<K, InputT, OutputT, W>.Context directContext = timer.directContext;
            if (timer.isGarbageCollection) {
                prefetchOnTrigger(directContext, timer.renamedContext);
            } else if (triggerRunner.shouldFire(directContext.window(), directContext.timers(), directContext.state())) {
                prefetchEmit(directContext, timer.renamedContext);
            }
        }
    }
    // Perform processing now that everything is prefetched.
    for (WindowActivation windowActivation : windowActivations.values()) {
        ReduceFn<K, InputT, OutputT, W>.Context directContext = windowActivation.directContext;
        ReduceFn<K, InputT, OutputT, W>.Context renamedContext = windowActivation.renamedContext;
        if (windowActivation.isGarbageCollection) {
            WindowTracing.debug("ReduceFnRunner: Cleaning up for key:{}; window:{} with inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
            boolean windowIsActiveAndOpen = windowActivation.windowIsActiveAndOpen();
            if (windowIsActiveAndOpen) {
                // We need to call onTrigger to emit the final pane if required.
                // The final pane *may* be ON_TIME if no prior ON_TIME pane has been emitted,
                // and the watermark has passed the end of the window.
                @Nullable Instant newHold = onTrigger(directContext, renamedContext, true, /* isFinished */
                windowActivation.isEndOfWindow);
                checkState(newHold == null, "Hold placed at %s despite isFinished being true.", newHold);
            }
            // Cleanup flavor B: Clear all the remaining state for this window since we'll never
            // see elements for it again.
            clearAllState(directContext, renamedContext, windowIsActiveAndOpen);
        } else {
            WindowTracing.debug("{}.onTimers: Triggering for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
            if (windowActivation.windowIsActiveAndOpen() && triggerRunner.shouldFire(directContext.window(), directContext.timers(), directContext.state())) {
                emit(directContext, renamedContext);
            }
            if (windowActivation.isEndOfWindow) {
                // If the window strategy trigger includes a watermark trigger then at this point
                // there should be no data holds, either because we'd already cleared them on an
                // earlier onTrigger, or because we just cleared them on the above emit.
                // We could assert this but it is very expensive.
                // Since we are processing an on-time firing we should schedule the garbage collection
                // timer. (If getAllowedLateness is zero then the timer event will be considered a
                // cleanup event and handled by the above).
                // Note we must do this even if the trigger is finished so that we are sure to cleanup
                // any final trigger finished bits.
                checkState(windowingStrategy.getAllowedLateness().isLongerThan(Duration.ZERO), "Unexpected zero getAllowedLateness");
                Instant cleanupTime = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy);
                WindowTracing.debug("ReduceFnRunner.onTimer: Scheduling cleanup timer for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), cleanupTime, timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
                checkState(!cleanupTime.isAfter(BoundedWindow.TIMESTAMP_MAX_VALUE), "Cleanup time %s is beyond end-of-time", cleanupTime);
                directContext.timers().setTimer(cleanupTime, TimeDomain.EVENT_TIME);
            }
        }
    }
}

Example 3 with WindowNamespace

use of org.apache.beam.runners.core.StateNamespaces.WindowNamespace in project beam by apache.

the class DoFnOperator method fireTimer.

// allow overriding this in WindowDoFnOperator
protected void fireTimer(TimerData timerData) {
    LOG.debug("Firing timer: {} at {} with output time {}", timerData.getTimerId(), timerData.getTimestamp().getMillis(), timerData.getOutputTimestamp().getMillis());
    StateNamespace namespace = timerData.getNamespace();
    // This is a user timer, so namespace must be WindowNamespace
    checkArgument(namespace instanceof WindowNamespace);
    BoundedWindow window = ((WindowNamespace) namespace).getWindow();
    timerInternals.onFiredOrDeletedTimer(timerData);
    pushbackDoFnRunner.onTimer(timerData.getTimerId(), timerData.getTimerFamilyId(), keyedStateInternals.getKey(), window, timerData.getTimestamp(), timerData.getOutputTimestamp(), timerData.getDomain());
}

Example 4 with WindowNamespace

use of org.apache.beam.runners.core.StateNamespaces.WindowNamespace in project beam by apache.

the class ApexParDoOperator method fireTimer.

@Override
public void fireTimer(Object key, Collection<TimerData> timerDataSet) {
    pushbackDoFnRunner.startBundle();
    @SuppressWarnings("unchecked") Coder<Object> keyCoder = (Coder) currentKeyStateInternals.getKeyCoder();
    ((StateInternalsProxy) currentKeyStateInternals).setKey(key);
    currentKeyTimerInternals.setContext(key, keyCoder, new Instant(this.currentInputWatermark), new Instant(this.currentOutputWatermark));
    for (TimerData timerData : timerDataSet) {
        StateNamespace namespace = timerData.getNamespace();
        checkArgument(namespace instanceof WindowNamespace);
        BoundedWindow window = ((WindowNamespace<?>) namespace).getWindow();
        pushbackDoFnRunner.onTimer(timerData.getTimerId(), window, timerData.getTimestamp(), timerData.getDomain());
    }
    pushbackDoFnRunner.finishBundle();
}

Example 5 with WindowNamespace

use of org.apache.beam.runners.core.StateNamespaces.WindowNamespace in project beam by apache.

the class SimpleParDoFn method processUserTimer.

private void processUserTimer(TimerData timer) throws Exception {
    if (fnSignature.timerDeclarations().containsKey(timer.getTimerId()) || fnSignature.timerFamilyDeclarations().containsKey(timer.getTimerFamilyId())) {
        BoundedWindow window = ((WindowNamespace) timer.getNamespace()).getWindow();
        fnRunner.onTimer(timer.getTimerId(), timer.getTimerFamilyId(), this.stepContext.stateInternals().getKey(), window, timer.getTimestamp(), timer.getOutputTimestamp(), timer.getDomain());
    }
}