Examples with TimerData org.apache.beam.runners.core.TimerInternals.TimerData used on opensource projects

Example 1 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class TimerInternalsTest method testCompareTo.

@Test
public void testCompareTo() {
    Instant firstTimestamp = new Instant(100);
    Instant secondTimestamp = new Instant(200);
    IntervalWindow firstWindow = new IntervalWindow(new Instant(0), firstTimestamp);
    IntervalWindow secondWindow = new IntervalWindow(firstTimestamp, secondTimestamp);
    Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
    StateNamespace firstWindowNs = StateNamespaces.window(windowCoder, firstWindow);
    StateNamespace secondWindowNs = StateNamespaces.window(windowCoder, secondWindow);
    TimerData firstEventTime = TimerData.of(firstWindowNs, firstTimestamp, TimeDomain.EVENT_TIME);
    TimerData secondEventTime = TimerData.of(firstWindowNs, secondTimestamp, TimeDomain.EVENT_TIME);
    TimerData thirdEventTime = TimerData.of(secondWindowNs, secondTimestamp, TimeDomain.EVENT_TIME);
    TimerData firstProcTime = TimerData.of(firstWindowNs, firstTimestamp, TimeDomain.PROCESSING_TIME);
    TimerData secondProcTime = TimerData.of(firstWindowNs, secondTimestamp, TimeDomain.PROCESSING_TIME);
    TimerData thirdProcTime = TimerData.of(secondWindowNs, secondTimestamp, TimeDomain.PROCESSING_TIME);
    assertThat(firstEventTime, comparesEqualTo(TimerData.of(firstWindowNs, firstTimestamp, TimeDomain.EVENT_TIME)));
    assertThat(firstEventTime, lessThan(secondEventTime));
    assertThat(secondEventTime, lessThan(thirdEventTime));
    assertThat(firstEventTime, lessThan(thirdEventTime));
    assertThat(secondProcTime, comparesEqualTo(TimerData.of(firstWindowNs, secondTimestamp, TimeDomain.PROCESSING_TIME)));
    assertThat(firstProcTime, lessThan(secondProcTime));
    assertThat(secondProcTime, lessThan(thirdProcTime));
    assertThat(firstProcTime, lessThan(thirdProcTime));
    assertThat(firstEventTime, not(comparesEqualTo(firstProcTime)));
    assertThat(firstProcTime, not(comparesEqualTo(TimerData.of(firstWindowNs, firstTimestamp, TimeDomain.SYNCHRONIZED_PROCESSING_TIME))));
}

Example 2 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class ApexTimerInternalsTest method testSerialization.

@Test
public void testSerialization() {
    TimerDataCoder timerDataCoder = TimerDataCoder.of(GlobalWindow.Coder.INSTANCE);
    TimerData timerData = TimerData.of("arbitrary-id", StateNamespaces.global(), new Instant(0), TimeDomain.EVENT_TIME);
    String key = "key";
    ApexTimerInternals<String> timerInternals = new ApexTimerInternals<>(timerDataCoder);
    timerInternals.setContext(key, StringUtf8Coder.of(), Instant.now(), null);
    timerInternals.setTimer(timerData);
    ApexTimerInternals<String> cloned;
    assertNotNull("Serialization", cloned = KryoCloneUtils.cloneObject(timerInternals));
    cloned.setContext(key, StringUtf8Coder.of(), Instant.now(), null);
    Map<?, Set<Slice>> timers = cloned.getTimerSet(TimeDomain.EVENT_TIME).getMap();
    assertEquals(1, timers.size());
}

Example 3 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class DoFnOperator method fireTimer.

// allow overriding this in WindowDoFnOperator
public void fireTimer(InternalTimer<?, TimerData> timer) {
    TimerInternals.TimerData timerData = timer.getNamespace();
    StateNamespace namespace = timerData.getNamespace();
    // This is a user timer, so namespace must be WindowNamespace
    checkArgument(namespace instanceof WindowNamespace);
    BoundedWindow window = ((WindowNamespace) namespace).getWindow();
    pushbackDoFnRunner.onTimer(timerData.getTimerId(), window, timerData.getTimestamp(), timerData.getDomain());
}

Example 4 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class ReduceFnRunner method onTimers.

public void onTimers(Iterable<TimerData> timers) throws Exception {
    if (!timers.iterator().hasNext()) {
        return;
    }
    // Create a reusable context for each window and begin prefetching necessary
    // state.
    Map<BoundedWindow, WindowActivation> windowActivations = new HashMap();
    for (TimerData timer : timers) {
        checkArgument(timer.getNamespace() instanceof WindowNamespace, "Expected timer to be in WindowNamespace, but was in %s", timer.getNamespace());
        @SuppressWarnings("unchecked") WindowNamespace<W> windowNamespace = (WindowNamespace<W>) timer.getNamespace();
        W window = windowNamespace.getWindow();
        WindowTracing.debug("ReduceFnRunner: Received timer key:{}; window:{}; data:{} with " + "inputWatermark:{}; outputWatermark:{}", key, window, timer, timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
        // that show up too late. Window GC is management by an event time timer
        if (TimeDomain.EVENT_TIME != timer.getDomain() && windowIsExpired(window)) {
            continue;
        }
        // the final pane.
        if (windowActivations.containsKey(window)) {
            continue;
        }
        ReduceFn<K, InputT, OutputT, W>.Context directContext = contextFactory.base(window, StateStyle.DIRECT);
        ReduceFn<K, InputT, OutputT, W>.Context renamedContext = contextFactory.base(window, StateStyle.RENAMED);
        WindowActivation windowActivation = new WindowActivation(directContext, renamedContext);
        windowActivations.put(window, windowActivation);
        // Perform prefetching of state to determine if the trigger should fire.
        if (windowActivation.isGarbageCollection) {
            triggerRunner.prefetchIsClosed(directContext.state());
        } else {
            triggerRunner.prefetchShouldFire(directContext.window(), directContext.state());
        }
    }
    // For those windows that are active and open, prefetch the triggering or emitting state.
    for (WindowActivation timer : windowActivations.values()) {
        if (timer.windowIsActiveAndOpen()) {
            ReduceFn<K, InputT, OutputT, W>.Context directContext = timer.directContext;
            if (timer.isGarbageCollection) {
                prefetchOnTrigger(directContext, timer.renamedContext);
            } else if (triggerRunner.shouldFire(directContext.window(), directContext.timers(), directContext.state())) {
                prefetchEmit(directContext, timer.renamedContext);
            }
        }
    }
    // Perform processing now that everything is prefetched.
    for (WindowActivation windowActivation : windowActivations.values()) {
        ReduceFn<K, InputT, OutputT, W>.Context directContext = windowActivation.directContext;
        ReduceFn<K, InputT, OutputT, W>.Context renamedContext = windowActivation.renamedContext;
        if (windowActivation.isGarbageCollection) {
            WindowTracing.debug("ReduceFnRunner: Cleaning up for key:{}; window:{} with inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
            boolean windowIsActiveAndOpen = windowActivation.windowIsActiveAndOpen();
            if (windowIsActiveAndOpen) {
                // We need to call onTrigger to emit the final pane if required.
                // The final pane *may* be ON_TIME if no prior ON_TIME pane has been emitted,
                // and the watermark has passed the end of the window.
                @Nullable Instant newHold = onTrigger(directContext, renamedContext, true, /* isFinished */
                windowActivation.isEndOfWindow);
                checkState(newHold == null, "Hold placed at %s despite isFinished being true.", newHold);
            }
            // Cleanup flavor B: Clear all the remaining state for this window since we'll never
            // see elements for it again.
            clearAllState(directContext, renamedContext, windowIsActiveAndOpen);
        } else {
            WindowTracing.debug("{}.onTimers: Triggering for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
            if (windowActivation.windowIsActiveAndOpen() && triggerRunner.shouldFire(directContext.window(), directContext.timers(), directContext.state())) {
                emit(directContext, renamedContext);
            }
            if (windowActivation.isEndOfWindow) {
                // If the window strategy trigger includes a watermark trigger then at this point
                // there should be no data holds, either because we'd already cleared them on an
                // earlier onTrigger, or because we just cleared them on the above emit.
                // We could assert this but it is very expensive.
                // Since we are processing an on-time firing we should schedule the garbage collection
                // timer. (If getAllowedLateness is zero then the timer event will be considered a
                // cleanup event and handled by the above).
                // Note we must do this even if the trigger is finished so that we are sure to cleanup
                // any final trigger finished bits.
                checkState(windowingStrategy.getAllowedLateness().isLongerThan(Duration.ZERO), "Unexpected zero getAllowedLateness");
                Instant cleanupTime = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy);
                WindowTracing.debug("ReduceFnRunner.onTimer: Scheduling cleanup timer for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), cleanupTime, timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
                checkState(!cleanupTime.isAfter(BoundedWindow.TIMESTAMP_MAX_VALUE), "Cleanup time %s is beyond end-of-time", cleanupTime);
                directContext.timers().setTimer(cleanupTime, TimeDomain.EVENT_TIME);
            }
        }
    }
}

Example 5 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class ReduceFnTester method advanceProcessingTime.

/**
 * Advance the processing time to the specified time, firing any timers that should fire.
 */
public void advanceProcessingTime(Instant newProcessingTime) throws Exception {
    timerInternals.advanceProcessingTime(newProcessingTime);
    ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner();
    while (true) {
        TimerData timer;
        List<TimerInternals.TimerData> timers = new ArrayList<>();
        while ((timer = timerInternals.removeNextProcessingTimer()) != null) {
            timers.add(timer);
        }
        if (timers.isEmpty()) {
            break;
        }
        runner.onTimers(timers);
    }
    runner.persist();
}