Examples with TimerData org.apache.beam.runners.core.TimerInternals.TimerData used on opensource projects

Example 6 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class WatermarkManagerTest method timerUpdateBuilderWithCompletedAfterBuildNotAddedToBuilt.

@Test
public void timerUpdateBuilderWithCompletedAfterBuildNotAddedToBuilt() {
    TimerUpdateBuilder builder = TimerUpdate.builder(null);
    TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
    TimerUpdate built = builder.build();
    builder.withCompletedTimers(ImmutableList.of(timer));
    assertThat(built.getCompletedTimers(), emptyIterable());
    builder.build();
    assertThat(built.getCompletedTimers(), emptyIterable());
}

Example 7 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class ReduceFnRunner method onTimers.

public void onTimers(Iterable<TimerData> timers) throws Exception {
    if (!timers.iterator().hasNext()) {
        return;
    }
    // Create a reusable context for each timer and begin prefetching necessary
    // state.
    List<EnrichedTimerData> enrichedTimers = new LinkedList();
    for (TimerData timer : timers) {
        checkArgument(timer.getNamespace() instanceof WindowNamespace, "Expected timer to be in WindowNamespace, but was in %s", timer.getNamespace());
        @SuppressWarnings("unchecked") WindowNamespace<W> windowNamespace = (WindowNamespace<W>) timer.getNamespace();
        W window = windowNamespace.getWindow();
        ReduceFn<K, InputT, OutputT, W>.Context<K, InputT, OutputT, W> directContext = contextFactory.base(window, StateStyle.DIRECT);
        ReduceFn<K, InputT, OutputT, W>.Context<K, InputT, OutputT, W> renamedContext = contextFactory.base(window, StateStyle.RENAMED);
        EnrichedTimerData enrichedTimer = new EnrichedTimerData(timer, directContext, renamedContext);
        enrichedTimers.add(enrichedTimer);
        // Perform prefetching of state to determine if the trigger should fire.
        if (enrichedTimer.isGarbageCollection) {
            triggerRunner.prefetchIsClosed(directContext.state());
        } else {
            triggerRunner.prefetchShouldFire(directContext.window(), directContext.state());
        }
    }
    // For those windows that are active and open, prefetch the triggering or emitting state.
    for (EnrichedTimerData timer : enrichedTimers) {
        if (timer.windowIsActiveAndOpen()) {
            ReduceFn<K, InputT, OutputT, W>.Context<K, InputT, OutputT, W> directContext = timer.directContext;
            if (timer.isGarbageCollection) {
                prefetchOnTrigger(directContext, timer.renamedContext);
            } else if (triggerRunner.shouldFire(directContext.window(), directContext.timers(), directContext.state())) {
                prefetchEmit(directContext, timer.renamedContext);
            }
        }
    }
    // Perform processing now that everything is prefetched.
    for (EnrichedTimerData timer : enrichedTimers) {
        ReduceFn<K, InputT, OutputT, W>.Context<K, InputT, OutputT, W> directContext = timer.directContext;
        ReduceFn<K, InputT, OutputT, W>.Context<K, InputT, OutputT, W> renamedContext = timer.renamedContext;
        if (timer.isGarbageCollection) {
            WindowTracing.debug("ReduceFnRunner.onTimer: Cleaning up for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timer.timestamp, timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
            boolean windowIsActiveAndOpen = timer.windowIsActiveAndOpen();
            if (windowIsActiveAndOpen) {
                // We need to call onTrigger to emit the final pane if required.
                // The final pane *may* be ON_TIME if no prior ON_TIME pane has been emitted,
                // and the watermark has passed the end of the window.
                @Nullable Instant newHold = onTrigger(directContext, renamedContext, true, /* isFinished */
                timer.isEndOfWindow);
                checkState(newHold == null, "Hold placed at %s despite isFinished being true.", newHold);
            }
            // Cleanup flavor B: Clear all the remaining state for this window since we'll never
            // see elements for it again.
            clearAllState(directContext, renamedContext, windowIsActiveAndOpen);
        } else {
            WindowTracing.debug("ReduceFnRunner.onTimer: Triggering for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timer.timestamp, timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
            if (timer.windowIsActiveAndOpen() && triggerRunner.shouldFire(directContext.window(), directContext.timers(), directContext.state())) {
                emit(directContext, renamedContext);
            }
            if (timer.isEndOfWindow) {
                // If the window strategy trigger includes a watermark trigger then at this point
                // there should be no data holds, either because we'd already cleared them on an
                // earlier onTrigger, or because we just cleared them on the above emit.
                // We could assert this but it is very expensive.
                // Since we are processing an on-time firing we should schedule the garbage collection
                // timer. (If getAllowedLateness is zero then the timer event will be considered a
                // cleanup event and handled by the above).
                // Note we must do this even if the trigger is finished so that we are sure to cleanup
                // any final trigger finished bits.
                checkState(windowingStrategy.getAllowedLateness().isLongerThan(Duration.ZERO), "Unexpected zero getAllowedLateness");
                Instant cleanupTime = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy);
                WindowTracing.debug("ReduceFnRunner.onTimer: Scheduling cleanup timer for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), cleanupTime, timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
                checkState(!cleanupTime.isAfter(BoundedWindow.TIMESTAMP_MAX_VALUE), "Cleanup time %s is beyond end-of-time", cleanupTime);
                directContext.timers().setTimer(cleanupTime, TimeDomain.EVENT_TIME);
            }
        }
    }
}

Example 8 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class TimerInternalsTest method testCompareTo.

@Test
public void testCompareTo() {
    Instant firstTimestamp = new Instant(100);
    Instant secondTimestamp = new Instant(200);
    IntervalWindow firstWindow = new IntervalWindow(new Instant(0), firstTimestamp);
    IntervalWindow secondWindow = new IntervalWindow(firstTimestamp, secondTimestamp);
    Coder<IntervalWindow> windowCoder = IntervalWindow.getCoder();
    StateNamespace firstWindowNs = StateNamespaces.window(windowCoder, firstWindow);
    StateNamespace secondWindowNs = StateNamespaces.window(windowCoder, secondWindow);
    TimerData firstEventTime = TimerData.of(firstWindowNs, firstTimestamp, TimeDomain.EVENT_TIME);
    TimerData secondEventTime = TimerData.of(firstWindowNs, secondTimestamp, TimeDomain.EVENT_TIME);
    TimerData thirdEventTime = TimerData.of(secondWindowNs, secondTimestamp, TimeDomain.EVENT_TIME);
    TimerData firstProcTime = TimerData.of(firstWindowNs, firstTimestamp, TimeDomain.PROCESSING_TIME);
    TimerData secondProcTime = TimerData.of(firstWindowNs, secondTimestamp, TimeDomain.PROCESSING_TIME);
    TimerData thirdProcTime = TimerData.of(secondWindowNs, secondTimestamp, TimeDomain.PROCESSING_TIME);
    assertThat(firstEventTime, comparesEqualTo(TimerData.of(firstWindowNs, firstTimestamp, TimeDomain.EVENT_TIME)));
    assertThat(firstEventTime, lessThan(secondEventTime));
    assertThat(secondEventTime, lessThan(thirdEventTime));
    assertThat(firstEventTime, lessThan(thirdEventTime));
    assertThat(secondProcTime, comparesEqualTo(TimerData.of(firstWindowNs, secondTimestamp, TimeDomain.PROCESSING_TIME)));
    assertThat(firstProcTime, lessThan(secondProcTime));
    assertThat(secondProcTime, lessThan(thirdProcTime));
    assertThat(firstProcTime, lessThan(thirdProcTime));
    assertThat(firstEventTime, not(comparesEqualTo(firstProcTime)));
    assertThat(firstProcTime, not(comparesEqualTo(TimerData.of(firstWindowNs, firstTimestamp, TimeDomain.SYNCHRONIZED_PROCESSING_TIME))));
}

Example 9 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class ReduceFnTester method advanceProcessingTime.

/** Advance the processing time to the specified time, firing any timers that should fire. */
public void advanceProcessingTime(Instant newProcessingTime) throws Exception {
    timerInternals.advanceProcessingTime(newProcessingTime);
    ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner();
    while (true) {
        TimerData timer;
        List<TimerInternals.TimerData> timers = new ArrayList<>();
        while ((timer = timerInternals.removeNextProcessingTimer()) != null) {
            timers.add(timer);
        }
        if (timers.isEmpty()) {
            break;
        }
        runner.onTimers(timers);
    }
    runner.persist();
}

Example 10 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class ReduceFnTester method advanceSynchronizedProcessingTime.

/**
   * Advance the synchronized processing time to the specified time,
   * firing any timers that should fire.
   */
public void advanceSynchronizedProcessingTime(Instant newSynchronizedProcessingTime) throws Exception {
    timerInternals.advanceSynchronizedProcessingTime(newSynchronizedProcessingTime);
    ReduceFnRunner<String, InputT, OutputT, W> runner = createRunner();
    while (true) {
        TimerData timer;
        List<TimerInternals.TimerData> timers = new ArrayList<>();
        while ((timer = timerInternals.removeNextSynchronizedProcessingTimer()) != null) {
            timers.add(timer);
        }
        if (timers.isEmpty()) {
            break;
        }
        runner.onTimers(timers);
    }
    runner.persist();
}