Examples with Instant org.joda.time.Instant used on opensource projects

Example 41 with Instant

use of org.joda.time.Instant in project beam by apache.

the class ReduceFnRunnerTest method testPaneInfoAllStates.

@Test
public void testPaneInfoAllStates() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY);
    tester.advanceInputWatermark(new Instant(0));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 1);
    assertThat(tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, false, Timing.EARLY))));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 2);
    assertThat(tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.EARLY, 1, -1))));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(false);
    tester.advanceInputWatermark(new Instant(15));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 3);
    assertThat(tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.ON_TIME, 2, 0))));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    injectElement(tester, 4);
    assertThat(tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, false, Timing.LATE, 3, 1))));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    injectElement(tester, 5);
    assertThat(tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(false, true, Timing.LATE, 4, 2))));
}

Example 42 with Instant

use of org.joda.time.Instant in project beam by apache.

the class ReduceFnRunnerTest method testPaneInfoSkipToFinish.

@Test
public void testPaneInfoSkipToFinish() throws Exception {
    ReduceFnTester<Integer, Iterable<Integer>, IntervalWindow> tester = ReduceFnTester.nonCombining(FixedWindows.of(Duration.millis(10)), mockTriggerStateMachine, AccumulationMode.DISCARDING_FIRED_PANES, Duration.millis(100), ClosingBehavior.FIRE_IF_NON_EMPTY);
    tester.advanceInputWatermark(new Instant(0));
    when(mockTriggerStateMachine.shouldFire(anyTriggerContext())).thenReturn(true);
    triggerShouldFinish(mockTriggerStateMachine);
    injectElement(tester, 1);
    assertThat(tester.extractOutput(), contains(WindowMatchers.valueWithPaneInfo(PaneInfo.createPane(true, true, Timing.EARLY))));
}

Example 43 with Instant

use of org.joda.time.Instant in project beam by apache.

the class SimpleDoFnRunner method onTimer.

@Override
public void onTimer(String timerId, BoundedWindow window, Instant timestamp, TimeDomain timeDomain) {
    // The effective timestamp is when derived elements will have their timestamp set, if not
    // otherwise specified. If this is an event time timer, then they have the timestamp of the
    // timer itself. Otherwise, they are set to the input timestamp, which is by definition
    // non-late.
    Instant effectiveTimestamp;
    switch(timeDomain) {
        case EVENT_TIME:
            effectiveTimestamp = timestamp;
            break;
        case PROCESSING_TIME:
        case SYNCHRONIZED_PROCESSING_TIME:
            effectiveTimestamp = stepContext.timerInternals().currentInputWatermarkTime();
            break;
        default:
            throw new IllegalArgumentException(String.format("Unknown time domain: %s", timeDomain));
    }
    OnTimerArgumentProvider argumentProvider = new OnTimerArgumentProvider(window, effectiveTimestamp, timeDomain);
    invoker.invokeOnTimer(timerId, argumentProvider);
}

Example 44 with Instant

use of org.joda.time.Instant in project beam by apache.

the class StateMerging method mergeWatermarks.

/**
   * Merge all watermark state in {@code sources} (which must include {@code result} if non-empty)
   * into {@code result}, where the final merge result window is {@code mergeResult}.
   */
public static <W extends BoundedWindow> void mergeWatermarks(Collection<WatermarkHoldState> sources, WatermarkHoldState result, W resultWindow) {
    if (sources.isEmpty()) {
        // Nothing to merge.
        return;
    }
    if (sources.size() == 1 && sources.contains(result) && result.getTimestampCombiner().dependsOnlyOnEarliestTimestamp()) {
        // Nothing to merge.
        return;
    }
    if (result.getTimestampCombiner().dependsOnlyOnWindow()) {
        // Clear sources.
        for (WatermarkHoldState source : sources) {
            source.clear();
        }
        // Update directly from window-derived hold.
        Instant hold = result.getTimestampCombiner().assign(resultWindow, BoundedWindow.TIMESTAMP_MIN_VALUE);
        checkState(hold.isAfter(BoundedWindow.TIMESTAMP_MIN_VALUE));
        result.add(hold);
    } else {
        // Prefetch.
        List<ReadableState<Instant>> futures = new ArrayList<>(sources.size());
        for (WatermarkHoldState source : sources) {
            futures.add(source);
        }
        // Read.
        List<Instant> outputTimesToMerge = new ArrayList<>(sources.size());
        for (ReadableState<Instant> future : futures) {
            Instant sourceOutputTime = future.read();
            if (sourceOutputTime != null) {
                outputTimesToMerge.add(sourceOutputTime);
            }
        }
        // Clear sources.
        for (WatermarkHoldState source : sources) {
            source.clear();
        }
        if (!outputTimesToMerge.isEmpty()) {
            // Merge and update.
            result.add(result.getTimestampCombiner().merge(resultWindow, outputTimesToMerge));
        }
    }
}

Example 45 with Instant

use of org.joda.time.Instant in project beam by apache.

the class ReduceFnRunner method onTimers.

public void onTimers(Iterable<TimerData> timers) throws Exception {
    if (!timers.iterator().hasNext()) {
        return;
    }
    // Create a reusable context for each timer and begin prefetching necessary
    // state.
    List<EnrichedTimerData> enrichedTimers = new LinkedList();
    for (TimerData timer : timers) {
        checkArgument(timer.getNamespace() instanceof WindowNamespace, "Expected timer to be in WindowNamespace, but was in %s", timer.getNamespace());
        @SuppressWarnings("unchecked") WindowNamespace<W> windowNamespace = (WindowNamespace<W>) timer.getNamespace();
        W window = windowNamespace.getWindow();
        ReduceFn<K, InputT, OutputT, W>.Context<K, InputT, OutputT, W> directContext = contextFactory.base(window, StateStyle.DIRECT);
        ReduceFn<K, InputT, OutputT, W>.Context<K, InputT, OutputT, W> renamedContext = contextFactory.base(window, StateStyle.RENAMED);
        EnrichedTimerData enrichedTimer = new EnrichedTimerData(timer, directContext, renamedContext);
        enrichedTimers.add(enrichedTimer);
        // Perform prefetching of state to determine if the trigger should fire.
        if (enrichedTimer.isGarbageCollection) {
            triggerRunner.prefetchIsClosed(directContext.state());
        } else {
            triggerRunner.prefetchShouldFire(directContext.window(), directContext.state());
        }
    }
    // For those windows that are active and open, prefetch the triggering or emitting state.
    for (EnrichedTimerData timer : enrichedTimers) {
        if (timer.windowIsActiveAndOpen()) {
            ReduceFn<K, InputT, OutputT, W>.Context<K, InputT, OutputT, W> directContext = timer.directContext;
            if (timer.isGarbageCollection) {
                prefetchOnTrigger(directContext, timer.renamedContext);
            } else if (triggerRunner.shouldFire(directContext.window(), directContext.timers(), directContext.state())) {
                prefetchEmit(directContext, timer.renamedContext);
            }
        }
    }
    // Perform processing now that everything is prefetched.
    for (EnrichedTimerData timer : enrichedTimers) {
        ReduceFn<K, InputT, OutputT, W>.Context<K, InputT, OutputT, W> directContext = timer.directContext;
        ReduceFn<K, InputT, OutputT, W>.Context<K, InputT, OutputT, W> renamedContext = timer.renamedContext;
        if (timer.isGarbageCollection) {
            WindowTracing.debug("ReduceFnRunner.onTimer: Cleaning up for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timer.timestamp, timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
            boolean windowIsActiveAndOpen = timer.windowIsActiveAndOpen();
            if (windowIsActiveAndOpen) {
                // We need to call onTrigger to emit the final pane if required.
                // The final pane *may* be ON_TIME if no prior ON_TIME pane has been emitted,
                // and the watermark has passed the end of the window.
                @Nullable Instant newHold = onTrigger(directContext, renamedContext, true, /* isFinished */
                timer.isEndOfWindow);
                checkState(newHold == null, "Hold placed at %s despite isFinished being true.", newHold);
            }
            // Cleanup flavor B: Clear all the remaining state for this window since we'll never
            // see elements for it again.
            clearAllState(directContext, renamedContext, windowIsActiveAndOpen);
        } else {
            WindowTracing.debug("ReduceFnRunner.onTimer: Triggering for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), timer.timestamp, timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
            if (timer.windowIsActiveAndOpen() && triggerRunner.shouldFire(directContext.window(), directContext.timers(), directContext.state())) {
                emit(directContext, renamedContext);
            }
            if (timer.isEndOfWindow) {
                // If the window strategy trigger includes a watermark trigger then at this point
                // there should be no data holds, either because we'd already cleared them on an
                // earlier onTrigger, or because we just cleared them on the above emit.
                // We could assert this but it is very expensive.
                // Since we are processing an on-time firing we should schedule the garbage collection
                // timer. (If getAllowedLateness is zero then the timer event will be considered a
                // cleanup event and handled by the above).
                // Note we must do this even if the trigger is finished so that we are sure to cleanup
                // any final trigger finished bits.
                checkState(windowingStrategy.getAllowedLateness().isLongerThan(Duration.ZERO), "Unexpected zero getAllowedLateness");
                Instant cleanupTime = LateDataUtils.garbageCollectionTime(directContext.window(), windowingStrategy);
                WindowTracing.debug("ReduceFnRunner.onTimer: Scheduling cleanup timer for key:{}; window:{} at {} with " + "inputWatermark:{}; outputWatermark:{}", key, directContext.window(), cleanupTime, timerInternals.currentInputWatermarkTime(), timerInternals.currentOutputWatermarkTime());
                checkState(!cleanupTime.isAfter(BoundedWindow.TIMESTAMP_MAX_VALUE), "Cleanup time %s is beyond end-of-time", cleanupTime);
                directContext.timers().setTimer(cleanupTime, TimeDomain.EVENT_TIME);
            }
        }
    }
}