Search in sources :

Example 96 with Instant

use of org.joda.time.Instant in project beam by apache.

the class JmsCheckpointMark method finalizeCheckpoint.

/**
   * Acknowledge all outstanding message. Since we believe that messages will be delivered in
   * timestamp order, and acknowledged messages will not be retried, the newest message in this
   * batch is a good bound for future messages.
   */
@Override
public void finalizeCheckpoint() {
    for (Message message : messages) {
        try {
            message.acknowledge();
            Instant currentMessageTimestamp = new Instant(message.getJMSTimestamp());
            if (currentMessageTimestamp.isAfter(oldestPendingTimestamp)) {
                oldestPendingTimestamp = currentMessageTimestamp;
            }
        } catch (Exception e) {
        // nothing to do
        }
    }
    messages.clear();
}
Also used : Message(javax.jms.Message) Instant(org.joda.time.Instant)

Example 97 with Instant

use of org.joda.time.Instant in project beam by apache.

the class TriggerStateMachineTester method injectElements.

public final void injectElements(Collection<TimestampedValue<InputT>> values) throws Exception {
    for (TimestampedValue<InputT> value : values) {
        WindowTracing.trace("TriggerTester.injectElements: {}", value);
    }
    List<WindowedValue<InputT>> windowedValues = Lists.newArrayListWithCapacity(values.size());
    for (TimestampedValue<InputT> input : values) {
        try {
            InputT value = input.getValue();
            Instant timestamp = input.getTimestamp();
            Collection<W> assignedWindows = windowFn.assignWindows(new TestAssignContext<W>(windowFn, value, timestamp, GlobalWindow.INSTANCE));
            for (W window : assignedWindows) {
                activeWindows.addActiveForTesting(window);
                // Today, triggers assume onTimer firing at the watermark time, whether or not they
                // explicitly set the timer themselves. So this tester must set it.
                timerInternals.setTimer(TimerData.of(windowNamespace(window), window.maxTimestamp(), TimeDomain.EVENT_TIME));
            }
            windowedValues.add(WindowedValue.of(value, timestamp, assignedWindows, PaneInfo.NO_FIRING));
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    for (WindowedValue<InputT> windowedValue : windowedValues) {
        for (BoundedWindow untypedWindow : windowedValue.getWindows()) {
            // SDK is responsible for type safety
            @SuppressWarnings("unchecked") W window = mergeResult((W) untypedWindow);
            TriggerStateMachine.OnElementContext context = contextFactory.createOnElementContext(window, new TestTimers(windowNamespace(window)), windowedValue.getTimestamp(), executableTrigger, getFinishedSet(window));
            if (!context.trigger().isFinished()) {
                executableTrigger.invokeOnElement(context);
            }
        }
    }
}
Also used : Instant(org.joda.time.Instant) WindowedValue(org.apache.beam.sdk.util.WindowedValue) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow)

Example 98 with Instant

use of org.joda.time.Instant in project beam by apache.

the class AfterWatermarkStateMachineTest method testEarlyAndAtWatermark.

@Test
public void testEarlyAndAtWatermark() throws Exception {
    tester = TriggerStateMachineTester.forTrigger(AfterWatermarkStateMachine.pastEndOfWindow().withEarlyFirings(mockEarly), FixedWindows.of(Duration.millis(100)));
    injectElements(1);
    IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(100));
    testRunningAsTrigger(mockEarly, window);
    // Fire due to watermark
    when(mockEarly.shouldFire(anyTriggerContext())).thenReturn(false);
    tester.advanceInputWatermark(new Instant(100));
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertTrue(tester.isMarkedFinished(window));
}
Also used : Instant(org.joda.time.Instant) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Example 99 with Instant

use of org.joda.time.Instant in project beam by apache.

the class OrFinallyStateMachineTest method testActualFiresButUntilFinishes.

/**
   * Tests that for {@code OrFinally(actual, until)} when {@code actual}
   * fires but does not finish, then {@code until} fires and finishes, the
   * whole thing fires and finished.
   */
@Test
public void testActualFiresButUntilFinishes() throws Exception {
    tester = TriggerStateMachineTester.forTrigger(new OrFinallyStateMachine(RepeatedlyStateMachine.forever(AfterPaneStateMachine.elementCountAtLeast(2)), AfterPaneStateMachine.elementCountAtLeast(3)), FixedWindows.of(Duration.millis(10)));
    IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10));
    // Before any firing
    tester.injectElements(1);
    assertFalse(tester.shouldFire(window));
    assertFalse(tester.isMarkedFinished(window));
    // The actual fires but doesn't finish
    tester.injectElements(2);
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertFalse(tester.isMarkedFinished(window));
    // The until fires and finishes; the trigger is finished
    tester.injectElements(3);
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertTrue(tester.isMarkedFinished(window));
}
Also used : Instant(org.joda.time.Instant) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Example 100 with Instant

use of org.joda.time.Instant in project beam by apache.

the class OrFinallyStateMachineTest method testShouldFireAfterMerge.

/**
   * Tests that if the first trigger rewinds to be non-finished in the merged window,
   * then it becomes the currently active trigger again, with real triggers.
   */
@Test
public void testShouldFireAfterMerge() throws Exception {
    tester = TriggerStateMachineTester.forTrigger(AfterEachStateMachine.inOrder(AfterPaneStateMachine.elementCountAtLeast(5).orFinally(AfterWatermarkStateMachine.pastEndOfWindow()), RepeatedlyStateMachine.forever(AfterPaneStateMachine.elementCountAtLeast(1))), Sessions.withGapDuration(Duration.millis(10)));
    // Finished the orFinally in the first window
    tester.injectElements(1);
    IntervalWindow firstWindow = new IntervalWindow(new Instant(1), new Instant(11));
    assertFalse(tester.shouldFire(firstWindow));
    tester.advanceInputWatermark(new Instant(11));
    assertTrue(tester.shouldFire(firstWindow));
    tester.fireIfShouldFire(firstWindow);
    // Set up second window where it is not done
    tester.injectElements(5);
    IntervalWindow secondWindow = new IntervalWindow(new Instant(5), new Instant(15));
    assertFalse(tester.shouldFire(secondWindow));
    // Merge them, if the merged window were on the second trigger, it would be ready
    tester.mergeWindows();
    IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(15));
    assertFalse(tester.shouldFire(mergedWindow));
    // Now adding 3 more makes the main trigger ready to fire
    tester.injectElements(1, 2, 3, 4, 5);
    tester.mergeWindows();
    assertTrue(tester.shouldFire(mergedWindow));
}
Also used : Instant(org.joda.time.Instant) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Aggregations

Instant (org.joda.time.Instant)410 Test (org.junit.Test)326 IntervalWindow (org.apache.beam.sdk.transforms.windowing.IntervalWindow)135 KV (org.apache.beam.sdk.values.KV)64 Category (org.junit.experimental.categories.Category)60 WindowedValue (org.apache.beam.sdk.util.WindowedValue)47 Duration (org.joda.time.Duration)44 ReadableInstant (org.joda.time.ReadableInstant)36 BoundedWindow (org.apache.beam.sdk.transforms.windowing.BoundedWindow)30 WatermarkHoldState (org.apache.beam.sdk.state.WatermarkHoldState)24 TimerData (org.apache.beam.runners.core.TimerInternals.TimerData)22 Matchers.emptyIterable (org.hamcrest.Matchers.emptyIterable)22 HashMap (java.util.HashMap)19 TransformWatermarks (org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks)18 StateNamespaceForTest (org.apache.beam.runners.core.StateNamespaceForTest)17 WindowMatchers.isSingleWindowedValue (org.apache.beam.runners.core.WindowMatchers.isSingleWindowedValue)17 WindowMatchers.isWindowedValue (org.apache.beam.runners.core.WindowMatchers.isWindowedValue)17 TupleTag (org.apache.beam.sdk.values.TupleTag)14 ArrayList (java.util.ArrayList)13 Map (java.util.Map)12