Search in sources :

Example 91 with Instant

use of org.joda.time.Instant in project beam by apache.

the class AfterEachStateMachineTest method testAfterEachInSequence.

/**
   * Tests that the {@link AfterEachStateMachine} trigger fires and finishes the first trigger then
   * the second.
   */
@Test
public void testAfterEachInSequence() throws Exception {
    tester = TriggerStateMachineTester.forTrigger(AfterEachStateMachine.inOrder(RepeatedlyStateMachine.forever(AfterPaneStateMachine.elementCountAtLeast(2)).orFinally(AfterPaneStateMachine.elementCountAtLeast(3)), RepeatedlyStateMachine.forever(AfterPaneStateMachine.elementCountAtLeast(5)).orFinally(AfterWatermarkStateMachine.pastEndOfWindow())), FixedWindows.of(Duration.millis(10)));
    IntervalWindow window = new IntervalWindow(new Instant(0), new Instant(10));
    // AfterCount(2) not ready
    tester.injectElements(1);
    assertFalse(tester.shouldFire(window));
    // AfterCount(2) ready, not finished
    tester.injectElements(2);
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertFalse(tester.isMarkedFinished(window));
    // orFinally(AfterCount(3)) ready and will finish the first
    tester.injectElements(1, 2, 3);
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertFalse(tester.isMarkedFinished(window));
    // Now running as the second trigger
    assertFalse(tester.shouldFire(window));
    // This quantity of elements would fire and finish if it were erroneously still the first
    tester.injectElements(1, 2, 3, 4);
    assertFalse(tester.shouldFire(window));
    // Now fire once
    tester.injectElements(5);
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertFalse(tester.isMarkedFinished(window));
    // This time advance the watermark to finish the whole mess.
    tester.advanceInputWatermark(new Instant(10));
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertTrue(tester.isMarkedFinished(window));
}
Also used : Instant(org.joda.time.Instant) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Example 92 with Instant

use of org.joda.time.Instant in project beam by apache.

the class AfterFirstStateMachineTest method testOnlyT1ShouldFireFixedWindows.

@Test
public void testOnlyT1ShouldFireFixedWindows() throws Exception {
    tester = TriggerStateMachineTester.forTrigger(AfterFirstStateMachine.of(mockTrigger1, mockTrigger2), FixedWindows.of(Duration.millis(10)));
    tester.injectElements(1);
    IntervalWindow window = new IntervalWindow(new Instant(1), new Instant(11));
    when(mockTrigger1.shouldFire(anyTriggerContext())).thenReturn(true);
    when(mockTrigger2.shouldFire(anyTriggerContext())).thenReturn(false);
    // should fire
    assertTrue(tester.shouldFire(window));
    tester.fireIfShouldFire(window);
    assertTrue(tester.isMarkedFinished(window));
}
Also used : Instant(org.joda.time.Instant) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Example 93 with Instant

use of org.joda.time.Instant in project beam by apache.

the class AfterPaneStateMachineTest method testAfterPaneElementCountSessions.

@Test
public void testAfterPaneElementCountSessions() throws Exception {
    tester = TriggerStateMachineTester.forTrigger(AfterPaneStateMachine.elementCountAtLeast(2), Sessions.withGapDuration(Duration.millis(10)));
    tester.injectElements(// in [1, 11)
    1, // in [2, 12)
    2);
    assertFalse(tester.shouldFire(new IntervalWindow(new Instant(1), new Instant(11))));
    assertFalse(tester.shouldFire(new IntervalWindow(new Instant(2), new Instant(12))));
    tester.mergeWindows();
    IntervalWindow mergedWindow = new IntervalWindow(new Instant(1), new Instant(12));
    assertTrue(tester.shouldFire(mergedWindow));
    tester.fireIfShouldFire(mergedWindow);
    assertTrue(tester.isMarkedFinished(mergedWindow));
    // Because we closed the previous window, we don't have it around to merge with. So there
    // will be a new FIRE_AND_FINISH result.
    tester.injectElements(// in [7, 17)
    7, // in [9, 19)
    9);
    tester.mergeWindows();
    IntervalWindow newMergedWindow = new IntervalWindow(new Instant(7), new Instant(19));
    assertTrue(tester.shouldFire(newMergedWindow));
    tester.fireIfShouldFire(newMergedWindow);
    assertTrue(tester.isMarkedFinished(newMergedWindow));
}
Also used : Instant(org.joda.time.Instant) IntervalWindow(org.apache.beam.sdk.transforms.windowing.IntervalWindow) Test(org.junit.Test)

Example 94 with Instant

use of org.joda.time.Instant in project beam by apache.

the class OrFinallyTrigger method getWatermarkThatGuaranteesFiring.

@Override
public Instant getWatermarkThatGuaranteesFiring(BoundedWindow window) {
    // This trigger fires once either the trigger or the until trigger fires.
    Instant actualDeadline = subTriggers.get(ACTUAL).getWatermarkThatGuaranteesFiring(window);
    Instant untilDeadline = subTriggers.get(UNTIL).getWatermarkThatGuaranteesFiring(window);
    return actualDeadline.isBefore(untilDeadline) ? actualDeadline : untilDeadline;
}
Also used : Instant(org.joda.time.Instant)

Example 95 with Instant

use of org.joda.time.Instant in project beam by apache.

the class WindowFnTestUtils method validateNonInterferingOutputTimes.

/**
   * Assigns the given {@code timestamp} to windows using the specified {@code windowFn}, and
   * verifies that result of {@code windowFn.getOutputTimestamp} for each window is within the
   * proper bound.
   */
public static <T, W extends BoundedWindow> void validateNonInterferingOutputTimes(WindowFn<T, W> windowFn, long timestamp) throws Exception {
    Collection<W> windows = WindowFnTestUtils.<T, W>assignedWindows(windowFn, timestamp);
    Instant instant = new Instant(timestamp);
    for (W window : windows) {
        Instant outputTimestamp = windowFn.getOutputTime(instant, window);
        assertFalse("getOutputTime must be greater than or equal to input timestamp", outputTimestamp.isBefore(instant));
        assertFalse("getOutputTime must be less than or equal to the max timestamp", outputTimestamp.isAfter(window.maxTimestamp()));
    }
}
Also used : ReadableInstant(org.joda.time.ReadableInstant) Instant(org.joda.time.Instant)

Aggregations

Instant (org.joda.time.Instant)411 Test (org.junit.Test)326 IntervalWindow (org.apache.beam.sdk.transforms.windowing.IntervalWindow)135 KV (org.apache.beam.sdk.values.KV)64 Category (org.junit.experimental.categories.Category)60 WindowedValue (org.apache.beam.sdk.util.WindowedValue)47 Duration (org.joda.time.Duration)44 ReadableInstant (org.joda.time.ReadableInstant)36 BoundedWindow (org.apache.beam.sdk.transforms.windowing.BoundedWindow)30 WatermarkHoldState (org.apache.beam.sdk.state.WatermarkHoldState)24 TimerData (org.apache.beam.runners.core.TimerInternals.TimerData)22 Matchers.emptyIterable (org.hamcrest.Matchers.emptyIterable)22 HashMap (java.util.HashMap)19 TransformWatermarks (org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks)18 StateNamespaceForTest (org.apache.beam.runners.core.StateNamespaceForTest)17 WindowMatchers.isSingleWindowedValue (org.apache.beam.runners.core.WindowMatchers.isSingleWindowedValue)17 WindowMatchers.isWindowedValue (org.apache.beam.runners.core.WindowMatchers.isWindowedValue)17 TupleTag (org.apache.beam.sdk.values.TupleTag)14 ArrayList (java.util.ArrayList)13 Map (java.util.Map)12