Search in sources :

Example 1 with AppliedPTransformInputWatermark

use of org.apache.beam.runners.direct.WatermarkManager.AppliedPTransformInputWatermark in project beam by apache.

the class WatermarkManagerTest method inputWatermarkDuplicates.

@Test
public void inputWatermarkDuplicates() {
    Watermark mockWatermark = Mockito.mock(Watermark.class);
    AppliedPTransformInputWatermark underTest = new AppliedPTransformInputWatermark("underTest", ImmutableList.of(mockWatermark), update -> {
    });
    // Refresh
    when(mockWatermark.get()).thenReturn(new Instant(0));
    underTest.refresh();
    assertEquals(new Instant(0), underTest.get());
    // Apply a timer update
    StructuralKey<String> key = StructuralKey.of("key", StringUtf8Coder.of());
    TimerData timer1 = TimerData.of("a", StateNamespaces.global(), new Instant(100), new Instant(100), TimeDomain.EVENT_TIME);
    TimerData timer2 = TimerData.of("a", StateNamespaces.global(), new Instant(200), new Instant(200), TimeDomain.EVENT_TIME);
    underTest.updateTimers(TimerUpdate.builder(key).setTimer(timer1).setTimer(timer2).build());
    // Only the last timer update should be observable
    assertEquals(timer2.getTimestamp(), underTest.getEarliestTimerTimestamp());
    // Advance the input watermark
    when(mockWatermark.get()).thenReturn(new Instant(1000));
    underTest.refresh();
    // input watermark is not held by timers
    assertEquals(new Instant(1000), underTest.get());
    // Examine the fired event time timers
    Map<StructuralKey<?>, List<TimerData>> fired = underTest.extractFiredEventTimeTimers();
    List<TimerData> timers = fired.get(key);
    assertNotNull(timers);
    assertThat(timers, contains(timer2));
    // Update based on timer firings
    underTest.updateTimers(TimerUpdate.builder(key).withCompletedTimers(timers).build());
    // Now we should be able to advance
    assertEquals(BoundedWindow.TIMESTAMP_MAX_VALUE, underTest.getEarliestTimerTimestamp());
    // Nothing left to fire
    fired = underTest.extractFiredEventTimeTimers();
    assertThat(fired.entrySet(), empty());
}
Also used : AppliedPTransformInputWatermark(org.apache.beam.runners.direct.WatermarkManager.AppliedPTransformInputWatermark) Instant(org.joda.time.Instant) TimerData(org.apache.beam.runners.core.TimerInternals.TimerData) PCollectionList(org.apache.beam.sdk.values.PCollectionList) List(java.util.List) ImmutableList(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList) Watermark(org.apache.beam.runners.direct.WatermarkManager.Watermark) AppliedPTransformInputWatermark(org.apache.beam.runners.direct.WatermarkManager.AppliedPTransformInputWatermark) StructuralKey(org.apache.beam.runners.local.StructuralKey) Test(org.junit.Test)

Aggregations

List (java.util.List)1 TimerData (org.apache.beam.runners.core.TimerInternals.TimerData)1 AppliedPTransformInputWatermark (org.apache.beam.runners.direct.WatermarkManager.AppliedPTransformInputWatermark)1 Watermark (org.apache.beam.runners.direct.WatermarkManager.Watermark)1 StructuralKey (org.apache.beam.runners.local.StructuralKey)1 PCollectionList (org.apache.beam.sdk.values.PCollectionList)1 ImmutableList (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList)1 Instant (org.joda.time.Instant)1 Test (org.junit.Test)1