Examples with TimerUpdate org.apache.beam.runners.direct.WatermarkManager.TimerUpdate used on opensource projects

Example 1 with TimerUpdate

use of org.apache.beam.runners.direct.WatermarkManager.TimerUpdate in project beam by apache.

the class WatermarkManagerTest method timerUpdateBuilderWithCompletedAfterBuildNotAddedToBuilt.

@Test
public void timerUpdateBuilderWithCompletedAfterBuildNotAddedToBuilt() {
    TimerUpdateBuilder builder = TimerUpdate.builder(null);
    Instant now = Instant.now();
    TimerData timer = TimerData.of(StateNamespaces.global(), now, now, TimeDomain.EVENT_TIME);
    TimerUpdate built = builder.build();
    builder.withCompletedTimers(ImmutableList.of(timer));
    assertThat(built.getCompletedTimers(), emptyIterable());
    builder.build();
    assertThat(built.getCompletedTimers(), emptyIterable());
}

Example 2 with TimerUpdate

use of org.apache.beam.runners.direct.WatermarkManager.TimerUpdate in project beam by apache.

the class WatermarkManagerTest method extractFiredTimersReturnsFiredSynchronizedProcessingTimeTimers.

@Test
public void extractFiredTimersReturnsFiredSynchronizedProcessingTimeTimers() {
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> initialTimers = manager.extractFiredTimers();
    // Watermarks haven't advanced
    assertThat(initialTimers, emptyIterable());
    // Advance WM of keyed past the first timer, but ahead of the second and third
    CommittedBundle<byte[]> impulseBundle = multiWindowedBundle(filtered);
    manager.updateWatermarks(null, TimerUpdate.empty(), graph.getProducer(impulse), null, Collections.singleton(impulseBundle), new Instant(1500L));
    TimerData earliestTimer = TimerData.of(StateNamespaces.global(), new Instant(999L), new Instant(999L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
    TimerData middleTimer = TimerData.of(StateNamespaces.global(), new Instant(5000L), new Instant(5000L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
    TimerData lastTimer = TimerData.of(StateNamespaces.global(), new Instant(10000L), new Instant(10000L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
    StructuralKey<byte[]> key = StructuralKey.of(new byte[] { 2, -2, 22 }, ByteArrayCoder.of());
    TimerUpdate update = TimerUpdate.builder(key).setTimer(lastTimer).setTimer(earliestTimer).setTimer(middleTimer).build();
    manager.updateWatermarks(impulseBundle, update, graph.getProducer(filtered), impulseBundle.withElements(Collections.emptyList()), Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(impulseToFlatten)), new Instant(1000L));
    manager.refreshAll();
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> firstFiredTimers = manager.extractFiredTimers();
    assertThat(firstFiredTimers, not(emptyIterable()));
    FiredTimers<AppliedPTransform<?, ?, ?>> firstFired = Iterables.getOnlyElement(firstFiredTimers);
    assertThat(firstFired.getTimers(), contains(earliestTimer));
    clock.set(new Instant(50_000L));
    manager.updateWatermarks(null, TimerUpdate.empty(), graph.getProducer(impulse), null, Collections.emptyList(), new Instant(50_000L));
    manager.refreshAll();
    assertTrue(manager.extractFiredTimers().isEmpty());
    manager.updateWatermarks(null, TimerUpdate.builder(key).withCompletedTimers(firstFired.getTimers()).build(), graph.getProducer(filtered), null, Collections.emptyList(), new Instant(1000L));
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> secondFiredTimers = manager.extractFiredTimers();
    assertThat(secondFiredTimers, not(emptyIterable()));
    FiredTimers<AppliedPTransform<?, ?, ?>> secondFired = Iterables.getOnlyElement(secondFiredTimers);
    // Contains, in order, middleTimer and then lastTimer
    assertThat(secondFired.getTimers(), contains(middleTimer, lastTimer));
}

Example 3 with TimerUpdate

use of org.apache.beam.runners.direct.WatermarkManager.TimerUpdate in project beam by apache.

the class WatermarkManagerTest method timerUpdateWithCompletedTimersNotAddedToExisting.

@Test
public void timerUpdateWithCompletedTimersNotAddedToExisting() {
    TimerUpdateBuilder builder = TimerUpdate.builder(null);
    Instant now = Instant.now();
    TimerData timer = TimerData.of(StateNamespaces.global(), now, now, TimeDomain.EVENT_TIME);
    TimerUpdate built = builder.build();
    assertThat(built.getCompletedTimers(), emptyIterable());
    assertThat(built.withCompletedTimers(ImmutableList.of(timer)).getCompletedTimers(), contains(timer));
    assertThat(built.getCompletedTimers(), emptyIterable());
}

Example 4 with TimerUpdate

use of org.apache.beam.runners.direct.WatermarkManager.TimerUpdate in project beam by apache.

the class WatermarkManagerTest method extractFiredTimersReturnsFiredEventTimeTimers.

@Test
public void extractFiredTimersReturnsFiredEventTimeTimers() {
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> initialTimers = manager.extractFiredTimers();
    // Watermarks haven't advanced
    assertThat(initialTimers, emptyIterable());
    // Advance WM of keyed past the first timer, but ahead of the second and third
    CommittedBundle<byte[]> impulseBundle = multiWindowedBundle(filtered);
    manager.updateWatermarks(null, TimerUpdate.empty(), graph.getProducer(impulse), null, Collections.singleton(impulseBundle), new Instant(1500L));
    manager.refreshAll();
    TimerData earliestTimer = TimerData.of(StateNamespaces.global(), new Instant(1000), new Instant(1000), TimeDomain.EVENT_TIME);
    TimerData middleTimer = TimerData.of(StateNamespaces.global(), new Instant(5000L), new Instant(5000L), TimeDomain.EVENT_TIME);
    TimerData lastTimer = TimerData.of(StateNamespaces.global(), new Instant(10000L), new Instant(10000L), TimeDomain.EVENT_TIME);
    StructuralKey<byte[]> key = StructuralKey.of(new byte[] { 1, 4, 9 }, ByteArrayCoder.of());
    TimerUpdate update = TimerUpdate.builder(key).setTimer(earliestTimer).setTimer(middleTimer).setTimer(lastTimer).build();
    manager.updateWatermarks(impulseBundle, update, graph.getProducer(filtered), impulseBundle.withElements(Collections.emptyList()), Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(impulseToFlatten)), new Instant(1000L));
    manager.refreshAll();
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> firstFiredTimers = manager.extractFiredTimers();
    assertThat(firstFiredTimers, not(emptyIterable()));
    FiredTimers<AppliedPTransform<?, ?, ?>> firstFired = Iterables.getOnlyElement(firstFiredTimers);
    assertThat(firstFired.getTimers(), contains(earliestTimer));
    manager.updateWatermarks(null, TimerUpdate.empty(), graph.getProducer(impulse), null, Collections.emptyList(), new Instant(50_000L));
    manager.refreshAll();
    assertTrue(manager.extractFiredTimers().isEmpty());
    // confirm processing of the firstExtracted timers
    manager.updateWatermarks(null, TimerUpdate.builder(key).withCompletedTimers(firstFired.getTimers()).build(), graph.getProducer(filtered), null, Collections.emptyList(), new Instant(1000L));
    manager.refreshAll();
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> secondFiredTimers = manager.extractFiredTimers();
    assertThat(secondFiredTimers, not(emptyIterable()));
    FiredTimers<AppliedPTransform<?, ?, ?>> secondFired = Iterables.getOnlyElement(secondFiredTimers);
    // Contains, in order, middleTimer and then lastTimer
    assertThat(secondFired.getTimers(), contains(middleTimer, lastTimer));
}

Example 5 with TimerUpdate

use of org.apache.beam.runners.direct.WatermarkManager.TimerUpdate in project beam by apache.

the class WatermarkManagerTest method eventTimeTimersCanBeReset.

@Test
public void eventTimeTimersCanBeReset() {
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> initialTimers = manager.extractFiredTimers();
    assertThat(initialTimers, emptyIterable());
    String timerId = "myTimer";
    StructuralKey<?> key = StructuralKey.of(-12L, VarLongCoder.of());
    TimerData initialTimer = TimerData.of(timerId, StateNamespaces.global(), new Instant(1000L), new Instant(1000L), TimeDomain.EVENT_TIME);
    TimerData overridingTimer = TimerData.of(timerId, StateNamespaces.global(), new Instant(2000L), new Instant(2000L), TimeDomain.EVENT_TIME);
    TimerUpdate initialUpdate = TimerUpdate.builder(key).setTimer(initialTimer).build();
    TimerUpdate overridingUpdate = TimerUpdate.builder(key).setTimer(overridingTimer).build();
    CommittedBundle<byte[]> impulseBundle = multiWindowedBundle(filtered);
    manager.updateWatermarks(impulseBundle, initialUpdate, graph.getProducer(filtered), impulseBundle.withElements(Collections.emptyList()), Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(impulseToFlatten)), new Instant(1000L));
    manager.refreshAll();
    // This update should override the previous timer.
    manager.updateWatermarks(impulseBundle, overridingUpdate, graph.getProducer(filtered), impulseBundle.withElements(Collections.emptyList()), Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(impulseToFlatten)), new Instant(1000L));
    manager.refreshAll();
    // Set WM past the timers.
    manager.updateWatermarks(null, TimerUpdate.empty(), graph.getProducer(impulse), null, Collections.singleton(impulseBundle), new Instant(3000L));
    manager.refreshAll();
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> firstFiredTimers = manager.extractFiredTimers();
    assertThat(firstFiredTimers, not(emptyIterable()));
    FiredTimers<AppliedPTransform<?, ?, ?>> firstFired = Iterables.getOnlyElement(firstFiredTimers);
    assertThat(firstFired.getTimers(), contains(overridingTimer));
}