Examples with TimerUpdate org.apache.beam.runners.direct.WatermarkManager.TimerUpdate used on opensource projects

Example 6 with TimerUpdate

use of org.apache.beam.runners.direct.WatermarkManager.TimerUpdate in project beam by apache.

the class WatermarkManagerTest method timerUpdateBuilderWithSetAfterBuildNotAddedToBuilt.

@Test
public void timerUpdateBuilderWithSetAfterBuildNotAddedToBuilt() {
    TimerUpdateBuilder builder = TimerUpdate.builder(null);
    Instant now = Instant.now();
    TimerData timer = TimerData.of(StateNamespaces.global(), now, now, TimeDomain.EVENT_TIME);
    TimerUpdate built = builder.build();
    builder.setTimer(timer);
    assertThat(built.getSetTimers(), emptyIterable());
    builder.build();
    assertThat(built.getSetTimers(), emptyIterable());
}

Example 7 with TimerUpdate

use of org.apache.beam.runners.direct.WatermarkManager.TimerUpdate in project beam by apache.

the class WatermarkManagerTest method processingTimeTimersCanBeReset.

@Test
public void processingTimeTimersCanBeReset() {
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> initialTimers = manager.extractFiredTimers();
    assertThat(initialTimers, emptyIterable());
    String timerId = "myTimer";
    StructuralKey<?> key = StructuralKey.of(-12L, VarLongCoder.of());
    TimerData initialTimer = TimerData.of(timerId, StateNamespaces.global(), new Instant(5000L), new Instant(5000L), TimeDomain.PROCESSING_TIME);
    TimerData overridingTimer = TimerData.of(timerId, StateNamespaces.global(), new Instant(10000L), new Instant(10000L), TimeDomain.PROCESSING_TIME);
    TimerUpdate initialUpdate = TimerUpdate.builder(key).setTimer(initialTimer).build();
    TimerUpdate overridingUpdate = TimerUpdate.builder(key).setTimer(overridingTimer).build();
    manager.updateWatermarks(null, initialUpdate, graph.getProducer(impulse), null, Collections.emptyList(), new Instant(5000L));
    manager.refreshAll();
    // This update should override the previous timer.
    manager.updateWatermarks(null, overridingUpdate, graph.getProducer(impulse), null, Collections.emptyList(), new Instant(10000L));
    // Set clock past the timers.
    clock.set(new Instant(50000L));
    manager.refreshAll();
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> firedTimers = manager.extractFiredTimers();
    assertThat(firedTimers, not(emptyIterable()));
    FiredTimers<AppliedPTransform<?, ?, ?>> timers = Iterables.getOnlyElement(firedTimers);
    assertThat(timers.getTimers(), contains(overridingTimer));
}

Example 8 with TimerUpdate

use of org.apache.beam.runners.direct.WatermarkManager.TimerUpdate in project beam by apache.

the class WatermarkManagerTest method getSynchronizedProcessingTimeOutputHeldToPendingTimers.

/**
 * Demonstrates that the Synchronized Processing Time output watermark cannot progress past
 * pending timers in the same set. This propagates to all downstream SynchronizedProcessingTimes.
 *
 * <p>Also demonstrate that the result is monotonic.
 */
@Test
public void getSynchronizedProcessingTimeOutputHeldToPendingTimers() {
    CommittedBundle<byte[]> impulseBundle = multiWindowedBundle(impulse, new byte[1], new byte[2], new byte[4], new byte[8]);
    manager.updateWatermarks(null, TimerUpdate.empty(), graph.getProducer(impulse), null, Collections.<CommittedBundle<?>>singleton(impulseBundle), new Instant(1248L));
    manager.refreshAll();
    TransformWatermarks filteredWms = manager.getWatermarks(graph.getProducer(filtered));
    TransformWatermarks filteredDoubledWms = manager.getWatermarks(graph.getProducer(filteredNotEmpty));
    Instant initialFilteredWm = filteredWms.getSynchronizedProcessingOutputTime();
    Instant initialFilteredDoubledWm = filteredDoubledWms.getSynchronizedProcessingOutputTime();
    StructuralKey<String> key = StructuralKey.of("key", StringUtf8Coder.of());
    CommittedBundle<byte[]> filteredBundle = multiWindowedBundle(filtered, new byte[2], new byte[8]);
    TimerData pastTimer = TimerData.of(StateNamespaces.global(), new Instant(250L), new Instant(250L), TimeDomain.PROCESSING_TIME);
    TimerData futureTimer = TimerData.of(StateNamespaces.global(), new Instant(4096L), new Instant(4096L), TimeDomain.PROCESSING_TIME);
    TimerUpdate timers = TimerUpdate.builder(key).setTimer(pastTimer).setTimer(futureTimer).build();
    manager.updateWatermarks(impulseBundle, timers, graph.getProducer(filtered), impulseBundle.withElements(Collections.emptyList()), Collections.<CommittedBundle<?>>singleton(filteredBundle), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    Instant startTime = clock.now();
    clock.set(startTime.plus(Duration.millis(250L)));
    // We're held based on the past timer
    assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(greaterThan(startTime)));
    assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(greaterThan(startTime)));
    // And we're monotonic
    assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(lessThan(initialFilteredWm)));
    assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(lessThan(initialFilteredDoubledWm)));
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> firedTimers = manager.extractFiredTimers();
    assertThat(Iterables.getOnlyElement(firedTimers).getTimers(), contains(pastTimer));
    // Our timer has fired, but has not been completed, so it holds our synchronized processing WM
    assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(greaterThan(startTime)));
    assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(greaterThan(startTime)));
    CommittedBundle<byte[]> filteredTimerBundle = bundleFactory.createKeyedBundle(key, filtered).commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
    CommittedBundle<byte[]> filteredTimerResult = bundleFactory.createKeyedBundle(key, filteredNotEmpty).commit(filteredWms.getSynchronizedProcessingOutputTime());
    // Complete the processing time timer
    manager.updateWatermarks(filteredTimerBundle, TimerUpdate.builder(key).withCompletedTimers(Collections.singleton(pastTimer)).build(), graph.getProducer(filtered), filteredTimerBundle.withElements(Collections.emptyList()), Collections.<CommittedBundle<?>>singleton(filteredTimerResult), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    clock.set(startTime.plus(Duration.millis(500L)));
    assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(greaterThan(clock.now())));
    // filtered should be held to the time at which the filteredTimerResult fired
    assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(lessThan(filteredTimerResult.getSynchronizedProcessingOutputWatermark())));
    manager.updateWatermarks(filteredTimerResult, TimerUpdate.empty(), graph.getProducer(filteredNotEmpty), filteredTimerResult.withElements(Collections.emptyList()), Collections.emptyList(), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    clock.set(new Instant(Long.MAX_VALUE));
    assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(greaterThan(new Instant(4096))));
}

Example 9 with TimerUpdate

use of org.apache.beam.runners.direct.WatermarkManager.TimerUpdate in project beam by apache.

the class WatermarkManagerTest method extractFiredTimersReturnsFiredProcessingTimeTimers.

@Test
public void extractFiredTimersReturnsFiredProcessingTimeTimers() {
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> initialTimers = manager.extractFiredTimers();
    // Watermarks haven't advanced
    assertThat(initialTimers, emptyIterable());
    // Advance WM of keyed past the first timer, but ahead of the second and third
    CommittedBundle<byte[]> impulseBundle = multiWindowedBundle(filtered);
    manager.updateWatermarks(null, TimerUpdate.empty(), graph.getProducer(impulse), null, Collections.singleton(impulseBundle), new Instant(1500L));
    TimerData earliestTimer = TimerData.of(StateNamespaces.global(), new Instant(999L), new Instant(999L), TimeDomain.PROCESSING_TIME);
    TimerData middleTimer = TimerData.of(StateNamespaces.global(), new Instant(5000L), new Instant(5000L), TimeDomain.PROCESSING_TIME);
    TimerData lastTimer = TimerData.of(StateNamespaces.global(), new Instant(10000L), new Instant(10000L), TimeDomain.PROCESSING_TIME);
    StructuralKey<?> key = StructuralKey.of(-12L, VarLongCoder.of());
    TimerUpdate update = TimerUpdate.builder(key).setTimer(lastTimer).setTimer(earliestTimer).setTimer(middleTimer).build();
    manager.updateWatermarks(impulseBundle, update, graph.getProducer(filtered), impulseBundle.withElements(Collections.emptyList()), Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(impulseToFlatten)), new Instant(1000L));
    manager.refreshAll();
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> firstFiredTimers = manager.extractFiredTimers();
    assertThat(firstFiredTimers, not(emptyIterable()));
    FiredTimers<AppliedPTransform<?, ?, ?>> firstFired = Iterables.getOnlyElement(firstFiredTimers);
    assertThat(firstFired.getTimers(), contains(earliestTimer));
    clock.set(new Instant(50_000L));
    manager.updateWatermarks(null, TimerUpdate.empty(), graph.getProducer(impulse), null, Collections.emptyList(), new Instant(50_000L));
    manager.refreshAll();
    assertTrue(manager.extractFiredTimers().isEmpty());
    manager.updateWatermarks(null, TimerUpdate.builder(key).withCompletedTimers(firstFired.getTimers()).build(), graph.getProducer(filtered), null, Collections.emptyList(), new Instant(1000L));
    manager.refreshAll();
    Collection<FiredTimers<AppliedPTransform<?, ?, ?>>> secondFiredTimers = manager.extractFiredTimers();
    assertThat(secondFiredTimers, not(emptyIterable()));
    FiredTimers<AppliedPTransform<?, ?, ?>> secondFired = Iterables.getOnlyElement(secondFiredTimers);
    // Contains, in order, middleTimer and then lastTimer
    assertThat(secondFired.getTimers(), contains(middleTimer, lastTimer));
}

Example 10 with TimerUpdate

use of org.apache.beam.runners.direct.WatermarkManager.TimerUpdate in project beam by apache.

the class WatermarkManagerTest method timerUpdateBuilderWithSetThenDeleteHasOnlyDeleted.

@Test
public void timerUpdateBuilderWithSetThenDeleteHasOnlyDeleted() {
    TimerUpdateBuilder builder = TimerUpdate.builder(null);
    Instant now = Instant.now();
    TimerData timer = TimerData.of(StateNamespaces.global(), now, now, TimeDomain.EVENT_TIME);
    TimerUpdate built = builder.setTimer(timer).deletedTimer(timer).build();
    assertThat(built.getSetTimers(), emptyIterable());
    assertThat(built.getDeletedTimers(), contains(timer));
}