Examples with FiredTimers org.apache.beam.runners.direct.WatermarkManager.FiredTimers used on opensource projects

Example 1 with FiredTimers

use of org.apache.beam.runners.direct.WatermarkManager.FiredTimers in project beam by apache.

the class WatermarkManagerTest method extractFiredTimersReturnsFiredEventTimeTimers.

@Test
public void extractFiredTimersReturnsFiredEventTimeTimers() {
    Collection<FiredTimers> initialTimers = manager.extractFiredTimers();
    // Watermarks haven't advanced
    assertThat(initialTimers, emptyIterable());
    // Advance WM of keyed past the first timer, but ahead of the second and third
    CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.singleton(createdBundle)), new Instant(1500L));
    manager.refreshAll();
    TimerData earliestTimer = TimerData.of(StateNamespaces.global(), new Instant(1000), TimeDomain.EVENT_TIME);
    TimerData middleTimer = TimerData.of(StateNamespaces.global(), new Instant(5000L), TimeDomain.EVENT_TIME);
    TimerData lastTimer = TimerData.of(StateNamespaces.global(), new Instant(10000L), TimeDomain.EVENT_TIME);
    StructuralKey<byte[]> key = StructuralKey.of(new byte[] { 1, 4, 9 }, ByteArrayCoder.of());
    TimerUpdate update = TimerUpdate.builder(key).setTimer(earliestTimer).setTimer(middleTimer).setTimer(lastTimer).build();
    manager.updateWatermarks(createdBundle, update, result(graph.getProducer(filtered), createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()), Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(intsToFlatten))), new Instant(1000L));
    manager.refreshAll();
    Collection<FiredTimers> firstFiredTimers = manager.extractFiredTimers();
    assertThat(firstFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
    FiredTimers firstFired = Iterables.getOnlyElement(firstFiredTimers);
    assertThat(firstFired.getTimers(), contains(earliestTimer));
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.<CommittedBundle<?>>emptyList()), new Instant(50_000L));
    manager.refreshAll();
    Collection<FiredTimers> secondFiredTimers = manager.extractFiredTimers();
    assertThat(secondFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
    FiredTimers secondFired = Iterables.getOnlyElement(secondFiredTimers);
    // Contains, in order, middleTimer and then lastTimer
    assertThat(secondFired.getTimers(), contains(middleTimer, lastTimer));
}

Example 2 with FiredTimers

use of org.apache.beam.runners.direct.WatermarkManager.FiredTimers in project beam by apache.

the class WatermarkManagerTest method extractFiredTimersReturnsFiredSynchronizedProcessingTimeTimers.

@Test
public void extractFiredTimersReturnsFiredSynchronizedProcessingTimeTimers() {
    Collection<FiredTimers> initialTimers = manager.extractFiredTimers();
    // Watermarks haven't advanced
    assertThat(initialTimers, emptyIterable());
    // Advance WM of keyed past the first timer, but ahead of the second and third
    CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.singleton(createdBundle)), new Instant(1500L));
    TimerData earliestTimer = TimerData.of(StateNamespaces.global(), new Instant(999L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
    TimerData middleTimer = TimerData.of(StateNamespaces.global(), new Instant(5000L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
    TimerData lastTimer = TimerData.of(StateNamespaces.global(), new Instant(10000L), TimeDomain.SYNCHRONIZED_PROCESSING_TIME);
    StructuralKey<byte[]> key = StructuralKey.of(new byte[] { 2, -2, 22 }, ByteArrayCoder.of());
    TimerUpdate update = TimerUpdate.builder(key).setTimer(lastTimer).setTimer(earliestTimer).setTimer(middleTimer).build();
    manager.updateWatermarks(createdBundle, update, result(graph.getProducer(filtered), createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()), Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(intsToFlatten))), new Instant(1000L));
    manager.refreshAll();
    Collection<FiredTimers> firstFiredTimers = manager.extractFiredTimers();
    assertThat(firstFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
    FiredTimers firstFired = Iterables.getOnlyElement(firstFiredTimers);
    assertThat(firstFired.getTimers(), contains(earliestTimer));
    clock.set(new Instant(50_000L));
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.<CommittedBundle<?>>emptyList()), new Instant(50_000L));
    manager.refreshAll();
    Collection<FiredTimers> secondFiredTimers = manager.extractFiredTimers();
    assertThat(secondFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
    FiredTimers secondFired = Iterables.getOnlyElement(secondFiredTimers);
    // Contains, in order, middleTimer and then lastTimer
    assertThat(secondFired.getTimers(), contains(middleTimer, lastTimer));
}

Example 3 with FiredTimers

use of org.apache.beam.runners.direct.WatermarkManager.FiredTimers in project beam by apache.

the class WatermarkManagerTest method getSynchronizedProcessingTimeOutputHeldToPendingTimers.

/**
   * Demonstrates that the Synchronized Processing Time output watermark cannot progress past
   * pending timers in the same set. This propagates to all downstream SynchronizedProcessingTimes.
   *
   * <p>Also demonstrate that the result is monotonic.
   */
//  @Test
public void getSynchronizedProcessingTimeOutputHeldToPendingTimers() {
    CommittedBundle<Integer> createdBundle = multiWindowedBundle(createdInts, 1, 2, 4, 8);
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.<CommittedBundle<?>>singleton(createdBundle)), new Instant(1248L));
    manager.refreshAll();
    TransformWatermarks filteredWms = manager.getWatermarks(graph.getProducer(filtered));
    TransformWatermarks filteredDoubledWms = manager.getWatermarks(graph.getProducer(filteredTimesTwo));
    Instant initialFilteredWm = filteredWms.getSynchronizedProcessingOutputTime();
    Instant initialFilteredDoubledWm = filteredDoubledWms.getSynchronizedProcessingOutputTime();
    StructuralKey<String> key = StructuralKey.of("key", StringUtf8Coder.of());
    CommittedBundle<Integer> filteredBundle = multiWindowedBundle(filtered, 2, 8);
    TimerData pastTimer = TimerData.of(StateNamespaces.global(), new Instant(250L), TimeDomain.PROCESSING_TIME);
    TimerData futureTimer = TimerData.of(StateNamespaces.global(), new Instant(4096L), TimeDomain.PROCESSING_TIME);
    TimerUpdate timers = TimerUpdate.builder(key).setTimer(pastTimer).setTimer(futureTimer).build();
    manager.updateWatermarks(createdBundle, timers, result(graph.getProducer(filtered), createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()), Collections.<CommittedBundle<?>>singleton(filteredBundle)), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    Instant startTime = clock.now();
    clock.set(startTime.plus(250L));
    // We're held based on the past timer
    assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
    assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
    // And we're monotonic
    assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(earlierThan(initialFilteredWm)));
    assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(earlierThan(initialFilteredDoubledWm)));
    Collection<FiredTimers> firedTimers = manager.extractFiredTimers();
    assertThat(Iterables.getOnlyElement(firedTimers).getTimers(), contains(pastTimer));
    // Our timer has fired, but has not been completed, so it holds our synchronized processing WM
    assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
    assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(laterThan(startTime)));
    CommittedBundle<Integer> filteredTimerBundle = bundleFactory.createKeyedBundle(key, filtered).commit(BoundedWindow.TIMESTAMP_MAX_VALUE);
    CommittedBundle<Integer> filteredTimerResult = bundleFactory.createKeyedBundle(key, filteredTimesTwo).commit(filteredWms.getSynchronizedProcessingOutputTime());
    // Complete the processing time timer
    manager.updateWatermarks(filteredTimerBundle, TimerUpdate.builder(key).withCompletedTimers(Collections.<TimerData>singleton(pastTimer)).build(), result(graph.getProducer(filtered), filteredTimerBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()), Collections.<CommittedBundle<?>>singleton(filteredTimerResult)), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    clock.set(startTime.plus(500L));
    assertThat(filteredWms.getSynchronizedProcessingOutputTime(), not(laterThan(clock.now())));
    // filtered should be held to the time at which the filteredTimerResult fired
    assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), not(earlierThan(filteredTimerResult.getSynchronizedProcessingOutputWatermark())));
    manager.updateWatermarks(filteredTimerResult, TimerUpdate.empty(), result(graph.getProducer(filteredTimesTwo), filteredTimerResult.withElements(Collections.<WindowedValue<Integer>>emptyList()), Collections.<CommittedBundle<?>>emptyList()), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
    clock.set(new Instant(Long.MAX_VALUE));
    assertThat(filteredWms.getSynchronizedProcessingOutputTime(), equalTo(new Instant(4096)));
    assertThat(filteredDoubledWms.getSynchronizedProcessingOutputTime(), equalTo(new Instant(4096)));
}

Example 4 with FiredTimers

use of org.apache.beam.runners.direct.WatermarkManager.FiredTimers in project beam by apache.

the class WatermarkManagerTest method extractFiredTimersReturnsFiredProcessingTimeTimers.

@Test
public void extractFiredTimersReturnsFiredProcessingTimeTimers() {
    Collection<FiredTimers> initialTimers = manager.extractFiredTimers();
    // Watermarks haven't advanced
    assertThat(initialTimers, emptyIterable());
    // Advance WM of keyed past the first timer, but ahead of the second and third
    CommittedBundle<Integer> createdBundle = multiWindowedBundle(filtered);
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.singleton(createdBundle)), new Instant(1500L));
    TimerData earliestTimer = TimerData.of(StateNamespaces.global(), new Instant(999L), TimeDomain.PROCESSING_TIME);
    TimerData middleTimer = TimerData.of(StateNamespaces.global(), new Instant(5000L), TimeDomain.PROCESSING_TIME);
    TimerData lastTimer = TimerData.of(StateNamespaces.global(), new Instant(10000L), TimeDomain.PROCESSING_TIME);
    StructuralKey<?> key = StructuralKey.of(-12L, VarLongCoder.of());
    TimerUpdate update = TimerUpdate.builder(key).setTimer(lastTimer).setTimer(earliestTimer).setTimer(middleTimer).build();
    manager.updateWatermarks(createdBundle, update, result(graph.getProducer(filtered), createdBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()), Collections.<CommittedBundle<?>>singleton(multiWindowedBundle(intsToFlatten))), new Instant(1000L));
    manager.refreshAll();
    Collection<FiredTimers> firstFiredTimers = manager.extractFiredTimers();
    assertThat(firstFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
    FiredTimers firstFired = Iterables.getOnlyElement(firstFiredTimers);
    assertThat(firstFired.getTimers(), contains(earliestTimer));
    clock.set(new Instant(50_000L));
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.<CommittedBundle<?>>emptyList()), new Instant(50_000L));
    manager.refreshAll();
    Collection<FiredTimers> secondFiredTimers = manager.extractFiredTimers();
    assertThat(secondFiredTimers, not(Matchers.<FiredTimers>emptyIterable()));
    FiredTimers secondFired = Iterables.getOnlyElement(secondFiredTimers);
    // Contains, in order, middleTimer and then lastTimer
    assertThat(secondFired.getTimers(), contains(middleTimer, lastTimer));
}

Example 5 with FiredTimers

use of org.apache.beam.runners.direct.WatermarkManager.FiredTimers in project beam by apache.

the class EvaluationContextTest method extractFiredTimersExtractsTimers.

@Test
public void extractFiredTimersExtractsTimers() {
    TransformResult<?> holdResult = StepTransformResult.withHold(createdProducer, new Instant(0)).build();
    context.handleResult(null, ImmutableList.<TimerData>of(), holdResult);
    StructuralKey<?> key = StructuralKey.of("foo".length(), VarIntCoder.of());
    TimerData toFire = TimerData.of(StateNamespaces.global(), new Instant(100L), TimeDomain.EVENT_TIME);
    TransformResult<?> timerResult = StepTransformResult.withoutHold(downstreamProducer).withState(CopyOnAccessInMemoryStateInternals.withUnderlying(key, null)).withTimerUpdate(TimerUpdate.builder(key).setTimer(toFire).build()).build();
    // haven't added any timers, must be empty
    assertThat(context.extractFiredTimers(), emptyIterable());
    context.handleResult(context.createKeyedBundle(key, created).commit(Instant.now()), ImmutableList.<TimerData>of(), timerResult);
    // timer hasn't fired
    assertThat(context.extractFiredTimers(), emptyIterable());
    TransformResult<?> advanceResult = StepTransformResult.withoutHold(createdProducer).build();
    // Should cause the downstream timer to fire
    context.handleResult(null, ImmutableList.<TimerData>of(), advanceResult);
    Collection<FiredTimers> fired = context.extractFiredTimers();
    assertThat(Iterables.getOnlyElement(fired).getKey(), Matchers.<StructuralKey<?>>equalTo(key));
    FiredTimers firedForKey = Iterables.getOnlyElement(fired);
    // Contains exclusively the fired timer
    assertThat(firedForKey.getTimers(), contains(toFire));
    // Don't reextract timers
    assertThat(context.extractFiredTimers(), emptyIterable());
}