Examples with TransformWatermarks org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks used on opensource projects

Example 11 with TransformWatermarks

use of org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks in project beam by apache.

the class WatermarkManagerTest method getSynchronizedProcessingTimeInputWatermarksHeldToPendingBundles.

@Test
public void getSynchronizedProcessingTimeInputWatermarksHeldToPendingBundles() {
    TransformWatermarks watermarks = manager.getWatermarks(graph.getProducer(createdInts));
    assertThat(watermarks.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
    assertThat(watermarks.getSynchronizedProcessingOutputTime(), equalTo(BoundedWindow.TIMESTAMP_MIN_VALUE));
    TransformWatermarks filteredWatermarks = manager.getWatermarks(graph.getProducer(filtered));
    // Non-root processing watermarks don't progress until data has been processed
    assertThat(filteredWatermarks.getSynchronizedProcessingInputTime(), not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
    assertThat(filteredWatermarks.getSynchronizedProcessingOutputTime(), not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
    CommittedBundle<Integer> createOutput = bundleFactory.createBundle(createdInts).commit(new Instant(1250L));
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.<CommittedBundle<?>>singleton(createOutput)), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    TransformWatermarks createAfterUpdate = manager.getWatermarks(graph.getProducer(createdInts));
    assertThat(createAfterUpdate.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
    assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
    TransformWatermarks filterAfterProduced = manager.getWatermarks(graph.getProducer(filtered));
    assertThat(filterAfterProduced.getSynchronizedProcessingInputTime(), not(laterThan(clock.now())));
    assertThat(filterAfterProduced.getSynchronizedProcessingOutputTime(), not(laterThan(clock.now())));
    clock.set(new Instant(1500L));
    assertThat(createAfterUpdate.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
    assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
    assertThat(filterAfterProduced.getSynchronizedProcessingInputTime(), not(laterThan(new Instant(1250L))));
    assertThat(filterAfterProduced.getSynchronizedProcessingOutputTime(), not(laterThan(new Instant(1250L))));
    CommittedBundle<?> filterOutputBundle = bundleFactory.createBundle(intsToFlatten).commit(new Instant(1250L));
    manager.updateWatermarks(createOutput, TimerUpdate.empty(), result(graph.getProducer(filtered), createOutput.withElements(Collections.<WindowedValue<Integer>>emptyList()), Collections.<CommittedBundle<?>>singleton(filterOutputBundle)), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    TransformWatermarks filterAfterConsumed = manager.getWatermarks(graph.getProducer(filtered));
    assertThat(filterAfterConsumed.getSynchronizedProcessingInputTime(), not(laterThan(createAfterUpdate.getSynchronizedProcessingOutputTime())));
    assertThat(filterAfterConsumed.getSynchronizedProcessingOutputTime(), not(laterThan(filterAfterConsumed.getSynchronizedProcessingInputTime())));
}

Example 12 with TransformWatermarks

use of org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks in project beam by apache.

the class WatermarkManagerTest method getSynchronizedProcessingTimeOutputTimeIsMonotonic.

/**
   * Demonstrates that if any earlier processing holds appear in the synchronized processing time
   * output hold the result is monotonic.
   */
@Test
public void getSynchronizedProcessingTimeOutputTimeIsMonotonic() {
    Instant startTime = clock.now();
    TransformWatermarks watermarks = manager.getWatermarks(graph.getProducer(createdInts));
    assertThat(watermarks.getSynchronizedProcessingInputTime(), equalTo(startTime));
    TransformWatermarks filteredWatermarks = manager.getWatermarks(graph.getProducer(filtered));
    // Non-root processing watermarks don't progress until data has been processed
    assertThat(filteredWatermarks.getSynchronizedProcessingInputTime(), not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
    assertThat(filteredWatermarks.getSynchronizedProcessingOutputTime(), not(laterThan(BoundedWindow.TIMESTAMP_MIN_VALUE)));
    CommittedBundle<Integer> createOutput = bundleFactory.createBundle(createdInts).commit(new Instant(1250L));
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.<CommittedBundle<?>>singleton(createOutput)), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    TransformWatermarks createAfterUpdate = manager.getWatermarks(graph.getProducer(createdInts));
    assertThat(createAfterUpdate.getSynchronizedProcessingInputTime(), not(laterThan(clock.now())));
    assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(), not(laterThan(clock.now())));
    CommittedBundle<Integer> createSecondOutput = bundleFactory.createBundle(createdInts).commit(new Instant(750L));
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.<CommittedBundle<?>>singleton(createSecondOutput)), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    assertThat(createAfterUpdate.getSynchronizedProcessingOutputTime(), equalTo(clock.now()));
}

Example 13 with TransformWatermarks

use of org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks in project beam by apache.

the class WatermarkManagerTest method getWatermarkForUpdatedSourceTransform.

/**
   * Demonstrates that getWatermark for a transform that consumes no input uses the Watermark
   * Hold value provided to it as the output watermark.
   */
@Test
public void getWatermarkForUpdatedSourceTransform() {
    CommittedBundle<Integer> output = multiWindowedBundle(createdInts, 1);
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.<CommittedBundle<?>>singleton(output)), new Instant(8000L));
    manager.refreshAll();
    TransformWatermarks updatedSourceWatermark = manager.getWatermarks(graph.getProducer(createdInts));
    assertThat(updatedSourceWatermark.getOutputWatermark(), equalTo(new Instant(8000L)));
}

Example 14 with TransformWatermarks

use of org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks in project beam by apache.

the class WatermarkManagerTest method getWatermarksAfterOnlyEmptyOutput.

/**
   * Demonstrates that after watermarks of an upstream transform are updated, but no output has been
   * produced, the watermarks of a downstream process are advanced.
   */
@Test
public void getWatermarksAfterOnlyEmptyOutput() {
    CommittedBundle<Integer> emptyCreateOutput = multiWindowedBundle(createdInts);
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.<CommittedBundle<?>>singleton(emptyCreateOutput)), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    TransformWatermarks updatedSourceWatermarks = manager.getWatermarks(graph.getProducer(createdInts));
    assertThat(updatedSourceWatermarks.getOutputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
    TransformWatermarks finishedFilterWatermarks = manager.getWatermarks(graph.getProducer(filtered));
    assertThat(finishedFilterWatermarks.getInputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
    assertThat(finishedFilterWatermarks.getOutputWatermark(), not(earlierThan(BoundedWindow.TIMESTAMP_MAX_VALUE)));
}

Example 15 with TransformWatermarks

use of org.apache.beam.runners.direct.WatermarkManager.TransformWatermarks in project beam by apache.

the class WatermarkManagerTest method updateWatermarkWithCompletedElementsNotPending.

@Test
public void updateWatermarkWithCompletedElementsNotPending() {
    WindowedValue<Integer> first = WindowedValue.timestampedValueInGlobalWindow(1, new Instant(22));
    CommittedBundle<Integer> createdBundle = bundleFactory.createBundle(createdInts).add(first).commit(clock.now());
    WindowedValue<Integer> second = WindowedValue.timestampedValueInGlobalWindow(2, new Instant(22));
    CommittedBundle<Integer> neverCreatedBundle = bundleFactory.createBundle(createdInts).add(second).commit(clock.now());
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.<CommittedBundle<?>>singleton(createdBundle)), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.updateWatermarks(neverCreatedBundle, TimerUpdate.empty(), result(graph.getProducer(filtered), neverCreatedBundle.withElements(Collections.<WindowedValue<Integer>>emptyList()), Collections.<CommittedBundle<?>>emptyList()), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    TransformWatermarks filteredWms = manager.getWatermarks(graph.getProducer(filtered));
    assertThat(filteredWms.getInputWatermark(), equalTo(new Instant(22L)));
}