Examples with TimerData org.apache.beam.runners.core.TimerInternals.TimerData used on opensource projects

Example 26 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class WatermarkManagerTest method timerUpdateBuilderWithDeleteAfterBuildNotAddedToBuilt.

@Test
public void timerUpdateBuilderWithDeleteAfterBuildNotAddedToBuilt() {
    TimerUpdateBuilder builder = TimerUpdate.builder(null);
    TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
    TimerUpdate built = builder.build();
    builder.deletedTimer(timer);
    assertThat(built.getDeletedTimers(), emptyIterable());
    builder.build();
    assertThat(built.getDeletedTimers(), emptyIterable());
}

Example 27 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class WatermarkManagerTest method timerUpdateBuilderWithSetThenDeleteHasOnlyDeleted.

@Test
public void timerUpdateBuilderWithSetThenDeleteHasOnlyDeleted() {
    TimerUpdateBuilder builder = TimerUpdate.builder(null);
    TimerData timer = TimerData.of(StateNamespaces.global(), Instant.now(), TimeDomain.EVENT_TIME);
    TimerUpdate built = builder.setTimer(timer).deletedTimer(timer).build();
    assertThat(built.getSetTimers(), emptyIterable());
    assertThat(built.getDeletedTimers(), contains(timer));
}

Example 28 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class WatermarkManagerTest method synchronizedProcessingInputTimeIsHeldToUpstreamProcessingTimeTimers.

@Test
public void synchronizedProcessingInputTimeIsHeldToUpstreamProcessingTimeTimers() {
    CommittedBundle<Integer> created = multiWindowedBundle(createdInts, 1, 2, 3);
    manager.updateWatermarks(null, TimerUpdate.empty(), result(graph.getProducer(createdInts), null, Collections.<CommittedBundle<?>>singleton(created)), new Instant(40_900L));
    manager.refreshAll();
    CommittedBundle<Integer> filteredBundle = multiWindowedBundle(filtered, 2, 4);
    Instant upstreamHold = new Instant(2048L);
    TimerData upstreamProcessingTimer = TimerData.of(StateNamespaces.global(), upstreamHold, TimeDomain.PROCESSING_TIME);
    manager.updateWatermarks(created, TimerUpdate.builder(StructuralKey.of("key", StringUtf8Coder.of())).setTimer(upstreamProcessingTimer).build(), result(graph.getProducer(filtered), created.withElements(Collections.<WindowedValue<Integer>>emptyList()), Collections.<CommittedBundle<?>>singleton(filteredBundle)), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    TransformWatermarks downstreamWms = manager.getWatermarks(graph.getProducer(filteredTimesTwo));
    assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(clock.now()));
    clock.set(BoundedWindow.TIMESTAMP_MAX_VALUE);
    assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(upstreamHold));
    manager.extractFiredTimers();
    // Pending processing time timers that have been fired but aren't completed hold the
    // synchronized processing time
    assertThat(downstreamWms.getSynchronizedProcessingInputTime(), equalTo(upstreamHold));
    CommittedBundle<Integer> otherCreated = multiWindowedBundle(createdInts, 4, 8, 12);
    manager.updateWatermarks(otherCreated, TimerUpdate.builder(StructuralKey.of("key", StringUtf8Coder.of())).withCompletedTimers(Collections.singleton(upstreamProcessingTimer)).build(), result(graph.getProducer(filtered), otherCreated.withElements(Collections.<WindowedValue<Integer>>emptyList()), Collections.<CommittedBundle<?>>emptyList()), BoundedWindow.TIMESTAMP_MAX_VALUE);
    manager.refreshAll();
    assertThat(downstreamWms.getSynchronizedProcessingInputTime(), not(earlierThan(clock.now())));
}

Example 29 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class ApexParDoOperator method fireTimer.

@Override
public void fireTimer(Object key, Collection<TimerData> timerDataSet) {
    pushbackDoFnRunner.startBundle();
    @SuppressWarnings("unchecked") Coder<Object> keyCoder = (Coder) currentKeyStateInternals.getKeyCoder();
    ((StateInternalsProxy) currentKeyStateInternals).setKey(key);
    currentKeyTimerInternals.setContext(key, keyCoder, new Instant(this.currentInputWatermark), new Instant(this.currentOutputWatermark));
    for (TimerData timerData : timerDataSet) {
        StateNamespace namespace = timerData.getNamespace();
        checkArgument(namespace instanceof WindowNamespace);
        BoundedWindow window = ((WindowNamespace<?>) namespace).getWindow();
        pushbackDoFnRunner.onTimer(timerData.getTimerId(), window, timerData.getTimestamp(), timerData.getDomain());
    }
    pushbackDoFnRunner.finishBundle();
}

Example 30 with TimerData

use of org.apache.beam.runners.core.TimerInternals.TimerData in project beam by apache.

the class ApexTimerInternalsTest method testEventTimeTimers.

@Test
public void testEventTimeTimers() {
    final Map<String, Collection<TimerData>> firedTimers = new HashMap<>();
    TimerProcessor<String> timerProcessor = new TimerProcessor<String>() {

        @Override
        public void fireTimer(String key, Collection<TimerData> timerData) {
            firedTimers.put(key, timerData);
        }
    };
    TimerDataCoder timerDataCoder = TimerDataCoder.of(GlobalWindow.Coder.INSTANCE);
    String key1 = "key1";
    Instant instant0 = new Instant(0);
    Instant instant1 = new Instant(1);
    Instant instant2 = new Instant(2);
    ApexTimerInternals<String> timerInternals = new ApexTimerInternals<>(timerDataCoder);
    timerInternals.setContext(key1, StringUtf8Coder.of(), Instant.now(), null);
    TimerData timerData0 = TimerData.of("timerData0", StateNamespaces.global(), instant0, TimeDomain.EVENT_TIME);
    timerInternals.setTimer(timerData0);
    TimerData timerData1 = TimerData.of("timerData1", StateNamespaces.global(), instant1, TimeDomain.EVENT_TIME);
    timerInternals.setTimer(timerData1);
    timerInternals.fireReadyTimers(instant0.getMillis(), timerProcessor, TimeDomain.EVENT_TIME);
    assertEquals(0, firedTimers.size());
    firedTimers.clear();
    timerInternals.fireReadyTimers(instant1.getMillis(), timerProcessor, TimeDomain.PROCESSING_TIME);
    assertEquals(0, firedTimers.size());
    timerInternals.fireReadyTimers(instant1.getMillis(), timerProcessor, TimeDomain.EVENT_TIME);
    assertEquals(1, firedTimers.size());
    assertEquals(Sets.newHashSet(timerData0), Sets.newHashSet(firedTimers.values().iterator().next()));
    firedTimers.clear();
    timerInternals.fireReadyTimers(instant2.getMillis(), timerProcessor, TimeDomain.EVENT_TIME);
    assertEquals(1, firedTimers.size());
    assertEquals(Sets.newHashSet(timerData1), Sets.newHashSet(firedTimers.values().iterator().next()));
    firedTimers.clear();
}