Examples with Timer org.apache.beam.runners.core.construction.Timer used on opensource projects

Example 6 with Timer

use of org.apache.beam.runners.core.construction.Timer in project beam by apache.

the class ProcessBundleHandlerTest method setupProcessBundleHandlerForSimpleRecordingDoFn.

private ProcessBundleHandler setupProcessBundleHandlerForSimpleRecordingDoFn(List<String> dataOutput, List<Timers> timerOutput, boolean enableOutputEmbedding) throws Exception {
    DoFnWithExecutionInformation doFnWithExecutionInformation = DoFnWithExecutionInformation.of(new SimpleDoFn(), SimpleDoFn.MAIN_OUTPUT_TAG, Collections.emptyMap(), DoFnSchemaInformation.create());
    RunnerApi.FunctionSpec functionSpec = RunnerApi.FunctionSpec.newBuilder().setUrn(ParDoTranslation.CUSTOM_JAVA_DO_FN_URN).setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(doFnWithExecutionInformation))).build();
    RunnerApi.ParDoPayload parDoPayload = ParDoPayload.newBuilder().setDoFn(functionSpec).putTimerFamilySpecs("tfs-" + SimpleDoFn.TIMER_FAMILY_ID, TimerFamilySpec.newBuilder().setTimeDomain(RunnerApi.TimeDomain.Enum.EVENT_TIME).setTimerFamilyCoderId("timer-coder").build()).build();
    BeamFnApi.ProcessBundleDescriptor processBundleDescriptor = ProcessBundleDescriptor.newBuilder().putTransforms("2L", PTransform.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build()).putOutputs("2L-output", "2L-output-pc").build()).putTransforms("3L", PTransform.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN).setPayload(parDoPayload.toByteString())).putInputs("3L-input", "2L-output-pc").build()).putPcollections("2L-output-pc", PCollection.newBuilder().setWindowingStrategyId("window-strategy").setCoderId("2L-output-coder").setIsBounded(IsBounded.Enum.BOUNDED).build()).putWindowingStrategies("window-strategy", WindowingStrategy.newBuilder().setWindowCoderId("window-strategy-coder").setWindowFn(FunctionSpec.newBuilder().setUrn("beam:window_fn:global_windows:v1")).setOutputTime(OutputTime.Enum.END_OF_WINDOW).setAccumulationMode(AccumulationMode.Enum.ACCUMULATING).setTrigger(Trigger.newBuilder().setAlways(Always.getDefaultInstance())).setClosingBehavior(ClosingBehavior.Enum.EMIT_ALWAYS).setOnTimeBehavior(OnTimeBehavior.Enum.FIRE_ALWAYS).build()).setTimerApiServiceDescriptor(ApiServiceDescriptor.newBuilder().setUrl("url").build()).putCoders("string_coder", CoderTranslation.toProto(StringUtf8Coder.of()).getCoder()).putCoders("2L-output-coder", Coder.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(ModelCoders.KV_CODER_URN).build()).addComponentCoderIds("string_coder").addComponentCoderIds("string_coder").build()).putCoders("window-strategy-coder", Coder.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(ModelCoders.GLOBAL_WINDOW_CODER_URN).build()).build()).putCoders("timer-coder", Coder.newBuilder().setSpec(FunctionSpec.newBuilder().setUrn(ModelCoders.TIMER_CODER_URN)).addComponentCoderIds("string_coder").addComponentCoderIds("window-strategy-coder").build()).build();
    Map<String, BeamFnApi.ProcessBundleDescriptor> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
    Map<String, PTransformRunnerFactory> urnToPTransformRunnerFactoryMap = Maps.newHashMap(REGISTERED_RUNNER_FACTORIES);
    urnToPTransformRunnerFactoryMap.put(DATA_INPUT_URN, (PTransformRunnerFactory<Object>) (context) -> {
        context.addIncomingDataEndpoint(ApiServiceDescriptor.getDefaultInstance(), KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()), (input) -> {
            dataOutput.add(input.getValue());
        });
        return null;
    });
    Mockito.doAnswer((invocation) -> new BeamFnDataOutboundAggregator(PipelineOptionsFactory.create(), invocation.getArgument(1), new StreamObserver<Elements>() {

        @Override
        public void onNext(Elements elements) {
            for (Timers timer : elements.getTimersList()) {
                timerOutput.addAll(elements.getTimersList());
            }
        }

        @Override
        public void onError(Throwable throwable) {
        }

        @Override
        public void onCompleted() {
        }
    }, invocation.getArgument(2))).when(beamFnDataClient).createOutboundAggregator(any(), any(), anyBoolean());
    return new ProcessBundleHandler(PipelineOptionsFactory.create(), enableOutputEmbedding ? Collections.singleton(BeamUrns.getUrn(StandardRunnerProtocols.Enum.CONTROL_RESPONSE_ELEMENTS_EMBEDDING)) : Collections.emptySet(), fnApiRegistry::get, beamFnDataClient, null, /* beamFnStateClient */
    null, /* finalizeBundleHandler */
    new ShortIdMap(), urnToPTransformRunnerFactoryMap, Caches.noop(), new BundleProcessorCache());
}

Example 7 with Timer

use of org.apache.beam.runners.core.construction.Timer in project beam by apache.

the class FnApiDoFnRunner method processTimer.

private <K> void processTimer(String timerIdOrTimerFamilyId, TimeDomain timeDomain, Timer<K> timer) {
    try {
        currentKey = timer.getUserKey();
        Iterator<BoundedWindow> windowIterator = (Iterator<BoundedWindow>) timer.getWindows().iterator();
        while (windowIterator.hasNext()) {
            currentWindow = windowIterator.next();
            Modifications bundleModifications = timerBundleTracker.getBundleModifications();
            Table<String, String, Timer<K>> modifiedTimerIds = bundleModifications.getModifiedTimerIds();
            NavigableSet<TimerInfo<K>> earlierTimers = bundleModifications.getModifiedTimersOrdered(timeDomain).headSet(TimerInfo.of(timer, "", timeDomain), true);
            while (!earlierTimers.isEmpty()) {
                TimerInfo<K> insertedTimer = earlierTimers.pollFirst();
                if (timerModified(modifiedTimerIds, insertedTimer.getTimerFamilyOrId(), insertedTimer.getTimer())) {
                    continue;
                }
                String timerId = insertedTimer.getTimer().getDynamicTimerTag().isEmpty() ? insertedTimer.getTimerFamilyOrId() : insertedTimer.getTimer().getDynamicTimerTag();
                String timerFamily = insertedTimer.getTimer().getDynamicTimerTag().isEmpty() ? "" : insertedTimer.getTimerFamilyOrId();
                // If this timer was created previously in the bundle as an overwrite of a previous timer,
                // we must make sure
                // to clear the old timer. Since we are firing the timer inline, the runner doesn't know
                // that the old timer
                // was overwritten, and will otherwise fire it - causing a spurious timer fire.
                modifiedTimerIds.put(insertedTimer.getTimerFamilyOrId(), insertedTimer.getTimer().getDynamicTimerTag(), Timer.cleared(insertedTimer.getTimer().getUserKey(), insertedTimer.getTimer().getDynamicTimerTag(), insertedTimer.getTimer().getWindows()));
                // It's important to call processTimer after inserting the above deletion, otherwise the
                // above line
                // would overwrite any looping timer with a deletion.
                processTimerDirect(timerFamily, timerId, insertedTimer.getTimeDomain(), insertedTimer.getTimer());
            }
            if (!timerModified(modifiedTimerIds, timerIdOrTimerFamilyId, timer)) {
                // The timerIdOrTimerFamilyId contains either a timerId from timer declaration or
                // timerFamilyId
                // from timer family declaration.
                boolean isFamily = timerIdOrTimerFamilyId.startsWith(TimerFamilyDeclaration.PREFIX);
                String timerId = isFamily ? "" : timerIdOrTimerFamilyId;
                String timerFamilyId = isFamily ? timerIdOrTimerFamilyId : "";
                processTimerDirect(timerFamilyId, timerId, timeDomain, timer);
            }
        }
    } finally {
        currentKey = null;
        currentTimer = null;
        currentTimeDomain = null;
        currentWindow = null;
    }
}

Example 8 with Timer

use of org.apache.beam.runners.core.construction.Timer in project beam by apache.

the class FnApiTimerBundleTracker method outputTimers.

public void outputTimers(Function<String, FnDataReceiver<Timer<?>>> getTimersReceiverFromTimerIdFn) {
    for (Cell<ByteString, ByteString, Modifications<K>> cell : timerModifications.cellSet()) {
        Modifications<K> modifications = cell.getValue();
        if (modifications != null) {
            for (Cell<String, String, Timer<K>> timerCell : modifications.getModifiedTimerIds().cellSet()) {
                String timerFamilyOrId = timerCell.getRowKey();
                Timer<K> timer = timerCell.getValue();
                try {
                    if (timerFamilyOrId != null && timer != null) {
                        getTimersReceiverFromTimerIdFn.apply(timerFamilyOrId).accept(timer);
                    }
                } catch (Throwable t) {
                    throw UserCodeException.wrap(t);
                }
            }
        }
    }
}

Example 9 with Timer

use of org.apache.beam.runners.core.construction.Timer in project beam by apache.

the class PipelineTranslatorUtils method fireTimer.

private static void fireTimer(TimerInternals.TimerData timer, Map<KV<String, String>, FnDataReceiver<Timer>> timerReceivers, Object currentTimerKey) {
    StateNamespace namespace = timer.getNamespace();
    Preconditions.checkArgument(namespace instanceof StateNamespaces.WindowNamespace);
    BoundedWindow window = ((StateNamespaces.WindowNamespace) namespace).getWindow();
    Instant timestamp = timer.getTimestamp();
    Instant outputTimestamp = timer.getOutputTimestamp();
    Timer<?> timerValue = Timer.of(currentTimerKey, timer.getTimerId(), Collections.singletonList(window), timestamp, outputTimestamp, PaneInfo.NO_FIRING);
    KV<String, String> transformAndTimerId = TimerReceiverFactory.decodeTimerDataTimerId(timer.getTimerFamilyId());
    FnDataReceiver<Timer> fnTimerReceiver = timerReceivers.get(transformAndTimerId);
    Preconditions.checkNotNull(fnTimerReceiver, "No FnDataReceiver found for %s", transformAndTimerId);
    try {
        fnTimerReceiver.accept(timerValue);
    } catch (Exception e) {
        throw new RuntimeException(String.format(Locale.ENGLISH, "Failed to process timer: %s", timerValue));
    }
}

Example 10 with Timer

use of org.apache.beam.runners.core.construction.Timer in project beam by apache.

the class DefaultJobBundleFactory method getTimerReceivers.

private static Map<KV<String, String>, RemoteOutputReceiver<Timer<?>>> getTimerReceivers(ExecutableProcessBundleDescriptor processBundleDescriptor, TimerReceiverFactory timerReceiverFactory) {
    ImmutableMap.Builder<KV<String, String>, RemoteOutputReceiver<Timer<?>>> timerReceivers = ImmutableMap.builder();
    for (Map.Entry<String, Map<String, TimerSpec>> transformTimerSpecs : processBundleDescriptor.getTimerSpecs().entrySet()) {
        for (TimerSpec timerSpec : transformTimerSpecs.getValue().values()) {
            FnDataReceiver<Timer<?>> receiver = (FnDataReceiver) timerReceiverFactory.create(timerSpec.transformId(), timerSpec.timerId());
            timerReceivers.put(KV.of(timerSpec.transformId(), timerSpec.timerId()), RemoteOutputReceiver.of(timerSpec.coder(), receiver));
        }
    }
    return timerReceivers.build();
}