Search in sources :

Example 1 with RegisterRequest

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.RegisterRequest in project beam by apache.

the class RegisterAndProcessBundleOperation method start.

@Override
public void start() throws Exception {
    try (Closeable scope = context.enterStart()) {
        super.start();
        // Only register once by using the presence of the future as a signal.
        if (registerFuture == null) {
            InstructionRequest request = InstructionRequest.newBuilder().setInstructionId(idGenerator.getId()).setRegister(registerRequest).build();
            registerFuture = instructionRequestHandler.handle(request);
        }
        checkState(registerRequest.getProcessBundleDescriptorCount() == 1, "Only one bundle registration at a time currently supported.");
        InstructionRequest processBundleRequest = InstructionRequest.newBuilder().setInstructionId(getProcessBundleInstructionId()).setProcessBundle(ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId(registerRequest.getProcessBundleDescriptor(0).getId())).build();
        deregisterStateHandler = beamFnStateDelegator.registerForProcessBundleInstructionId(getProcessBundleInstructionId(), this::delegateByStateKeyType);
        processBundleResponse = getRegisterResponse(registerFuture).thenCompose(registerResponse -> instructionRequestHandler.handle(processBundleRequest));
    }
}
Also used : MoreObjects(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.MoreObjects) LoggerFactory(org.slf4j.LoggerFactory) StateAppendResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateAppendResponse) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) Map(java.util.Map) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) NameContext(org.apache.beam.runners.dataflow.worker.counters.NameContext) GlobalWindow(org.apache.beam.sdk.transforms.windowing.GlobalWindow) StateClearResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateClearResponse) StateDelegator(org.apache.beam.runners.fnexecution.state.StateDelegator) Materializations(org.apache.beam.sdk.transforms.Materializations) IdGenerator(org.apache.beam.sdk.fn.IdGenerator) KvCoder(org.apache.beam.sdk.coders.KvCoder) OperationContext(org.apache.beam.runners.dataflow.worker.util.common.worker.OperationContext) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) BeamFnApi(org.apache.beam.model.fnexecution.v1.BeamFnApi) List(java.util.List) InstructionRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) CompletionStage(java.util.concurrent.CompletionStage) RequestCase(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest.RequestCase) StateRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest) InstructionRequestHandler(org.apache.beam.runners.fnexecution.control.InstructionRequestHandler) Entry(java.util.Map.Entry) DataflowStepContext(org.apache.beam.runners.dataflow.worker.DataflowExecutionContext.DataflowStepContext) RegisterRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.RegisterRequest) StateKey(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateKey) StateGetResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateGetResponse) PTransform(org.apache.beam.model.pipeline.v1.RunnerApi.PTransform) MonitoringInfoEncodings.decodeInt64Counter(org.apache.beam.runners.core.metrics.MonitoringInfoEncodings.decodeInt64Counter) ByteStringCoder(org.apache.beam.runners.dataflow.worker.ByteStringCoder) Coder(org.apache.beam.sdk.coders.Coder) CompletableFuture(java.util.concurrent.CompletableFuture) StateNamespaces(org.apache.beam.runners.core.StateNamespaces) StateTags(org.apache.beam.runners.core.StateTags) TextFormat(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.TextFormat) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) DataflowOperationContext(org.apache.beam.runners.dataflow.worker.DataflowOperationContext) InstructionResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionResponse) Maps(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps) MonitoringInfoConstants(org.apache.beam.runners.core.metrics.MonitoringInfoConstants) ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) OutputReceiver(org.apache.beam.runners.dataflow.worker.util.common.worker.OutputReceiver) ProcessBundleProgressRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleProgressRequest) RemoteGrpcPortWrite(org.apache.beam.sdk.fn.data.RemoteGrpcPortWrite) Nullable(org.checkerframework.checker.nullness.qual.Nullable) MonitoringInfo(org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo) SideInputReader(org.apache.beam.runners.core.SideInputReader) RunnerApi(org.apache.beam.model.pipeline.v1.RunnerApi) ProcessBundleRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleRequest) Logger(org.slf4j.Logger) Table(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Table) IOException(java.io.IOException) RemoteGrpcPortRead(org.apache.beam.sdk.fn.data.RemoteGrpcPortRead) ExecutionException(java.util.concurrent.ExecutionException) Operation(org.apache.beam.runners.dataflow.worker.util.common.worker.Operation) BagState(org.apache.beam.sdk.state.BagState) Preconditions.checkState(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState) PCollectionView(org.apache.beam.sdk.values.PCollectionView) Closeable(java.io.Closeable) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) MoreFutures(org.apache.beam.sdk.util.MoreFutures) StateResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse) InstructionRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest) Closeable(java.io.Closeable)

Aggregations

Closeable (java.io.Closeable)1 IOException (java.io.IOException)1 ArrayList (java.util.ArrayList)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Entry (java.util.Map.Entry)1 Set (java.util.Set)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 CompletionStage (java.util.concurrent.CompletionStage)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 ExecutionException (java.util.concurrent.ExecutionException)1 BeamFnApi (org.apache.beam.model.fnexecution.v1.BeamFnApi)1 InstructionRequest (org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest)1 InstructionResponse (org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionResponse)1 ProcessBundleDescriptor (org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor)1 ProcessBundleProgressRequest (org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleProgressRequest)1 ProcessBundleRequest (org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleRequest)1 RegisterRequest (org.apache.beam.model.fnexecution.v1.BeamFnApi.RegisterRequest)1 StateAppendResponse (org.apache.beam.model.fnexecution.v1.BeamFnApi.StateAppendResponse)1