Search in sources :

Example 6 with ProcessBundleDescriptor

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor in project beam by apache.

the class ProcessBundleHandlerTest method testCreatingPTransformExceptionsArePropagated.

@Test
public void testCreatingPTransformExceptionsArePropagated() throws Exception {
    BeamFnApi.ProcessBundleDescriptor processBundleDescriptor = BeamFnApi.ProcessBundleDescriptor.newBuilder().putTransforms("2L", RunnerApi.PTransform.newBuilder().setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build()).build()).build();
    Map<String, BeamFnApi.ProcessBundleDescriptor> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
    ProcessBundleHandler handler = new ProcessBundleHandler(PipelineOptionsFactory.create(), Collections.emptySet(), fnApiRegistry::get, beamFnDataClient, null, /* beamFnStateGrpcClientCache */
    null, /* finalizeBundleHandler */
    new ShortIdMap(), ImmutableMap.of(DATA_INPUT_URN, (context) -> {
        throw new IllegalStateException("TestException");
    }), Caches.noop(), new BundleProcessorCache());
    assertThrows("TestException", IllegalStateException.class, () -> handler.processBundle(BeamFnApi.InstructionRequest.newBuilder().setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("1L")).build()));
}
Also used : BeamFnDataOutboundAggregator(org.apache.beam.sdk.fn.data.BeamFnDataOutboundAggregator) Elements(org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements) TimerSpecs(org.apache.beam.sdk.state.TimerSpecs) Assert.assertNotSame(org.junit.Assert.assertNotSame) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) MockitoAnnotations(org.mockito.MockitoAnnotations) FunctionSpec(org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec) MetricsContainerStepMap(org.apache.beam.runners.core.metrics.MetricsContainerStepMap) Arrays.asList(java.util.Arrays.asList) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Map(java.util.Map) GlobalWindow(org.apache.beam.sdk.transforms.windowing.GlobalWindow) Uninterruptibles(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles) ApiServiceDescriptor(org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor) ShortIdMap(org.apache.beam.runners.core.metrics.ShortIdMap) KvCoder(org.apache.beam.sdk.coders.KvCoder) PTransformTranslation(org.apache.beam.runners.core.construction.PTransformTranslation) TimerEndpoint(org.apache.beam.sdk.fn.data.TimerEndpoint) Set(java.util.Set) BeamFnApi(org.apache.beam.model.fnexecution.v1.BeamFnApi) Data(org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements.Data) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) StandardRunnerProtocols(org.apache.beam.model.pipeline.v1.RunnerApi.StandardRunnerProtocols) Matchers.contains(org.hamcrest.Matchers.contains) PTransformRunnerFactory(org.apache.beam.fn.harness.PTransformRunnerFactory) Matchers.is(org.hamcrest.Matchers.is) Mockito.eq(org.mockito.Mockito.eq) Mockito.mock(org.mockito.Mockito.mock) ClosingBehavior(org.apache.beam.model.pipeline.v1.RunnerApi.ClosingBehavior) KV(org.apache.beam.sdk.values.KV) TimerMap(org.apache.beam.sdk.state.TimerMap) ExecutionStateTracker(org.apache.beam.runners.core.metrics.ExecutionStateTracker) Mock(org.mockito.Mock) BundleFinalizer(org.apache.beam.sdk.transforms.DoFn.BundleFinalizer) RunWith(org.junit.runner.RunWith) TimerFamilySpec(org.apache.beam.model.pipeline.v1.RunnerApi.TimerFamilySpec) ArgumentMatchers.anyBoolean(org.mockito.ArgumentMatchers.anyBoolean) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Assert.assertSame(org.junit.Assert.assertSame) Timers(org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements.Timers) PCollectionConsumerRegistry(org.apache.beam.fn.harness.data.PCollectionConsumerRegistry) PCollection(org.apache.beam.model.pipeline.v1.RunnerApi.PCollection) TimerSpec(org.apache.beam.sdk.state.TimerSpec) TupleTag(org.apache.beam.sdk.values.TupleTag) Cache(org.apache.beam.fn.harness.Cache) BeamFnDataClient(org.apache.beam.fn.harness.data.BeamFnDataClient) Maps(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Before(org.junit.Before) RunnerApi(org.apache.beam.model.pipeline.v1.RunnerApi) DoFn(org.apache.beam.sdk.transforms.DoFn) PTransformFunctionRegistry(org.apache.beam.fn.harness.data.PTransformFunctionRegistry) CloseableFnDataReceiver(org.apache.beam.sdk.fn.data.CloseableFnDataReceiver) ProcessBundleRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleRequest) TimerFamilyDeclaration(org.apache.beam.sdk.transforms.reflect.DoFnSignature.TimerFamilyDeclaration) Assert.assertTrue(org.junit.Assert.assertTrue) Mockito.times(org.mockito.Mockito.times) IOException(java.io.IOException) Test(org.junit.Test) OnTimeBehavior(org.apache.beam.model.pipeline.v1.RunnerApi.OnTimeBehavior) ProgressRequestCallback(org.apache.beam.fn.harness.PTransformRunnerFactory.ProgressRequestCallback) BeamUrns(org.apache.beam.runners.core.construction.BeamUrns) DataEndpoint(org.apache.beam.sdk.fn.data.DataEndpoint) Assert.assertNull(org.junit.Assert.assertNull) AccumulationMode(org.apache.beam.model.pipeline.v1.RunnerApi.AccumulationMode) OutputTime(org.apache.beam.model.pipeline.v1.RunnerApi.OutputTime) Preconditions.checkState(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState) Timer(org.apache.beam.runners.core.construction.Timer) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) TimeDomain(org.apache.beam.sdk.state.TimeDomain) Assert.assertEquals(org.junit.Assert.assertEquals) StateResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse) Coder(org.apache.beam.model.pipeline.v1.RunnerApi.Coder) BeamFnStateClient(org.apache.beam.fn.harness.state.BeamFnStateClient) CoderTranslation(org.apache.beam.runners.core.construction.CoderTranslation) Mockito.argThat(org.mockito.Mockito.argThat) DoFnSchemaInformation(org.apache.beam.sdk.transforms.DoFnSchemaInformation) BundleProcessorCache(org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessorCache) IsBounded(org.apache.beam.model.pipeline.v1.RunnerApi.IsBounded) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) BundleProcessor(org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessor) REGISTERED_RUNNER_FACTORIES(org.apache.beam.fn.harness.control.ProcessBundleHandler.REGISTERED_RUNNER_FACTORIES) PaneInfo(org.apache.beam.sdk.transforms.windowing.PaneInfo) Collection(java.util.Collection) DoFnWithExecutionInformation(org.apache.beam.sdk.util.DoFnWithExecutionInformation) ModelCoders(org.apache.beam.runners.core.construction.ModelCoders) List(java.util.List) InstructionRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest) StateRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest) SerializableUtils(org.apache.beam.sdk.util.SerializableUtils) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Trigger(org.apache.beam.model.pipeline.v1.RunnerApi.Trigger) Matchers.equalTo(org.hamcrest.Matchers.equalTo) StreamObserver(org.apache.beam.vendor.grpc.v1p43p2.io.grpc.stub.StreamObserver) WindowingStrategy(org.apache.beam.model.pipeline.v1.RunnerApi.WindowingStrategy) ThrowingRunnable(org.apache.beam.sdk.function.ThrowingRunnable) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Always(org.apache.beam.model.pipeline.v1.RunnerApi.Trigger.Always) PTransform(org.apache.beam.model.pipeline.v1.RunnerApi.PTransform) Assert.assertThrows(org.junit.Assert.assertThrows) IsEmptyCollection.empty(org.hamcrest.collection.IsEmptyCollection.empty) CompletableFuture(java.util.concurrent.CompletableFuture) BeamFnStateGrpcClientCache(org.apache.beam.fn.harness.state.BeamFnStateGrpcClientCache) PipelineOptionsFactory(org.apache.beam.sdk.options.PipelineOptionsFactory) BeamFnDataReadRunner(org.apache.beam.fn.harness.BeamFnDataReadRunner) StringUtf8Coder(org.apache.beam.sdk.coders.StringUtf8Coder) HashSet(java.util.HashSet) CacheToken(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleRequest.CacheToken) ParDoPayload(org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload) InstructionResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionResponse) ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) ParDoTranslation(org.apache.beam.runners.core.construction.ParDoTranslation) Assert.assertNotNull(org.junit.Assert.assertNotNull) Mockito.when(org.mockito.Mockito.when) JUnit4(org.junit.runners.JUnit4) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) Mockito(org.mockito.Mockito) Matchers.emptyIterable(org.hamcrest.Matchers.emptyIterable) Instant(org.joda.time.Instant) Caches(org.apache.beam.fn.harness.Caches) CallbackRegistration(org.apache.beam.fn.harness.control.FinalizeBundleHandler.CallbackRegistration) Collections(java.util.Collections) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) BeamFnApi(org.apache.beam.model.fnexecution.v1.BeamFnApi) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) BundleProcessorCache(org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessorCache) ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) ShortIdMap(org.apache.beam.runners.core.metrics.ShortIdMap) Test(org.junit.Test)

Example 7 with ProcessBundleDescriptor

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor in project beam by apache.

the class ProcessBundleHandlerTest method testPTransformStartExceptionsArePropagated.

@Test
public void testPTransformStartExceptionsArePropagated() {
    BeamFnApi.ProcessBundleDescriptor processBundleDescriptor = BeamFnApi.ProcessBundleDescriptor.newBuilder().putTransforms("2L", RunnerApi.PTransform.newBuilder().setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build()).build()).build();
    Map<String, BeamFnApi.ProcessBundleDescriptor> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
    ProcessBundleHandler handler = new ProcessBundleHandler(PipelineOptionsFactory.create(), Collections.emptySet(), fnApiRegistry::get, beamFnDataClient, null, /* beamFnStateGrpcClientCache */
    null, /* finalizeBundleHandler */
    new ShortIdMap(), ImmutableMap.of(DATA_INPUT_URN, (PTransformRunnerFactory<Object>) (context) -> {
        context.addStartBundleFunction(ProcessBundleHandlerTest::throwException);
        return null;
    }), Caches.noop(), new BundleProcessorCache());
    assertThrows("TestException", IllegalStateException.class, () -> handler.processBundle(BeamFnApi.InstructionRequest.newBuilder().setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("1L")).build()));
    // BundleProcessor is not re-added back to the BundleProcessorCache in case of an exception
    // during bundle processing
    assertThat(handler.bundleProcessorCache.getCachedBundleProcessors().get("1L"), empty());
}
Also used : ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) BeamFnApi(org.apache.beam.model.fnexecution.v1.BeamFnApi) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) BundleProcessorCache(org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessorCache) PTransformRunnerFactory(org.apache.beam.fn.harness.PTransformRunnerFactory) ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) ShortIdMap(org.apache.beam.runners.core.metrics.ShortIdMap) Test(org.junit.Test)

Example 8 with ProcessBundleDescriptor

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor in project beam by apache.

the class ProcessBundleHandlerTest method testPTransformFinishExceptionsArePropagated.

@Test
public void testPTransformFinishExceptionsArePropagated() throws Exception {
    BeamFnApi.ProcessBundleDescriptor processBundleDescriptor = BeamFnApi.ProcessBundleDescriptor.newBuilder().putTransforms("2L", RunnerApi.PTransform.newBuilder().setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build()).build()).build();
    Map<String, BeamFnApi.ProcessBundleDescriptor> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
    ProcessBundleHandler handler = new ProcessBundleHandler(PipelineOptionsFactory.create(), Collections.emptySet(), fnApiRegistry::get, beamFnDataClient, null, /* beamFnStateGrpcClientCache */
    null, /* finalizeBundleHandler */
    new ShortIdMap(), ImmutableMap.of(DATA_INPUT_URN, (PTransformRunnerFactory<Object>) (context) -> {
        context.addFinishBundleFunction(ProcessBundleHandlerTest::throwException);
        return null;
    }), Caches.noop(), new BundleProcessorCache());
    assertThrows("TestException", IllegalStateException.class, () -> handler.processBundle(BeamFnApi.InstructionRequest.newBuilder().setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("1L")).build()));
    // BundleProcessor is not re-added back to the BundleProcessorCache in case of an exception
    // during bundle processing
    assertThat(handler.bundleProcessorCache.getCachedBundleProcessors().get("1L"), empty());
}
Also used : ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) BeamFnApi(org.apache.beam.model.fnexecution.v1.BeamFnApi) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) BundleProcessorCache(org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessorCache) PTransformRunnerFactory(org.apache.beam.fn.harness.PTransformRunnerFactory) ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) ShortIdMap(org.apache.beam.runners.core.metrics.ShortIdMap) Test(org.junit.Test)

Example 9 with ProcessBundleDescriptor

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor in project beam by apache.

the class ProcessBundleHandlerTest method testOrderOfStartAndFinishCalls.

@Test
public void testOrderOfStartAndFinishCalls() throws Exception {
    BeamFnApi.ProcessBundleDescriptor processBundleDescriptor = BeamFnApi.ProcessBundleDescriptor.newBuilder().putTransforms("2L", RunnerApi.PTransform.newBuilder().setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build()).putOutputs("2L-output", "2L-output-pc").build()).putTransforms("3L", RunnerApi.PTransform.newBuilder().setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_OUTPUT_URN).build()).putInputs("3L-input", "2L-output-pc").build()).putPcollections("2L-output-pc", RunnerApi.PCollection.getDefaultInstance()).build();
    Map<String, BeamFnApi.ProcessBundleDescriptor> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
    List<RunnerApi.PTransform> transformsProcessed = new ArrayList<>();
    List<String> orderOfOperations = new ArrayList<>();
    PTransformRunnerFactory<Object> startFinishRecorder = (context) -> {
        String pTransformId = context.getPTransformId();
        transformsProcessed.add(context.getPTransform());
        Supplier<String> processBundleInstructionId = context.getProcessBundleInstructionIdSupplier();
        context.addStartBundleFunction(() -> {
            assertThat(processBundleInstructionId.get(), equalTo("999L"));
            orderOfOperations.add("Start" + pTransformId);
        });
        context.addFinishBundleFunction(() -> {
            assertThat(processBundleInstructionId.get(), equalTo("999L"));
            orderOfOperations.add("Finish" + pTransformId);
        });
        return null;
    };
    ProcessBundleHandler handler = new ProcessBundleHandler(PipelineOptionsFactory.create(), Collections.emptySet(), fnApiRegistry::get, beamFnDataClient, null, /* beamFnStateClient */
    null, /* finalizeBundleHandler */
    new ShortIdMap(), ImmutableMap.of(DATA_INPUT_URN, startFinishRecorder, DATA_OUTPUT_URN, startFinishRecorder), Caches.noop(), new BundleProcessorCache());
    handler.processBundle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId("999L").setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("1L")).build());
    // Processing of transforms is performed in reverse order.
    assertThat(transformsProcessed, contains(processBundleDescriptor.getTransformsMap().get("3L"), processBundleDescriptor.getTransformsMap().get("2L")));
    // Start should occur in reverse order while finish calls should occur in forward order
    assertThat(orderOfOperations, contains("Start3L", "Start2L", "Finish2L", "Finish3L"));
}
Also used : BeamFnDataOutboundAggregator(org.apache.beam.sdk.fn.data.BeamFnDataOutboundAggregator) Elements(org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements) TimerSpecs(org.apache.beam.sdk.state.TimerSpecs) Assert.assertNotSame(org.junit.Assert.assertNotSame) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) MockitoAnnotations(org.mockito.MockitoAnnotations) FunctionSpec(org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec) MetricsContainerStepMap(org.apache.beam.runners.core.metrics.MetricsContainerStepMap) Arrays.asList(java.util.Arrays.asList) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Map(java.util.Map) GlobalWindow(org.apache.beam.sdk.transforms.windowing.GlobalWindow) Uninterruptibles(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles) ApiServiceDescriptor(org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor) ShortIdMap(org.apache.beam.runners.core.metrics.ShortIdMap) KvCoder(org.apache.beam.sdk.coders.KvCoder) PTransformTranslation(org.apache.beam.runners.core.construction.PTransformTranslation) TimerEndpoint(org.apache.beam.sdk.fn.data.TimerEndpoint) Set(java.util.Set) BeamFnApi(org.apache.beam.model.fnexecution.v1.BeamFnApi) Data(org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements.Data) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) StandardRunnerProtocols(org.apache.beam.model.pipeline.v1.RunnerApi.StandardRunnerProtocols) Matchers.contains(org.hamcrest.Matchers.contains) PTransformRunnerFactory(org.apache.beam.fn.harness.PTransformRunnerFactory) Matchers.is(org.hamcrest.Matchers.is) Mockito.eq(org.mockito.Mockito.eq) Mockito.mock(org.mockito.Mockito.mock) ClosingBehavior(org.apache.beam.model.pipeline.v1.RunnerApi.ClosingBehavior) KV(org.apache.beam.sdk.values.KV) TimerMap(org.apache.beam.sdk.state.TimerMap) ExecutionStateTracker(org.apache.beam.runners.core.metrics.ExecutionStateTracker) Mock(org.mockito.Mock) BundleFinalizer(org.apache.beam.sdk.transforms.DoFn.BundleFinalizer) RunWith(org.junit.runner.RunWith) TimerFamilySpec(org.apache.beam.model.pipeline.v1.RunnerApi.TimerFamilySpec) ArgumentMatchers.anyBoolean(org.mockito.ArgumentMatchers.anyBoolean) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Assert.assertSame(org.junit.Assert.assertSame) Timers(org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements.Timers) PCollectionConsumerRegistry(org.apache.beam.fn.harness.data.PCollectionConsumerRegistry) PCollection(org.apache.beam.model.pipeline.v1.RunnerApi.PCollection) TimerSpec(org.apache.beam.sdk.state.TimerSpec) TupleTag(org.apache.beam.sdk.values.TupleTag) Cache(org.apache.beam.fn.harness.Cache) BeamFnDataClient(org.apache.beam.fn.harness.data.BeamFnDataClient) Maps(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Before(org.junit.Before) RunnerApi(org.apache.beam.model.pipeline.v1.RunnerApi) DoFn(org.apache.beam.sdk.transforms.DoFn) PTransformFunctionRegistry(org.apache.beam.fn.harness.data.PTransformFunctionRegistry) CloseableFnDataReceiver(org.apache.beam.sdk.fn.data.CloseableFnDataReceiver) ProcessBundleRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleRequest) TimerFamilyDeclaration(org.apache.beam.sdk.transforms.reflect.DoFnSignature.TimerFamilyDeclaration) Assert.assertTrue(org.junit.Assert.assertTrue) Mockito.times(org.mockito.Mockito.times) IOException(java.io.IOException) Test(org.junit.Test) OnTimeBehavior(org.apache.beam.model.pipeline.v1.RunnerApi.OnTimeBehavior) ProgressRequestCallback(org.apache.beam.fn.harness.PTransformRunnerFactory.ProgressRequestCallback) BeamUrns(org.apache.beam.runners.core.construction.BeamUrns) DataEndpoint(org.apache.beam.sdk.fn.data.DataEndpoint) Assert.assertNull(org.junit.Assert.assertNull) AccumulationMode(org.apache.beam.model.pipeline.v1.RunnerApi.AccumulationMode) OutputTime(org.apache.beam.model.pipeline.v1.RunnerApi.OutputTime) Preconditions.checkState(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState) Timer(org.apache.beam.runners.core.construction.Timer) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) TimeDomain(org.apache.beam.sdk.state.TimeDomain) Assert.assertEquals(org.junit.Assert.assertEquals) StateResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse) Coder(org.apache.beam.model.pipeline.v1.RunnerApi.Coder) BeamFnStateClient(org.apache.beam.fn.harness.state.BeamFnStateClient) CoderTranslation(org.apache.beam.runners.core.construction.CoderTranslation) Mockito.argThat(org.mockito.Mockito.argThat) DoFnSchemaInformation(org.apache.beam.sdk.transforms.DoFnSchemaInformation) BundleProcessorCache(org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessorCache) IsBounded(org.apache.beam.model.pipeline.v1.RunnerApi.IsBounded) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) BundleProcessor(org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessor) REGISTERED_RUNNER_FACTORIES(org.apache.beam.fn.harness.control.ProcessBundleHandler.REGISTERED_RUNNER_FACTORIES) PaneInfo(org.apache.beam.sdk.transforms.windowing.PaneInfo) Collection(java.util.Collection) DoFnWithExecutionInformation(org.apache.beam.sdk.util.DoFnWithExecutionInformation) ModelCoders(org.apache.beam.runners.core.construction.ModelCoders) List(java.util.List) InstructionRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest) StateRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest) SerializableUtils(org.apache.beam.sdk.util.SerializableUtils) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Trigger(org.apache.beam.model.pipeline.v1.RunnerApi.Trigger) Matchers.equalTo(org.hamcrest.Matchers.equalTo) StreamObserver(org.apache.beam.vendor.grpc.v1p43p2.io.grpc.stub.StreamObserver) WindowingStrategy(org.apache.beam.model.pipeline.v1.RunnerApi.WindowingStrategy) ThrowingRunnable(org.apache.beam.sdk.function.ThrowingRunnable) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Always(org.apache.beam.model.pipeline.v1.RunnerApi.Trigger.Always) PTransform(org.apache.beam.model.pipeline.v1.RunnerApi.PTransform) Assert.assertThrows(org.junit.Assert.assertThrows) IsEmptyCollection.empty(org.hamcrest.collection.IsEmptyCollection.empty) CompletableFuture(java.util.concurrent.CompletableFuture) BeamFnStateGrpcClientCache(org.apache.beam.fn.harness.state.BeamFnStateGrpcClientCache) PipelineOptionsFactory(org.apache.beam.sdk.options.PipelineOptionsFactory) BeamFnDataReadRunner(org.apache.beam.fn.harness.BeamFnDataReadRunner) StringUtf8Coder(org.apache.beam.sdk.coders.StringUtf8Coder) HashSet(java.util.HashSet) CacheToken(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleRequest.CacheToken) ParDoPayload(org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload) InstructionResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionResponse) ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) ParDoTranslation(org.apache.beam.runners.core.construction.ParDoTranslation) Assert.assertNotNull(org.junit.Assert.assertNotNull) Mockito.when(org.mockito.Mockito.when) JUnit4(org.junit.runners.JUnit4) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) Mockito(org.mockito.Mockito) Matchers.emptyIterable(org.hamcrest.Matchers.emptyIterable) Instant(org.joda.time.Instant) Caches(org.apache.beam.fn.harness.Caches) CallbackRegistration(org.apache.beam.fn.harness.control.FinalizeBundleHandler.CallbackRegistration) Collections(java.util.Collections) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) BeamFnApi(org.apache.beam.model.fnexecution.v1.BeamFnApi) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) BundleProcessorCache(org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessorCache) ArrayList(java.util.ArrayList) ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) ShortIdMap(org.apache.beam.runners.core.metrics.ShortIdMap) Supplier(java.util.function.Supplier) PTransform(org.apache.beam.model.pipeline.v1.RunnerApi.PTransform) Test(org.junit.Test)

Example 10 with ProcessBundleDescriptor

use of org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor in project beam by apache.

the class ProcessBundleHandlerTest method testBundleProcessorIsResetWhenAddedBackToCache.

@Test
public void testBundleProcessorIsResetWhenAddedBackToCache() throws Exception {
    BeamFnApi.ProcessBundleDescriptor processBundleDescriptor = BeamFnApi.ProcessBundleDescriptor.newBuilder().putTransforms("2L", RunnerApi.PTransform.newBuilder().setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build()).build()).build();
    Map<String, BeamFnApi.ProcessBundleDescriptor> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
    ProcessBundleHandler handler = new ProcessBundleHandler(PipelineOptionsFactory.create(), Collections.emptySet(), fnApiRegistry::get, beamFnDataClient, null, /* beamFnStateGrpcClientCache */
    null, /* finalizeBundleHandler */
    new ShortIdMap(), ImmutableMap.of(DATA_INPUT_URN, (context) -> null), Caches.noop(), new TestBundleProcessorCache());
    assertThat(TestBundleProcessor.resetCnt, equalTo(0));
    handler.processBundle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId("998L").setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("1L")).build());
    // Check that BundleProcessor is reset when added back to the cache
    assertThat(TestBundleProcessor.resetCnt, equalTo(1));
    // BundleProcessor is added back to the BundleProcessorCache
    assertThat(handler.bundleProcessorCache.getCachedBundleProcessors().size(), equalTo(1));
    assertThat(handler.bundleProcessorCache.getCachedBundleProcessors().get("1L").size(), equalTo(1));
    // Add a reset handler that throws to test discarding the bundle processor on reset failure.
    Iterables.getOnlyElement(handler.bundleProcessorCache.getCachedBundleProcessors().get("1L")).getResetFunctions().add(() -> {
        throw new IllegalStateException("ResetFailed");
    });
    handler.processBundle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId("999L").setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("1L")).build());
    // BundleProcessor is discarded instead of being added back to the BundleProcessorCache
    assertThat(handler.bundleProcessorCache.getCachedBundleProcessors().get("1L").size(), equalTo(0));
}
Also used : BeamFnDataOutboundAggregator(org.apache.beam.sdk.fn.data.BeamFnDataOutboundAggregator) Elements(org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements) TimerSpecs(org.apache.beam.sdk.state.TimerSpecs) Assert.assertNotSame(org.junit.Assert.assertNotSame) ImmutableMap(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap) MockitoAnnotations(org.mockito.MockitoAnnotations) FunctionSpec(org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec) MetricsContainerStepMap(org.apache.beam.runners.core.metrics.MetricsContainerStepMap) Arrays.asList(java.util.Arrays.asList) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Map(java.util.Map) GlobalWindow(org.apache.beam.sdk.transforms.windowing.GlobalWindow) Uninterruptibles(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.util.concurrent.Uninterruptibles) ApiServiceDescriptor(org.apache.beam.model.pipeline.v1.Endpoints.ApiServiceDescriptor) ShortIdMap(org.apache.beam.runners.core.metrics.ShortIdMap) KvCoder(org.apache.beam.sdk.coders.KvCoder) PTransformTranslation(org.apache.beam.runners.core.construction.PTransformTranslation) TimerEndpoint(org.apache.beam.sdk.fn.data.TimerEndpoint) Set(java.util.Set) BeamFnApi(org.apache.beam.model.fnexecution.v1.BeamFnApi) Data(org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements.Data) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) StandardRunnerProtocols(org.apache.beam.model.pipeline.v1.RunnerApi.StandardRunnerProtocols) Matchers.contains(org.hamcrest.Matchers.contains) PTransformRunnerFactory(org.apache.beam.fn.harness.PTransformRunnerFactory) Matchers.is(org.hamcrest.Matchers.is) Mockito.eq(org.mockito.Mockito.eq) Mockito.mock(org.mockito.Mockito.mock) ClosingBehavior(org.apache.beam.model.pipeline.v1.RunnerApi.ClosingBehavior) KV(org.apache.beam.sdk.values.KV) TimerMap(org.apache.beam.sdk.state.TimerMap) ExecutionStateTracker(org.apache.beam.runners.core.metrics.ExecutionStateTracker) Mock(org.mockito.Mock) BundleFinalizer(org.apache.beam.sdk.transforms.DoFn.BundleFinalizer) RunWith(org.junit.runner.RunWith) TimerFamilySpec(org.apache.beam.model.pipeline.v1.RunnerApi.TimerFamilySpec) ArgumentMatchers.anyBoolean(org.mockito.ArgumentMatchers.anyBoolean) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Assert.assertSame(org.junit.Assert.assertSame) Timers(org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements.Timers) PCollectionConsumerRegistry(org.apache.beam.fn.harness.data.PCollectionConsumerRegistry) PCollection(org.apache.beam.model.pipeline.v1.RunnerApi.PCollection) TimerSpec(org.apache.beam.sdk.state.TimerSpec) TupleTag(org.apache.beam.sdk.values.TupleTag) Cache(org.apache.beam.fn.harness.Cache) BeamFnDataClient(org.apache.beam.fn.harness.data.BeamFnDataClient) Maps(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Maps) MatcherAssert.assertThat(org.hamcrest.MatcherAssert.assertThat) Before(org.junit.Before) RunnerApi(org.apache.beam.model.pipeline.v1.RunnerApi) DoFn(org.apache.beam.sdk.transforms.DoFn) PTransformFunctionRegistry(org.apache.beam.fn.harness.data.PTransformFunctionRegistry) CloseableFnDataReceiver(org.apache.beam.sdk.fn.data.CloseableFnDataReceiver) ProcessBundleRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleRequest) TimerFamilyDeclaration(org.apache.beam.sdk.transforms.reflect.DoFnSignature.TimerFamilyDeclaration) Assert.assertTrue(org.junit.Assert.assertTrue) Mockito.times(org.mockito.Mockito.times) IOException(java.io.IOException) Test(org.junit.Test) OnTimeBehavior(org.apache.beam.model.pipeline.v1.RunnerApi.OnTimeBehavior) ProgressRequestCallback(org.apache.beam.fn.harness.PTransformRunnerFactory.ProgressRequestCallback) BeamUrns(org.apache.beam.runners.core.construction.BeamUrns) DataEndpoint(org.apache.beam.sdk.fn.data.DataEndpoint) Assert.assertNull(org.junit.Assert.assertNull) AccumulationMode(org.apache.beam.model.pipeline.v1.RunnerApi.AccumulationMode) OutputTime(org.apache.beam.model.pipeline.v1.RunnerApi.OutputTime) Preconditions.checkState(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions.checkState) Timer(org.apache.beam.runners.core.construction.Timer) BoundedWindow(org.apache.beam.sdk.transforms.windowing.BoundedWindow) TimeDomain(org.apache.beam.sdk.state.TimeDomain) Assert.assertEquals(org.junit.Assert.assertEquals) StateResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateResponse) Coder(org.apache.beam.model.pipeline.v1.RunnerApi.Coder) BeamFnStateClient(org.apache.beam.fn.harness.state.BeamFnStateClient) CoderTranslation(org.apache.beam.runners.core.construction.CoderTranslation) Mockito.argThat(org.mockito.Mockito.argThat) DoFnSchemaInformation(org.apache.beam.sdk.transforms.DoFnSchemaInformation) BundleProcessorCache(org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessorCache) IsBounded(org.apache.beam.model.pipeline.v1.RunnerApi.IsBounded) Mockito.verifyNoMoreInteractions(org.mockito.Mockito.verifyNoMoreInteractions) Iterables(org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables) BundleProcessor(org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessor) REGISTERED_RUNNER_FACTORIES(org.apache.beam.fn.harness.control.ProcessBundleHandler.REGISTERED_RUNNER_FACTORIES) PaneInfo(org.apache.beam.sdk.transforms.windowing.PaneInfo) Collection(java.util.Collection) DoFnWithExecutionInformation(org.apache.beam.sdk.util.DoFnWithExecutionInformation) ModelCoders(org.apache.beam.runners.core.construction.ModelCoders) List(java.util.List) InstructionRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest) StateRequest(org.apache.beam.model.fnexecution.v1.BeamFnApi.StateRequest) SerializableUtils(org.apache.beam.sdk.util.SerializableUtils) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Trigger(org.apache.beam.model.pipeline.v1.RunnerApi.Trigger) Matchers.equalTo(org.hamcrest.Matchers.equalTo) StreamObserver(org.apache.beam.vendor.grpc.v1p43p2.io.grpc.stub.StreamObserver) WindowingStrategy(org.apache.beam.model.pipeline.v1.RunnerApi.WindowingStrategy) ThrowingRunnable(org.apache.beam.sdk.function.ThrowingRunnable) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Always(org.apache.beam.model.pipeline.v1.RunnerApi.Trigger.Always) PTransform(org.apache.beam.model.pipeline.v1.RunnerApi.PTransform) Assert.assertThrows(org.junit.Assert.assertThrows) IsEmptyCollection.empty(org.hamcrest.collection.IsEmptyCollection.empty) CompletableFuture(java.util.concurrent.CompletableFuture) BeamFnStateGrpcClientCache(org.apache.beam.fn.harness.state.BeamFnStateGrpcClientCache) PipelineOptionsFactory(org.apache.beam.sdk.options.PipelineOptionsFactory) BeamFnDataReadRunner(org.apache.beam.fn.harness.BeamFnDataReadRunner) StringUtf8Coder(org.apache.beam.sdk.coders.StringUtf8Coder) HashSet(java.util.HashSet) CacheToken(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleRequest.CacheToken) ParDoPayload(org.apache.beam.model.pipeline.v1.RunnerApi.ParDoPayload) InstructionResponse(org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionResponse) ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) ParDoTranslation(org.apache.beam.runners.core.construction.ParDoTranslation) Assert.assertNotNull(org.junit.Assert.assertNotNull) Mockito.when(org.mockito.Mockito.when) JUnit4(org.junit.runners.JUnit4) Mockito.verify(org.mockito.Mockito.verify) TimeUnit(java.util.concurrent.TimeUnit) Mockito(org.mockito.Mockito) Matchers.emptyIterable(org.hamcrest.Matchers.emptyIterable) Instant(org.joda.time.Instant) Caches(org.apache.beam.fn.harness.Caches) CallbackRegistration(org.apache.beam.fn.harness.control.FinalizeBundleHandler.CallbackRegistration) Collections(java.util.Collections) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) BeamFnApi(org.apache.beam.model.fnexecution.v1.BeamFnApi) ProcessBundleDescriptor(org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor) ByteString(org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString) ShortIdMap(org.apache.beam.runners.core.metrics.ShortIdMap) Test(org.junit.Test)

Aggregations

ProcessBundleDescriptor (org.apache.beam.model.fnexecution.v1.BeamFnApi.ProcessBundleDescriptor)21 Test (org.junit.Test)19 BeamFnApi (org.apache.beam.model.fnexecution.v1.BeamFnApi)18 ShortIdMap (org.apache.beam.runners.core.metrics.ShortIdMap)14 ByteString (org.apache.beam.vendor.grpc.v1p43p2.com.google.protobuf.ByteString)14 PTransformRunnerFactory (org.apache.beam.fn.harness.PTransformRunnerFactory)13 BundleProcessorCache (org.apache.beam.fn.harness.control.ProcessBundleHandler.BundleProcessorCache)13 Map (java.util.Map)8 RunnerApi (org.apache.beam.model.pipeline.v1.RunnerApi)8 IOException (java.io.IOException)7 BeamFnStateClient (org.apache.beam.fn.harness.state.BeamFnStateClient)6 Elements (org.apache.beam.model.fnexecution.v1.BeamFnApi.Elements)6 InstructionRequest (org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionRequest)6 InstructionResponse (org.apache.beam.model.fnexecution.v1.BeamFnApi.InstructionResponse)6 ImmutableMap (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableMap)6 Collections (java.util.Collections)5 List (java.util.List)5 Iterables (org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.Iterables)5 ArrayList (java.util.ArrayList)4 FunctionSpec (org.apache.beam.model.pipeline.v1.RunnerApi.FunctionSpec)4