Examples with ShortIdMap org.apache.beam.runners.core.metrics.ShortIdMap used on opensource projects

Example 6 with ShortIdMap

use of org.apache.beam.runners.core.metrics.ShortIdMap in project beam by apache.

the class ProcessBundleHandlerTest method testProgressBeforeBundleDoesNotFail.

@Test
public void testProgressBeforeBundleDoesNotFail() throws Exception {
    ProcessBundleHandler handler = new ProcessBundleHandler(PipelineOptionsFactory.create(), Collections.emptySet(), null, beamFnDataClient, null, /* beamFnStateClient */
    null, /* finalizeBundleHandler */
    new ShortIdMap(), ImmutableMap.of(), Caches.noop(), new BundleProcessorCache());
    handler.progress(BeamFnApi.InstructionRequest.newBuilder().setInstructionId("999L").setProcessBundleProgress(BeamFnApi.ProcessBundleProgressRequest.newBuilder().setInstructionId("unknown-id")).build());
    BeamFnApi.InstructionResponse response = handler.trySplit(BeamFnApi.InstructionRequest.newBuilder().setInstructionId("999L").setProcessBundleSplit(BeamFnApi.ProcessBundleSplitRequest.newBuilder().setInstructionId("unknown-id")).build()).build();
    assertNotNull(response.getProcessBundleProgress());
    assertEquals(0, response.getProcessBundleProgress().getMonitoringInfosCount());
}

Example 7 with ShortIdMap

use of org.apache.beam.runners.core.metrics.ShortIdMap in project beam by apache.

the class ProcessBundleHandlerTest method testInstructionIsUnregisteredFromBeamFnDataClientOnSuccess.

@Test
public void testInstructionIsUnregisteredFromBeamFnDataClientOnSuccess() throws Exception {
    BeamFnApi.ProcessBundleDescriptor processBundleDescriptor = BeamFnApi.ProcessBundleDescriptor.newBuilder().putTransforms("2L", RunnerApi.PTransform.newBuilder().setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build()).build()).build();
    Map<String, BeamFnApi.ProcessBundleDescriptor> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
    Mockito.doAnswer((invocation) -> {
        String instructionId = invocation.getArgument(0, String.class);
        CloseableFnDataReceiver<BeamFnApi.Elements> data = invocation.getArgument(2, CloseableFnDataReceiver.class);
        data.accept(BeamFnApi.Elements.newBuilder().addData(BeamFnApi.Elements.Data.newBuilder().setInstructionId(instructionId).setTransformId("2L").setIsLast(true)).build());
        return null;
    }).when(beamFnDataClient).registerReceiver(any(), any(), any());
    ProcessBundleHandler handler = new ProcessBundleHandler(PipelineOptionsFactory.create(), Collections.emptySet(), fnApiRegistry::get, beamFnDataClient, null, /* beamFnStateGrpcClientCache */
    null, /* finalizeBundleHandler */
    new ShortIdMap(), ImmutableMap.of(DATA_INPUT_URN, (PTransformRunnerFactory<Object>) (context) -> {
        context.addIncomingDataEndpoint(ApiServiceDescriptor.getDefaultInstance(), StringUtf8Coder.of(), (input) -> {
        });
        return null;
    }), Caches.noop(), new BundleProcessorCache());
    handler.processBundle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId("instructionId").setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("1L")).build());
    // Ensure that we unregister during successful processing
    verify(beamFnDataClient).registerReceiver(eq("instructionId"), any(), any());
    verify(beamFnDataClient).unregisterReceiver(eq("instructionId"), any());
    verifyNoMoreInteractions(beamFnDataClient);
}

Example 8 with ShortIdMap

use of org.apache.beam.runners.core.metrics.ShortIdMap in project beam by apache.

the class ProcessBundleHandlerTest method testStateCallsFailIfNoStateApiServiceDescriptorSpecified.

@Test
public void testStateCallsFailIfNoStateApiServiceDescriptorSpecified() throws Exception {
    BeamFnApi.ProcessBundleDescriptor processBundleDescriptor = BeamFnApi.ProcessBundleDescriptor.newBuilder().putTransforms("2L", RunnerApi.PTransform.newBuilder().setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build()).build()).build();
    Map<String, BeamFnApi.ProcessBundleDescriptor> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
    ProcessBundleHandler handler = new ProcessBundleHandler(PipelineOptionsFactory.create(), Collections.emptySet(), fnApiRegistry::get, beamFnDataClient, null, /* beamFnStateGrpcClientCache */
    null, /* finalizeBundleHandler */
    new ShortIdMap(), ImmutableMap.of(DATA_INPUT_URN, new PTransformRunnerFactory<Object>() {

        @Override
        public Object createRunnerForPTransform(Context context) throws IOException {
            BeamFnStateClient beamFnStateClient = context.getBeamFnStateClient();
            context.addStartBundleFunction(() -> doStateCalls(beamFnStateClient));
            return null;
        }

        @SuppressWarnings("FutureReturnValueIgnored")
        private void doStateCalls(BeamFnStateClient beamFnStateClient) {
            beamFnStateClient.handle(StateRequest.newBuilder().setInstructionId("SUCCESS"));
        }
    }), Caches.noop(), new BundleProcessorCache());
    assertThrows("State API calls are unsupported", IllegalStateException.class, () -> handler.processBundle(BeamFnApi.InstructionRequest.newBuilder().setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("1L")).build()));
}

Example 9 with ShortIdMap

use of org.apache.beam.runners.core.metrics.ShortIdMap in project beam by apache.

the class ProcessBundleHandlerTest method testOrderOfSetupTeardownCalls.

@Test
public void testOrderOfSetupTeardownCalls() throws Exception {
    DoFnWithExecutionInformation doFnWithExecutionInformation = DoFnWithExecutionInformation.of(new TestDoFn(), TestDoFn.mainOutput, Collections.emptyMap(), DoFnSchemaInformation.create());
    RunnerApi.FunctionSpec functionSpec = RunnerApi.FunctionSpec.newBuilder().setUrn(ParDoTranslation.CUSTOM_JAVA_DO_FN_URN).setPayload(ByteString.copyFrom(SerializableUtils.serializeToByteArray(doFnWithExecutionInformation))).build();
    RunnerApi.ParDoPayload parDoPayload = RunnerApi.ParDoPayload.newBuilder().setDoFn(functionSpec).build();
    BeamFnApi.ProcessBundleDescriptor processBundleDescriptor = BeamFnApi.ProcessBundleDescriptor.newBuilder().putTransforms("2L", PTransform.newBuilder().setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(DATA_INPUT_URN).build()).putOutputs("2L-output", "2L-output-pc").build()).putTransforms("3L", PTransform.newBuilder().setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(PTransformTranslation.PAR_DO_TRANSFORM_URN).setPayload(parDoPayload.toByteString())).putInputs("3L-input", "2L-output-pc").build()).putPcollections("2L-output-pc", PCollection.newBuilder().setWindowingStrategyId("window-strategy").setCoderId("2L-output-coder").setIsBounded(IsBounded.Enum.BOUNDED).build()).putWindowingStrategies("window-strategy", WindowingStrategy.newBuilder().setWindowCoderId("window-strategy-coder").setWindowFn(RunnerApi.FunctionSpec.newBuilder().setUrn("beam:window_fn:global_windows:v1")).setOutputTime(RunnerApi.OutputTime.Enum.END_OF_WINDOW).setAccumulationMode(RunnerApi.AccumulationMode.Enum.ACCUMULATING).setTrigger(RunnerApi.Trigger.newBuilder().setAlways(RunnerApi.Trigger.Always.getDefaultInstance())).setClosingBehavior(RunnerApi.ClosingBehavior.Enum.EMIT_ALWAYS).setOnTimeBehavior(RunnerApi.OnTimeBehavior.Enum.FIRE_ALWAYS).build()).putCoders("2L-output-coder", CoderTranslation.toProto(StringUtf8Coder.of()).getCoder()).putCoders("window-strategy-coder", Coder.newBuilder().setSpec(RunnerApi.FunctionSpec.newBuilder().setUrn(ModelCoders.GLOBAL_WINDOW_CODER_URN).build()).build()).build();
    Map<String, BeamFnApi.ProcessBundleDescriptor> fnApiRegistry = ImmutableMap.of("1L", processBundleDescriptor);
    Map<String, PTransformRunnerFactory> urnToPTransformRunnerFactoryMap = Maps.newHashMap(REGISTERED_RUNNER_FACTORIES);
    urnToPTransformRunnerFactoryMap.put(DATA_INPUT_URN, (context) -> null);
    ProcessBundleHandler handler = new ProcessBundleHandler(PipelineOptionsFactory.create(), Collections.emptySet(), fnApiRegistry::get, beamFnDataClient, null, /* beamFnStateClient */
    null, /* finalizeBundleHandler */
    new ShortIdMap(), urnToPTransformRunnerFactoryMap, Caches.noop(), new BundleProcessorCache());
    handler.processBundle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId("998L").setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("1L")).build());
    handler.processBundle(BeamFnApi.InstructionRequest.newBuilder().setInstructionId("999L").setProcessBundle(BeamFnApi.ProcessBundleRequest.newBuilder().setProcessBundleDescriptorId("1L")).build());
    handler.shutdown();
    // setup and teardown should occur only once when processing multiple bundles for the same
    // descriptor
    assertThat(TestDoFn.orderOfOperations, contains("setUp", "startBundle", "finishBundle", "startBundle", "finishBundle", "tearDown"));
}

Example 10 with ShortIdMap

use of org.apache.beam.runners.core.metrics.ShortIdMap in project beam by apache.

the class HarnessMonitoringInfosInstructionHandlerTest method testReturnsProcessWideMonitoringInfos.

@Test
public void testReturnsProcessWideMonitoringInfos() {
    MetricsEnvironment.setProcessWideContainer(MetricsContainerImpl.createProcessWideContainer());
    HashMap<String, String> labels = new HashMap<String, String>();
    labels.put(MonitoringInfoConstants.Labels.SERVICE, "service");
    labels.put(MonitoringInfoConstants.Labels.METHOD, "method");
    labels.put(MonitoringInfoConstants.Labels.RESOURCE, "resource");
    labels.put(MonitoringInfoConstants.Labels.PTRANSFORM, "transform");
    labels.put(MonitoringInfoConstants.Labels.STATUS, "ok");
    MonitoringInfoMetricName name = MonitoringInfoMetricName.named(MonitoringInfoConstants.Urns.API_REQUEST_COUNT, labels);
    Counter counter = LabeledMetrics.counter(name, true);
    counter.inc(7);
    ShortIdMap metricsShortIds = new ShortIdMap();
    HarnessMonitoringInfosInstructionHandler testObject = new HarnessMonitoringInfosInstructionHandler(metricsShortIds);
    BeamFnApi.InstructionRequest.Builder builder = BeamFnApi.InstructionRequest.newBuilder();
    BeamFnApi.InstructionResponse.Builder responseBuilder = testObject.harnessMonitoringInfos(builder.build());
    BeamFnApi.InstructionResponse response = responseBuilder.build();
    assertEquals(1, response.getHarnessMonitoringInfos().getMonitoringDataMap().size());
    // Expect a payload to be set for "metric0".
    assertTrue(!response.getHarnessMonitoringInfos().getMonitoringDataMap().get("metric0").isEmpty());
}