Examples with MonitoringInfo org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo used on opensource projects

Example 11 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class BeamFnMapTaskExecutorTest method testExtractCounterUpdatesReturnsValidProgressTrackerCounterUpdatesIfPresent.

@Test(timeout = ReadOperation.DEFAULT_PROGRESS_UPDATE_PERIOD_MS * 60)
public void testExtractCounterUpdatesReturnsValidProgressTrackerCounterUpdatesIfPresent() throws Exception {
    final CountDownLatch progressSentLatch = new CountDownLatch(1);
    final CountDownLatch processBundleLatch = new CountDownLatch(1);
    final int expectedCounterValue = 5;
    final MonitoringInfo expectedMonitoringInfo = MonitoringInfo.newBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).putLabels(MonitoringInfoConstants.Labels.NAME, "ExpectedCounter").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "anyString").setType(MonitoringInfoConstants.TypeUrns.SUM_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "ExpectedPTransform").setPayload(encodeInt64Counter(expectedCounterValue)).build();
    InstructionRequestHandler instructionRequestHandler = new InstructionRequestHandler() {

        @Override
        public CompletionStage<InstructionResponse> handle(InstructionRequest request) {
            switch(request.getRequestCase()) {
                case REGISTER:
                    return CompletableFuture.completedFuture(responseFor(request).build());
                case PROCESS_BUNDLE:
                    return MoreFutures.supplyAsync(() -> {
                        processBundleLatch.await();
                        return responseFor(request).setProcessBundle(BeamFnApi.ProcessBundleResponse.newBuilder().addMonitoringInfos(expectedMonitoringInfo)).build();
                    });
                case PROCESS_BUNDLE_PROGRESS:
                    progressSentLatch.countDown();
                    return CompletableFuture.completedFuture(responseFor(request).setProcessBundleProgress(BeamFnApi.ProcessBundleProgressResponse.newBuilder().addMonitoringInfos(expectedMonitoringInfo)).build());
                default:
                    throw new RuntimeException("Reached unexpected code path");
            }
        }

        @Override
        public void registerProcessBundleDescriptor(ProcessBundleDescriptor descriptor) {
        }

        @Override
        public void close() {
        }
    };
    Map<String, DataflowStepContext> stepContextMap = new HashMap<>();
    stepContextMap.put("ExpectedPTransform", generateDataflowStepContext("Expected"));
    RegisterAndProcessBundleOperation processOperation = new RegisterAndProcessBundleOperation(IdGenerators.decrementingLongs(), instructionRequestHandler, mockBeamFnStateDelegator, REGISTER_REQUEST, ImmutableMap.of(), stepContextMap, ImmutableMap.of(), ImmutableTable.of(), ImmutableMap.of(), mockContext);
    BeamFnMapTaskExecutor mapTaskExecutor = BeamFnMapTaskExecutor.forOperations(ImmutableList.of(readOperation, grpcPortWriteOperation, processOperation), executionStateTracker);
    // Launch the BeamFnMapTaskExecutor and wait until we are sure there has been one
    // tentative update
    CompletionStage<Void> doneFuture = MoreFutures.runAsync(mapTaskExecutor::execute);
    progressSentLatch.await();
    Iterable<CounterUpdate> metricsCounterUpdates = Collections.emptyList();
    while (Iterables.size(metricsCounterUpdates) == 0) {
        Thread.sleep(ReadOperation.DEFAULT_PROGRESS_UPDATE_PERIOD_MS);
        metricsCounterUpdates = mapTaskExecutor.extractMetricUpdates();
    }
    // Get the final metrics
    processBundleLatch.countDown();
    MoreFutures.get(doneFuture);
    metricsCounterUpdates = mapTaskExecutor.extractMetricUpdates();
    assertThat(Iterables.size(metricsCounterUpdates), equalTo(1));
    CounterUpdate resultCounter = metricsCounterUpdates.iterator().next();
    assertTrue(new CounterHamcrestMatchers.CounterUpdateIntegerValueMatcher(expectedCounterValue).matches(resultCounter));
    assertEquals("ExpectedCounter", resultCounter.getStructuredNameAndMetadata().getName().getName());
}

Example 12 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class FnApiMonitoringInfoToCounterUpdateTransformerTest method testTransformUtilizesRelevantCounterTransformer.

@Test
public void testTransformUtilizesRelevantCounterTransformer() {
    Map<String, MonitoringInfoToCounterUpdateTransformer> genericTransformers = new HashMap<>();
    final String validUrn = "urn1";
    final String validType = "type1";
    genericTransformers.put(validUrn, mockTransformer1);
    genericTransformers.put("any:other:urn", mockTransformer2);
    FnApiMonitoringInfoToCounterUpdateTransformer testObject = new FnApiMonitoringInfoToCounterUpdateTransformer(genericTransformers);
    CounterUpdate expectedResult = new CounterUpdate();
    when(mockTransformer1.transform(any())).thenReturn(expectedResult);
    MonitoringInfo monitoringInfo = MonitoringInfo.newBuilder().setUrn(validUrn).setType(validType).putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyValue").build();
    CounterUpdate result = testObject.transform(monitoringInfo);
    assertSame(expectedResult, result);
    verifyZeroInteractions(mockTransformer2);
}

Example 13 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class UserDistributionMonitoringInfoToCounterUpdateTransformerTest method testTransformReturnsValidCounterUpdateWhenValidUserMonitoringInfoReceived.

@Test
public void testTransformReturnsValidCounterUpdateWhenValidUserMonitoringInfoReceived() throws Exception {
    Map<String, DataflowStepContext> stepContextMapping = new HashMap<>();
    NameContext nc = NameContext.create("anyStageName", "anyOriginalName", "anySystemName", "anyUserName");
    DataflowStepContext dsc = mock(DataflowStepContext.class);
    when(dsc.getNameContext()).thenReturn(nc);
    stepContextMapping.put("anyValue", dsc);
    MonitoringInfo monitoringInfo = MonitoringInfo.newBuilder().setUrn(Urns.USER_DISTRIBUTION_INT64).setType(TypeUrns.DISTRIBUTION_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.NAME, "anyName").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "anyNamespace").putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyValue").setPayload(encodeInt64Distribution(DistributionData.create(2L, /* sum */
    1L, /* count */
    3L, /* min */
    4L))).build();
    UserDistributionMonitoringInfoToCounterUpdateTransformer testObject = new UserDistributionMonitoringInfoToCounterUpdateTransformer(mockSpecValidator, stepContextMapping);
    when(mockSpecValidator.validate(any())).thenReturn(Optional.empty());
    CounterUpdate result = testObject.transform(monitoringInfo);
    assertNotNull(result);
    assertEqualsAsJson("{cumulative:true, distribution:{count:{highBits:0, lowBits:1}, " + "max:{highBits:0, lowBits:4}, min:{highBits:0, lowBits:3}, " + "sum:{highBits:0, lowBits:2}}, " + "structuredNameAndMetadata:{metadata:{kind:'DISTRIBUTION'}, " + "name:{name:'anyName', origin:'USER', originNamespace:'anyNamespace', " + "originalStepName:'anyOriginalName'}}}", result);
}

Example 14 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class ExecutionTimeMonitoringInfoToCounterUpdateTransformerTest method testTransformReturnsNullIfMonitoringInfoWithUnknownPTransformLabelPresent.

@Test
public void testTransformReturnsNullIfMonitoringInfoWithUnknownPTransformLabelPresent() {
    Map<String, DataflowStepContext> stepContextMapping = new HashMap<>();
    ExecutionTimeMonitoringInfoToCounterUpdateTransformer testObject = new ExecutionTimeMonitoringInfoToCounterUpdateTransformer(mockSpecValidator, stepContextMapping);
    when(mockSpecValidator.validate(any())).thenReturn(Optional.empty());
    MonitoringInfo monitoringInfo = MonitoringInfo.newBuilder().setUrn(Urns.PROCESS_BUNDLE_MSECS).setType(TypeUrns.SUM_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyValue").build();
    assertNull(testObject.transform(monitoringInfo));
}

Example 15 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class ExecutionTimeMonitoringInfoToCounterUpdateTransformerTest method testTransformReturnsNullIfSpecValidationFails.

@Test
public void testTransformReturnsNullIfSpecValidationFails() {
    Map<String, DataflowStepContext> stepContextMapping = new HashMap<>();
    ExecutionTimeMonitoringInfoToCounterUpdateTransformer testObject = new ExecutionTimeMonitoringInfoToCounterUpdateTransformer(mockSpecValidator, stepContextMapping);
    Optional<String> error = Optional.of("Error text");
    when(mockSpecValidator.validate(any())).thenReturn(error);
    MonitoringInfo monitoringInfo = MonitoringInfo.newBuilder().setUrn(Urns.USER_SUM_INT64).setType(TypeUrns.SUM_INT64_TYPE).build();
    assertNull(testObject.transform(monitoringInfo));
}