Examples with MonitoringInfo org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo used on opensource projects

Example 6 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class PCollectionConsumerRegistryTest method singleConsumer.

@Test
public void singleConsumer() throws Exception {
    final String pCollectionA = "pCollectionA";
    final String pTransformIdA = "pTransformIdA";
    MetricsContainerStepMap metricsContainerRegistry = new MetricsContainerStepMap();
    PCollectionConsumerRegistry consumers = new PCollectionConsumerRegistry(metricsContainerRegistry, mock(ExecutionStateTracker.class));
    FnDataReceiver<WindowedValue<String>> consumerA1 = mock(FnDataReceiver.class);
    consumers.register(pCollectionA, pTransformIdA, consumerA1, StringUtf8Coder.of());
    FnDataReceiver<WindowedValue<String>> wrapperConsumer = (FnDataReceiver<WindowedValue<String>>) (FnDataReceiver) consumers.getMultiplexingConsumer(pCollectionA);
    String elementValue = "elem";
    WindowedValue<String> element = valueInGlobalWindow(elementValue);
    int numElements = 10;
    for (int i = 0; i < numElements; i++) {
        wrapperConsumer.accept(element);
    }
    // Check that the underlying consumers are each invoked per element.
    verify(consumerA1, times(numElements)).accept(element);
    assertThat(consumers.keySet(), contains(pCollectionA));
    List<MonitoringInfo> expected = new ArrayList<>();
    SimpleMonitoringInfoBuilder builder = new SimpleMonitoringInfoBuilder();
    builder.setUrn(MonitoringInfoConstants.Urns.ELEMENT_COUNT);
    builder.setLabel(MonitoringInfoConstants.Labels.PCOLLECTION, pCollectionA);
    builder.setInt64SumValue(numElements);
    expected.add(builder.build());
    long elementByteSize = StringUtf8Coder.of().getEncodedElementByteSize(elementValue);
    builder = new SimpleMonitoringInfoBuilder();
    builder.setUrn(Urns.SAMPLED_BYTE_SIZE);
    builder.setLabel(MonitoringInfoConstants.Labels.PCOLLECTION, pCollectionA);
    builder.setInt64DistributionValue(DistributionData.create(numElements * elementByteSize, numElements, elementByteSize, elementByteSize));
    expected.add(builder.build());
    // Clear the timestamp before comparison.
    Iterable<MonitoringInfo> result = Iterables.filter(metricsContainerRegistry.getMonitoringInfos(), monitoringInfo -> monitoringInfo.containsLabels(Labels.PCOLLECTION));
    assertThat(result, containsInAnyOrder(expected.toArray()));
}

Example 7 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class SimpleMonitoringInfoBuilderTest method testUserDistribution.

@Test
public void testUserDistribution() throws Exception {
    SimpleMonitoringInfoBuilder builder = new SimpleMonitoringInfoBuilder();
    builder.setUrn(MonitoringInfoConstants.Urns.USER_DISTRIBUTION_INT64);
    builder.setLabel(MonitoringInfoConstants.Labels.NAME, "myName");
    builder.setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "myNamespace");
    builder.setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "myStep");
    assertNull(builder.build());
    builder.setInt64DistributionValue(DistributionData.create(10, 2, 1, 9));
    // Pass now that the spec is fully met.
    MonitoringInfo monitoringInfo = builder.build();
    assertTrue(monitoringInfo != null);
    assertEquals(MonitoringInfoConstants.Urns.USER_DISTRIBUTION_INT64, monitoringInfo.getUrn());
    assertEquals("myName", monitoringInfo.getLabelsOrDefault(MonitoringInfoConstants.Labels.NAME, ""));
    assertEquals("myNamespace", monitoringInfo.getLabelsOrDefault(MonitoringInfoConstants.Labels.NAMESPACE, ""));
    assertEquals(MonitoringInfoConstants.TypeUrns.DISTRIBUTION_INT64_TYPE, monitoringInfo.getType());
    DistributionData data = MonitoringInfoEncodings.decodeInt64Distribution(monitoringInfo.getPayload());
    assertEquals(10L, data.sum());
    assertEquals(2L, data.count());
    assertEquals(9L, data.max());
    assertEquals(1L, data.min());
}

Example 8 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class SpecMonitoringInfoValidatorTest method validateReturnsNoErrorOnValidMonitoringInfo.

@Test
public void validateReturnsNoErrorOnValidMonitoringInfo() {
    MonitoringInfo testInput = MonitoringInfo.newBuilder().setUrn(Urns.USER_SUM_INT64).putLabels(MonitoringInfoConstants.Labels.NAME, "anyCounter").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "").putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyString").setType(TypeUrns.SUM_INT64_TYPE).build();
    assertFalse(new SpecMonitoringInfoValidator().validate(testInput).isPresent());
    testInput = MonitoringInfo.newBuilder().setUrn(Urns.USER_DISTRIBUTION_INT64).putLabels(MonitoringInfoConstants.Labels.NAME, "anyDistribution").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "namespace").putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyString").setType(TypeUrns.DISTRIBUTION_INT64_TYPE).putLabels("dummy", "value").build();
    assertFalse(new SpecMonitoringInfoValidator().validate(testInput).isPresent());
    testInput = MonitoringInfo.newBuilder().setUrn(MonitoringInfoConstants.Urns.ELEMENT_COUNT).setType(TypeUrns.SUM_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "value").putLabels(MonitoringInfoConstants.Labels.PCOLLECTION, "anotherValue").build();
    assertFalse(new SpecMonitoringInfoValidator().validate(testInput).isPresent());
}

Example 9 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class UserMonitoringInfoToCounterUpdateTransformerTest method testTransformThrowsIfMonitoringInfoWithWrongUrnPrefixReceived.

@Test
public void testTransformThrowsIfMonitoringInfoWithWrongUrnPrefixReceived() {
    Map<String, DataflowStepContext> stepContextMapping = new HashMap<>();
    MonitoringInfo monitoringInfo = MonitoringInfo.newBuilder().setUrn(Urns.ELEMENT_COUNT).setType(TypeUrns.SUM_INT64_TYPE).build();
    UserMonitoringInfoToCounterUpdateTransformer testObject = new UserMonitoringInfoToCounterUpdateTransformer(mockSpecValidator, stepContextMapping);
    when(mockSpecValidator.validate(any())).thenReturn(Optional.empty());
    exception.expect(RuntimeException.class);
    testObject.transform(monitoringInfo);
}

Example 10 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class UserMonitoringInfoToCounterUpdateTransformerTest method testTransformReturnsValidCounterUpdateWhenValidUserMonitoringInfoReceived.

@Test
public void testTransformReturnsValidCounterUpdateWhenValidUserMonitoringInfoReceived() throws Exception {
    Map<String, DataflowStepContext> stepContextMapping = new HashMap<>();
    NameContext nc = NameContext.create("anyStageName", "anyOriginalName", "anySystemName", "anyUserName");
    DataflowStepContext dsc = mock(DataflowStepContext.class);
    when(dsc.getNameContext()).thenReturn(nc);
    stepContextMapping.put("anyValue", dsc);
    MonitoringInfo monitoringInfo = MonitoringInfo.newBuilder().setUrn(Urns.USER_SUM_INT64).setType(TypeUrns.SUM_INT64_TYPE).putLabels(MonitoringInfoConstants.Labels.NAME, "anyName").putLabels(MonitoringInfoConstants.Labels.NAMESPACE, "anyNamespace").putLabels(MonitoringInfoConstants.Labels.PTRANSFORM, "anyValue").setPayload(encodeInt64Counter(1L)).build();
    UserMonitoringInfoToCounterUpdateTransformer testObject = new UserMonitoringInfoToCounterUpdateTransformer(mockSpecValidator, stepContextMapping);
    when(mockSpecValidator.validate(any())).thenReturn(Optional.empty());
    CounterUpdate result = testObject.transform(monitoringInfo);
    assertNotNull(result);
    assertEqualsAsJson("{cumulative:true, integer:{highBits:0, lowBits:1}, " + "structuredNameAndMetadata:{metadata:{kind:'SUM'}, " + "name:{name:'anyName', origin:'USER', originNamespace:'anyNamespace', " + "originalStepName:'anyOriginalName'}}}", result);
}