Examples with MonitoringInfo org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo used on opensource projects

Example 1 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project flink by apache.

the class FlinkMetricContainerTest method testCounterMonitoringInfoUpdate.

@Test
public void testCounterMonitoringInfoUpdate() {
    SimpleCounter userCounter = new SimpleCounter();
    when(metricGroup.counter("myCounter")).thenReturn(userCounter);
    MonitoringInfo userMonitoringInfo = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, DEFAULT_NAMESPACE).setLabel(MonitoringInfoConstants.Labels.NAME, "myCounter").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "anyPTransform").setInt64SumValue(111).build();
    assertThat(userCounter.getCount(), is(0L));
    container.updateMetrics("step", ImmutableList.of(userMonitoringInfo));
    assertThat(userCounter.getCount(), is(111L));
}

Example 2 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project flink by apache.

the class FlinkMetricContainerTest method testGaugeMonitoringInfoUpdate.

@Test
public void testGaugeMonitoringInfoUpdate() {
    MonitoringInfo userMonitoringInfo = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, DEFAULT_NAMESPACE).setLabel(MonitoringInfoConstants.Labels.NAME, "myGauge").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "anyPTransform").setInt64LatestValue(GaugeData.create(111L)).build();
    container.updateMetrics("step", ImmutableList.of(userMonitoringInfo));
    verify(metricGroup).gauge(eq("myGauge"), argThat((ArgumentMatcher<FlinkMetricContainer.FlinkGauge>) argument -> {
        Long actual = argument.getValue();
        return actual.equals(111L);
    }));
}

Example 3 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project flink by apache.

the class FlinkMetricContainerTest method testMeterMonitoringInfoUpdate.

@Test
public void testMeterMonitoringInfoUpdate() {
    MeterView userMeter = new MeterView(new SimpleCounter());
    when(metricGroup.meter(eq("myMeter"), any(Meter.class))).thenReturn(userMeter);
    String namespace = "[\"key\", \"value\", \"MetricGroupType.key\", \"MetricGroupType.value\", \"60\"]";
    MonitoringInfo userMonitoringInfo = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, namespace).setLabel(MonitoringInfoConstants.Labels.NAME, "myMeter").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "anyPTransform").setInt64SumValue(111).build();
    assertThat(userMeter.getCount(), is(0L));
    assertThat(userMeter.getRate(), is(0.0));
    container.updateMetrics("step", ImmutableList.of(userMonitoringInfo));
    userMeter.update();
    assertThat(userMeter.getCount(), is(111L));
    // 111 div 60 = 1.85
    assertThat(userMeter.getRate(), is(1.85));
}

Example 4 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project flink by apache.

the class FlinkMetricContainerTest method testDistributionMonitoringInfoUpdate.

@Test
public void testDistributionMonitoringInfoUpdate() {
    MonitoringInfo userMonitoringInfo = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_DISTRIBUTION_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, DEFAULT_NAMESPACE).setLabel(MonitoringInfoConstants.Labels.NAME, "myDistribution").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "anyPTransform").setInt64DistributionValue(DistributionData.create(30, 10, 1, 5)).build();
    container.updateMetrics("step", ImmutableList.of(userMonitoringInfo));
    // The one Flink distribution that gets created is a FlinkDistributionGauge; here we verify
    // its initial (and in this test, final) value
    verify(metricGroup).gauge(eq("myDistribution"), argThat((ArgumentMatcher<FlinkMetricContainer.FlinkDistributionGauge>) argument -> {
        DistributionResult actual = argument.getValue();
        DistributionResult expected = DistributionResult.create(30, 10, 1, 5);
        return actual.equals(expected);
    }));
}

Example 5 with MonitoringInfo

use of org.apache.beam.model.pipeline.v1.MetricsApi.MonitoringInfo in project beam by apache.

the class PCollectionConsumerRegistryTest method testLazyByteSizeEstimation.

@Test
public void testLazyByteSizeEstimation() throws Exception {
    final String pCollectionA = "pCollectionA";
    final String pTransformIdA = "pTransformIdA";
    MetricsContainerStepMap metricsContainerRegistry = new MetricsContainerStepMap();
    PCollectionConsumerRegistry consumers = new PCollectionConsumerRegistry(metricsContainerRegistry, mock(ExecutionStateTracker.class));
    FnDataReceiver<WindowedValue<Iterable<String>>> consumerA1 = mock(FnDataReceiver.class);
    consumers.register(pCollectionA, pTransformIdA, consumerA1, IterableCoder.of(StringUtf8Coder.of()));
    FnDataReceiver<WindowedValue<Iterable<String>>> wrapperConsumer = (FnDataReceiver<WindowedValue<Iterable<String>>>) (FnDataReceiver) consumers.getMultiplexingConsumer(pCollectionA);
    String elementValue = "elem";
    long elementByteSize = StringUtf8Coder.of().getEncodedElementByteSize(elementValue);
    WindowedValue<Iterable<String>> element = valueInGlobalWindow(new TestElementByteSizeObservableIterable<>(Arrays.asList(elementValue, elementValue), elementByteSize));
    int numElements = 10;
    // Mock doing work on the iterable items
    doAnswer((Answer<Void>) invocation -> {
        Object[] args = invocation.getArguments();
        WindowedValue<Iterable<String>> arg = (WindowedValue<Iterable<String>>) args[0];
        Iterator it = arg.getValue().iterator();
        while (it.hasNext()) {
            it.next();
        }
        return null;
    }).when(consumerA1).accept(element);
    for (int i = 0; i < numElements; i++) {
        wrapperConsumer.accept(element);
    }
    // Check that the underlying consumers are each invoked per element.
    verify(consumerA1, times(numElements)).accept(element);
    assertThat(consumers.keySet(), contains(pCollectionA));
    List<MonitoringInfo> expected = new ArrayList<>();
    SimpleMonitoringInfoBuilder builder = new SimpleMonitoringInfoBuilder();
    builder.setUrn(MonitoringInfoConstants.Urns.ELEMENT_COUNT);
    builder.setLabel(MonitoringInfoConstants.Labels.PCOLLECTION, pCollectionA);
    builder.setInt64SumValue(numElements);
    expected.add(builder.build());
    builder = new SimpleMonitoringInfoBuilder();
    builder.setUrn(Urns.SAMPLED_BYTE_SIZE);
    builder.setLabel(MonitoringInfoConstants.Labels.PCOLLECTION, pCollectionA);
    long expectedBytes = (elementByteSize + 1) * 2 + // Additional 5 bytes are due to size and hasNext = false (1 byte).
    5;
    builder.setInt64DistributionValue(DistributionData.create(numElements * expectedBytes, numElements, expectedBytes, expectedBytes));
    expected.add(builder.build());
    // Clear the timestamp before comparison.
    Iterable<MonitoringInfo> result = Iterables.filter(metricsContainerRegistry.getMonitoringInfos(), monitoringInfo -> monitoringInfo.containsLabels(Labels.PCOLLECTION));
    assertThat(result, containsInAnyOrder(expected.toArray()));
}