Examples with DistributionResult org.apache.beam.sdk.metrics.DistributionResult used on opensource projects

Example 6 with DistributionResult

use of org.apache.beam.sdk.metrics.DistributionResult in project beam by apache.

the class SparkBeamMetric method renderAll.

Map<String, ?> renderAll() {
    Map<String, Object> metrics = new HashMap<>();
    MetricResults metricResults = asAttemptedOnlyMetricResults(MetricsAccumulator.getInstance().value());
    MetricQueryResults metricQueryResults = metricResults.queryMetrics(MetricsFilter.builder().build());
    for (MetricResult<Long> metricResult : metricQueryResults.counters()) {
        metrics.put(renderName(metricResult), metricResult.attempted());
    }
    for (MetricResult<DistributionResult> metricResult : metricQueryResults.distributions()) {
        DistributionResult result = metricResult.attempted();
        metrics.put(renderName(metricResult) + ".count", result.count());
        metrics.put(renderName(metricResult) + ".sum", result.sum());
        metrics.put(renderName(metricResult) + ".min", result.min());
        metrics.put(renderName(metricResult) + ".max", result.max());
        metrics.put(renderName(metricResult) + ".mean", result.mean());
    }
    for (MetricResult<GaugeResult> metricResult : metricQueryResults.gauges()) {
        metrics.put(renderName(metricResult), metricResult.attempted().value());
    }
    return metrics;
}

Example 7 with DistributionResult

use of org.apache.beam.sdk.metrics.DistributionResult in project flink by apache.

the class FlinkMetricContainer method updateDistributions.

private void updateDistributions(Iterable<MetricResult<DistributionResult>> distributions) {
    for (MetricResult<DistributionResult> metricResult : distributions) {
        if (!isUserMetric(metricResult)) {
            continue;
        }
        // get identifier
        String flinkMetricIdentifier = getFlinkMetricIdentifierString(metricResult.getKey());
        DistributionResult update = metricResult.getAttempted();
        // update flink metric
        FlinkDistributionGauge gauge = flinkDistributionGaugeCache.get(flinkMetricIdentifier);
        if (gauge == null) {
            MetricGroup metricGroup = registerMetricGroup(metricResult.getKey(), baseMetricGroup);
            gauge = metricGroup.gauge(metricResult.getKey().metricName().getName(), new FlinkDistributionGauge(update));
            flinkDistributionGaugeCache.put(flinkMetricIdentifier, gauge);
        } else {
            gauge.update(update);
        }
    }
}

Example 8 with DistributionResult

use of org.apache.beam.sdk.metrics.DistributionResult in project beam by apache.

the class FlinkMetricContainer method updateDistributions.

private void updateDistributions(Iterable<MetricResult<DistributionResult>> distributions) {
    for (MetricResult<DistributionResult> metricResult : distributions) {
        String flinkMetricName = getFlinkMetricNameString(metricResult.getKey());
        DistributionResult update = metricResult.getAttempted();
        // update flink metric
        FlinkDistributionGauge gauge = flinkDistributionGaugeCache.get(flinkMetricName);
        if (gauge == null) {
            gauge = runtimeContext.getMetricGroup().gauge(flinkMetricName, new FlinkDistributionGauge(update));
            flinkDistributionGaugeCache.put(flinkMetricName, gauge);
        } else {
            gauge.update(update);
        }
    }
}

Example 9 with DistributionResult

use of org.apache.beam.sdk.metrics.DistributionResult in project beam by apache.

the class FlinkMetricContainerTest method testDropUnexpectedMonitoringInfoTypes.

@Test
public void testDropUnexpectedMonitoringInfoTypes() {
    MetricsContainerImpl step = container.getMetricsContainer("step");
    MonitoringInfo intCounter = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns1").setLabel(MonitoringInfoConstants.Labels.NAME, "int_counter").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setInt64SumValue(111).build();
    MonitoringInfo doubleCounter = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_DOUBLE).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns2").setLabel(MonitoringInfoConstants.Labels.NAME, "double_counter").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setDoubleSumValue(222).build();
    MonitoringInfo intDistribution = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_DISTRIBUTION_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns3").setLabel(MonitoringInfoConstants.Labels.NAME, "int_distribution").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setInt64DistributionValue(DistributionData.create(30, 10, 1, 5)).build();
    MonitoringInfo doubleDistribution = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_DISTRIBUTION_DOUBLE).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns4").setLabel(MonitoringInfoConstants.Labels.NAME, "double_distribution").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setDoubleDistributionValue(10, 30, 1, 5).build();
    // Mock out the counter that Flink returns; the distribution gets created by
    // FlinkMetricContainer, not by Flink itself, so we verify it in a different way below
    SimpleCounter counter = new SimpleCounter();
    when(metricGroup.counter("ns1.int_counter")).thenReturn(counter);
    container.updateMetrics("step", ImmutableList.of(intCounter, doubleCounter, intDistribution, doubleDistribution));
    // Flink's MetricGroup should only have asked for one counter (the integer-typed one) to be
    // created (the double-typed one is dropped currently)
    verify(metricGroup).counter(eq("ns1.int_counter"));
    // Verify that the counter injected into flink has the right value
    assertThat(counter.getCount(), is(111L));
    // Verify the counter in the java SDK MetricsContainer
    long count = ((CounterCell) step.tryGetCounter(MonitoringInfoMetricName.of(intCounter))).getCumulative();
    assertThat(count, is(111L));
    // The one Flink distribution that gets created is a FlinkDistributionGauge; here we verify its
    // initial (and in this test, final) value
    verify(metricGroup).gauge(eq("ns3.int_distribution"), argThat(new ArgumentMatcher<FlinkDistributionGauge>() {

        @Override
        public boolean matches(FlinkDistributionGauge argument) {
            DistributionResult actual = ((FlinkDistributionGauge) argument).getValue();
            DistributionResult expected = DistributionResult.create(30, 10, 1, 5);
            return actual.equals(expected);
        }
    }));
    // Verify that the Java SDK MetricsContainer holds the same information
    DistributionData distributionData = ((DistributionCell) step.getDistribution(MonitoringInfoMetricName.of(intDistribution))).getCumulative();
    assertThat(distributionData, is(DistributionData.create(30, 10, 1, 5)));
}