Examples with DistributionData org.apache.beam.runners.core.metrics.DistributionData used on opensource projects

Example 1 with DistributionData

use of org.apache.beam.runners.core.metrics.DistributionData in project beam by apache.

the class PortableMetrics method convertDistributionMonitoringInfoToDistribution.

private static MetricResult<DistributionResult> convertDistributionMonitoringInfoToDistribution(MetricsApi.MonitoringInfo monitoringInfo) {
    Map<String, String> labelsMap = monitoringInfo.getLabelsMap();
    MetricKey key = MetricKey.create(labelsMap.get(STEP_NAME_LABEL), MetricName.named(labelsMap.get(NAMESPACE_LABEL), labelsMap.get(METRIC_NAME_LABEL)));
    DistributionData data = decodeInt64Distribution(monitoringInfo.getPayload());
    DistributionResult result = DistributionResult.create(data.sum(), data.count(), data.min(), data.max());
    return MetricResult.create(key, false, result);
}

Example 2 with DistributionData

use of org.apache.beam.runners.core.metrics.DistributionData in project beam by apache.

the class MeanByteCountMonitoringInfoToCounterUpdateTransformer method transform.

/**
 * Generates CounterUpdate to send to DFE based on ElementCount MonitoringInfo.
 *
 * @param monitoringInfo Monitoring info to transform.
 * @return CounterUpdate generated based on provided monitoringInfo
 */
@Override
@Nullable
public CounterUpdate transform(MonitoringInfo monitoringInfo) {
    Optional<String> validationResult = validate(monitoringInfo);
    if (validationResult.isPresent()) {
        LOG.debug(validationResult.get());
        return null;
    }
    DistributionData data = decodeInt64Distribution(monitoringInfo.getPayload());
    final String pcollectionId = monitoringInfo.getLabelsMap().get(MonitoringInfoConstants.Labels.PCOLLECTION);
    final String pcollectionName = pcollectionIdToNameContext.get(pcollectionId).userName();
    String counterName = pcollectionName + "-MeanByteCount";
    NameAndKind name = new NameAndKind();
    name.setName(counterName).setKind(Kind.MEAN.toString());
    return new CounterUpdate().setNameAndKind(name).setCumulative(true).setIntegerMean(new IntegerMean().setSum(longToSplitInt(data.sum())).setCount(longToSplitInt(data.count())));
}

Example 3 with DistributionData

use of org.apache.beam.runners.core.metrics.DistributionData in project beam by apache.

the class UserDistributionMonitoringInfoToCounterUpdateTransformer method transform.

/**
 * Transforms user counter MonitoringInfo to relevant CounterUpdate.
 *
 * @return Relevant CounterUpdate or null if transformation failed.
 */
@Override
@Nullable
public CounterUpdate transform(MonitoringInfo monitoringInfo) {
    Optional<String> validationResult = validate(monitoringInfo);
    if (validationResult.isPresent()) {
        LOG.debug(validationResult.get());
        return null;
    }
    DistributionData data = decodeInt64Distribution(monitoringInfo.getPayload());
    Map<String, String> miLabels = monitoringInfo.getLabelsMap();
    final String ptransform = miLabels.get(MonitoringInfoConstants.Labels.PTRANSFORM);
    final String counterName = miLabels.get(MonitoringInfoConstants.Labels.NAME);
    final String counterNamespace = miLabels.get(MonitoringInfoConstants.Labels.NAMESPACE);
    CounterStructuredNameAndMetadata name = new CounterStructuredNameAndMetadata();
    DataflowStepContext stepContext = transformIdMapping.get(ptransform);
    name.setName(new CounterStructuredName().setOrigin(Origin.USER.toString()).setName(counterName).setOriginalStepName(stepContext.getNameContext().originalName()).setOriginNamespace(counterNamespace)).setMetadata(new CounterMetadata().setKind(Kind.DISTRIBUTION.toString()));
    return new CounterUpdate().setStructuredNameAndMetadata(name).setCumulative(true).setDistribution(new DistributionUpdate().setMax(DataflowCounterUpdateExtractor.longToSplitInt(data.max())).setMin(DataflowCounterUpdateExtractor.longToSplitInt(data.min())).setSum(DataflowCounterUpdateExtractor.longToSplitInt(data.sum())).setCount(DataflowCounterUpdateExtractor.longToSplitInt(data.count())));
}

Example 4 with DistributionData

use of org.apache.beam.runners.core.metrics.DistributionData in project beam by apache.

the class FlinkMetricContainerTest method testDropUnexpectedMonitoringInfoTypes.

@Test
public void testDropUnexpectedMonitoringInfoTypes() {
    MetricsContainerImpl step = container.getMetricsContainer("step");
    MonitoringInfo intCounter = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns1").setLabel(MonitoringInfoConstants.Labels.NAME, "int_counter").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setInt64SumValue(111).build();
    MonitoringInfo doubleCounter = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_SUM_DOUBLE).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns2").setLabel(MonitoringInfoConstants.Labels.NAME, "double_counter").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setDoubleSumValue(222).build();
    MonitoringInfo intDistribution = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_DISTRIBUTION_INT64).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns3").setLabel(MonitoringInfoConstants.Labels.NAME, "int_distribution").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setInt64DistributionValue(DistributionData.create(30, 10, 1, 5)).build();
    MonitoringInfo doubleDistribution = new SimpleMonitoringInfoBuilder().setUrn(MonitoringInfoConstants.Urns.USER_DISTRIBUTION_DOUBLE).setLabel(MonitoringInfoConstants.Labels.NAMESPACE, "ns4").setLabel(MonitoringInfoConstants.Labels.NAME, "double_distribution").setLabel(MonitoringInfoConstants.Labels.PTRANSFORM, "step").setDoubleDistributionValue(10, 30, 1, 5).build();
    // Mock out the counter that Flink returns; the distribution gets created by
    // FlinkMetricContainer, not by Flink itself, so we verify it in a different way below
    SimpleCounter counter = new SimpleCounter();
    when(metricGroup.counter("ns1.int_counter")).thenReturn(counter);
    container.updateMetrics("step", ImmutableList.of(intCounter, doubleCounter, intDistribution, doubleDistribution));
    // Flink's MetricGroup should only have asked for one counter (the integer-typed one) to be
    // created (the double-typed one is dropped currently)
    verify(metricGroup).counter(eq("ns1.int_counter"));
    // Verify that the counter injected into flink has the right value
    assertThat(counter.getCount(), is(111L));
    // Verify the counter in the java SDK MetricsContainer
    long count = ((CounterCell) step.tryGetCounter(MonitoringInfoMetricName.of(intCounter))).getCumulative();
    assertThat(count, is(111L));
    // The one Flink distribution that gets created is a FlinkDistributionGauge; here we verify its
    // initial (and in this test, final) value
    verify(metricGroup).gauge(eq("ns3.int_distribution"), argThat(new ArgumentMatcher<FlinkDistributionGauge>() {

        @Override
        public boolean matches(FlinkDistributionGauge argument) {
            DistributionResult actual = ((FlinkDistributionGauge) argument).getValue();
            DistributionResult expected = DistributionResult.create(30, 10, 1, 5);
            return actual.equals(expected);
        }
    }));
    // Verify that the Java SDK MetricsContainer holds the same information
    DistributionData distributionData = ((DistributionCell) step.getDistribution(MonitoringInfoMetricName.of(intDistribution))).getCumulative();
    assertThat(distributionData, is(DistributionData.create(30, 10, 1, 5)));
}