Examples with MetricKey org.apache.beam.runners.core.metrics.MetricKey used on opensource projects

Example 1 with MetricKey

use of org.apache.beam.runners.core.metrics.MetricKey in project beam by apache.

the class DataflowMetrics method populateMetricQueryResults.

/**
   * Take a list of metric updates coming from the Dataflow service, and format it into a
   * Metrics API MetricQueryResults instance.
   * @param metricUpdates
   * @return a populated MetricQueryResults object.
   */
private MetricQueryResults populateMetricQueryResults(List<com.google.api.services.dataflow.model.MetricUpdate> metricUpdates, MetricsFilter filter) {
    // Separate metric updates by name and by tentative/committed.
    HashMap<MetricKey, com.google.api.services.dataflow.model.MetricUpdate> tentativeByName = new HashMap<>();
    HashMap<MetricKey, com.google.api.services.dataflow.model.MetricUpdate> committedByName = new HashMap<>();
    HashSet<MetricKey> metricHashKeys = new HashSet<>();
    // actual metrics counters.
    for (com.google.api.services.dataflow.model.MetricUpdate update : metricUpdates) {
        if (Objects.equal(update.getName().getOrigin(), "user") && isMetricTentative(update) && update.getName().getContext().containsKey("namespace")) {
            tentativeByName.put(metricHashKey(update), update);
            metricHashKeys.add(metricHashKey(update));
        } else if (Objects.equal(update.getName().getOrigin(), "user") && update.getName().getContext().containsKey("namespace") && !isMetricTentative(update)) {
            committedByName.put(metricHashKey(update), update);
            metricHashKeys.add(metricHashKey(update));
        }
    }
    // Create the lists with the metric result information.
    ImmutableList.Builder<MetricResult<Long>> counterResults = ImmutableList.builder();
    ImmutableList.Builder<MetricResult<DistributionResult>> distributionResults = ImmutableList.builder();
    ImmutableList.Builder<MetricResult<GaugeResult>> gaugeResults = ImmutableList.builder();
    for (MetricKey metricKey : metricHashKeys) {
        if (!MetricFiltering.matches(filter, metricKey)) {
            // Skip unmatched metrics early.
            continue;
        }
        // wrap it in a try-catch and log errors.
        try {
            String metricName = metricKey.metricName().name();
            if (metricName.endsWith("[MIN]") || metricName.endsWith("[MAX]") || metricName.endsWith("[MEAN]") || metricName.endsWith("[COUNT]")) {
                // Skip distribution metrics, as these are not yet properly supported.
                LOG.warn("Distribution metrics are not yet supported. You can see them in the Dataflow" + " User Interface");
                continue;
            }
            String namespace = metricKey.metricName().namespace();
            String step = metricKey.stepName();
            Long committed = ((Number) committedByName.get(metricKey).getScalar()).longValue();
            Long attempted = ((Number) tentativeByName.get(metricKey).getScalar()).longValue();
            counterResults.add(DataflowMetricResult.create(MetricName.named(namespace, metricName), step, committed, attempted));
        } catch (Exception e) {
            LOG.warn("Error handling metric {} for filter {}, skipping result.", metricKey, filter);
        }
    }
    return DataflowMetricQueryResults.create(counterResults.build(), distributionResults.build(), gaugeResults.build());
}