Examples with MetricResult org.apache.beam.sdk.metrics.MetricResult used on opensource projects

Example 1 with MetricResult

use of org.apache.beam.sdk.metrics.MetricResult in project beam by apache.

the class KafkaIOTest method testUnboundedSourceMetrics.

@Test
public void testUnboundedSourceMetrics() {
    int numElements = 1000;
    String readStep = "readFromKafka";
    p.apply(readStep, mkKafkaReadTransform(numElements, new ValueAsTimestampFn()).withoutMetadata());
    PipelineResult result = p.run();
    String splitId = "0";
    MetricName elementsRead = SourceMetrics.elementsRead().getName();
    MetricName elementsReadBySplit = SourceMetrics.elementsReadBySplit(splitId).getName();
    MetricName bytesRead = SourceMetrics.bytesRead().getName();
    MetricName bytesReadBySplit = SourceMetrics.bytesReadBySplit(splitId).getName();
    MetricName backlogElementsOfSplit = SourceMetrics.backlogElementsOfSplit(splitId).getName();
    MetricName backlogBytesOfSplit = SourceMetrics.backlogBytesOfSplit(splitId).getName();
    MetricQueryResults metrics = result.metrics().queryMetrics(MetricsFilter.builder().build());
    Iterable<MetricResult<Long>> counters = metrics.counters();
    assertThat(counters, hasItem(attemptedMetricsResult(elementsRead.namespace(), elementsRead.name(), readStep, 1000L)));
    assertThat(counters, hasItem(attemptedMetricsResult(elementsReadBySplit.namespace(), elementsReadBySplit.name(), readStep, 1000L)));
    assertThat(counters, hasItem(attemptedMetricsResult(bytesRead.namespace(), bytesRead.name(), readStep, 12000L)));
    assertThat(counters, hasItem(attemptedMetricsResult(bytesReadBySplit.namespace(), bytesReadBySplit.name(), readStep, 12000L)));
    MetricQueryResults backlogElementsMetrics = result.metrics().queryMetrics(MetricsFilter.builder().addNameFilter(MetricNameFilter.named(backlogElementsOfSplit.namespace(), backlogElementsOfSplit.name())).build());
    // since gauge values may be inconsistent in some environments assert only on their existence.
    assertThat(backlogElementsMetrics.gauges(), IsIterableWithSize.<MetricResult<GaugeResult>>iterableWithSize(1));
    MetricQueryResults backlogBytesMetrics = result.metrics().queryMetrics(MetricsFilter.builder().addNameFilter(MetricNameFilter.named(backlogBytesOfSplit.namespace(), backlogBytesOfSplit.name())).build());
    // since gauge values may be inconsistent in some environments assert only on their existence.
    assertThat(backlogBytesMetrics.gauges(), IsIterableWithSize.<MetricResult<GaugeResult>>iterableWithSize(1));
}

Example 2 with MetricResult

use of org.apache.beam.sdk.metrics.MetricResult in project beam by apache.

the class DataflowMetrics method queryServiceForMetrics.

private MetricQueryResults queryServiceForMetrics(MetricsFilter filter) {
    List<com.google.api.services.dataflow.model.MetricUpdate> metricUpdates;
    ImmutableList<MetricResult<Long>> counters = ImmutableList.of();
    ImmutableList<MetricResult<DistributionResult>> distributions = ImmutableList.of();
    ImmutableList<MetricResult<GaugeResult>> gauges = ImmutableList.of();
    JobMetrics jobMetrics;
    try {
        jobMetrics = dataflowClient.getJobMetrics(dataflowPipelineJob.jobId);
    } catch (IOException e) {
        LOG.warn("Unable to query job metrics.\n");
        return DataflowMetricQueryResults.create(counters, distributions, gauges);
    }
    metricUpdates = firstNonNull(jobMetrics.getMetrics(), Collections.<com.google.api.services.dataflow.model.MetricUpdate>emptyList());
    return populateMetricQueryResults(metricUpdates, filter);
}

Example 3 with MetricResult

use of org.apache.beam.sdk.metrics.MetricResult in project beam by apache.

the class ResumeFromCheckpointStreamingTest method testWithResume.

@Test
@Category(UsesCheckpointRecovery.class)
public void testWithResume() throws Exception {
    // write to Kafka
    produce(ImmutableMap.of("k1", new Instant(100), "k2", new Instant(200), "k3", new Instant(300), "k4", new Instant(400)));
    MetricsFilter metricsFilter = MetricsFilter.builder().addNameFilter(MetricNameFilter.inNamespace(ResumeFromCheckpointStreamingTest.class)).build();
    // first run should expect EOT matching the last injected element.
    SparkPipelineResult res = run(Optional.of(new Instant(400)), 0);
    assertThat(res.metrics().queryMetrics(metricsFilter).counters(), hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(), "allMessages", "EOFShallNotPassFn", 4L)));
    assertThat(res.metrics().queryMetrics(metricsFilter).counters(), hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(), "processedMessages", "EOFShallNotPassFn", 4L)));
    //--- between executions:
    //- clear state.
    clean();
    //- write a bit more.
    produce(ImmutableMap.of("k5", new Instant(499), // to be dropped from [0, 500).
    "EOF", // to be dropped from [0, 500).
    new Instant(500)));
    // recovery should resume from last read offset, and read the second batch of input.
    res = runAgain(1);
    // assertions 2:
    assertThat(res.metrics().queryMetrics(metricsFilter).counters(), hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(), "processedMessages", "EOFShallNotPassFn", 5L)));
    assertThat(res.metrics().queryMetrics(metricsFilter).counters(), hasItem(attemptedMetricsResult(ResumeFromCheckpointStreamingTest.class.getName(), "allMessages", "EOFShallNotPassFn", 6L)));
    long successAssertions = 0;
    Iterable<MetricResult<Long>> counterResults = res.metrics().queryMetrics(MetricsFilter.builder().addNameFilter(MetricNameFilter.named(PAssertWithoutFlatten.class, PAssert.SUCCESS_COUNTER)).build()).counters();
    for (MetricResult<Long> counter : counterResults) {
        if (counter.attempted().longValue() > 0) {
            successAssertions++;
        }
    }
    assertThat(String.format("Expected %d successful assertions, but found %d.", 1L, successAssertions), successAssertions, is(1L));
    // validate assertion didn't fail.
    long failedAssertions = 0;
    Iterable<MetricResult<Long>> failCounterResults = res.metrics().queryMetrics(MetricsFilter.builder().addNameFilter(MetricNameFilter.named(PAssertWithoutFlatten.class, PAssert.FAILURE_COUNTER)).build()).counters();
    for (MetricResult<Long> counter : failCounterResults) {
        if (counter.attempted().longValue() > 0) {
            failedAssertions++;
        }
    }
    assertThat(String.format("Found %d failed assertions.", failedAssertions), failedAssertions, is(0L));
}

Example 4 with MetricResult

use of org.apache.beam.sdk.metrics.MetricResult in project beam by apache.

the class DataflowMetrics method populateMetricQueryResults.

/**
   * Take a list of metric updates coming from the Dataflow service, and format it into a
   * Metrics API MetricQueryResults instance.
   * @param metricUpdates
   * @return a populated MetricQueryResults object.
   */
private MetricQueryResults populateMetricQueryResults(List<com.google.api.services.dataflow.model.MetricUpdate> metricUpdates, MetricsFilter filter) {
    // Separate metric updates by name and by tentative/committed.
    HashMap<MetricKey, com.google.api.services.dataflow.model.MetricUpdate> tentativeByName = new HashMap<>();
    HashMap<MetricKey, com.google.api.services.dataflow.model.MetricUpdate> committedByName = new HashMap<>();
    HashSet<MetricKey> metricHashKeys = new HashSet<>();
    // actual metrics counters.
    for (com.google.api.services.dataflow.model.MetricUpdate update : metricUpdates) {
        if (Objects.equal(update.getName().getOrigin(), "user") && isMetricTentative(update) && update.getName().getContext().containsKey("namespace")) {
            tentativeByName.put(metricHashKey(update), update);
            metricHashKeys.add(metricHashKey(update));
        } else if (Objects.equal(update.getName().getOrigin(), "user") && update.getName().getContext().containsKey("namespace") && !isMetricTentative(update)) {
            committedByName.put(metricHashKey(update), update);
            metricHashKeys.add(metricHashKey(update));
        }
    }
    // Create the lists with the metric result information.
    ImmutableList.Builder<MetricResult<Long>> counterResults = ImmutableList.builder();
    ImmutableList.Builder<MetricResult<DistributionResult>> distributionResults = ImmutableList.builder();
    ImmutableList.Builder<MetricResult<GaugeResult>> gaugeResults = ImmutableList.builder();
    for (MetricKey metricKey : metricHashKeys) {
        if (!MetricFiltering.matches(filter, metricKey)) {
            // Skip unmatched metrics early.
            continue;
        }
        // wrap it in a try-catch and log errors.
        try {
            String metricName = metricKey.metricName().name();
            if (metricName.endsWith("[MIN]") || metricName.endsWith("[MAX]") || metricName.endsWith("[MEAN]") || metricName.endsWith("[COUNT]")) {
                // Skip distribution metrics, as these are not yet properly supported.
                LOG.warn("Distribution metrics are not yet supported. You can see them in the Dataflow" + " User Interface");
                continue;
            }
            String namespace = metricKey.metricName().namespace();
            String step = metricKey.stepName();
            Long committed = ((Number) committedByName.get(metricKey).getScalar()).longValue();
            Long attempted = ((Number) tentativeByName.get(metricKey).getScalar()).longValue();
            counterResults.add(DataflowMetricResult.create(MetricName.named(namespace, metricName), step, committed, attempted));
        } catch (Exception e) {
            LOG.warn("Error handling metric {} for filter {}, skipping result.", metricKey, filter);
        }
    }
    return DataflowMetricQueryResults.create(counterResults.build(), distributionResults.build(), gaugeResults.build());
}