Examples with MetricQueryResults org.apache.beam.sdk.metrics.MetricQueryResults used on opensource projects

Example 6 with MetricQueryResults

use of org.apache.beam.sdk.metrics.MetricQueryResults in project beam by apache.

the class KafkaIOTest method testSinkMetrics.

@Test
public void testSinkMetrics() throws Exception {
    // Simply read from kafka source and write to kafka sink. Then verify the metrics are reported.
    int numElements = 1000;
    synchronized (MOCK_PRODUCER_LOCK) {
        MOCK_PRODUCER.clear();
        ProducerSendCompletionThread completionThread = new ProducerSendCompletionThread().start();
        String topic = "test";
        p.apply(mkKafkaReadTransform(numElements, new ValueAsTimestampFn()).withoutMetadata()).apply("writeToKafka", KafkaIO.<Integer, Long>write().withBootstrapServers("none").withTopic(topic).withKeySerializer(IntegerSerializer.class).withValueSerializer(LongSerializer.class).withProducerFactoryFn(new ProducerFactoryFn()));
        PipelineResult result = p.run();
        MetricName elementsWritten = SinkMetrics.elementsWritten().getName();
        MetricQueryResults metrics = result.metrics().queryMetrics(MetricsFilter.builder().addNameFilter(MetricNameFilter.inNamespace(elementsWritten.namespace())).build());
        assertThat(metrics.counters(), hasItem(attemptedMetricsResult(elementsWritten.namespace(), elementsWritten.name(), "writeToKafka", 1000L)));
        completionThread.shutdown();
    }
}

Example 7 with MetricQueryResults

use of org.apache.beam.sdk.metrics.MetricQueryResults in project beam by apache.

the class KafkaIOTest method testUnboundedSourceMetrics.

@Test
public void testUnboundedSourceMetrics() {
    int numElements = 1000;
    String readStep = "readFromKafka";
    p.apply(readStep, mkKafkaReadTransform(numElements, new ValueAsTimestampFn()).withoutMetadata());
    PipelineResult result = p.run();
    String splitId = "0";
    MetricName elementsRead = SourceMetrics.elementsRead().getName();
    MetricName elementsReadBySplit = SourceMetrics.elementsReadBySplit(splitId).getName();
    MetricName bytesRead = SourceMetrics.bytesRead().getName();
    MetricName bytesReadBySplit = SourceMetrics.bytesReadBySplit(splitId).getName();
    MetricName backlogElementsOfSplit = SourceMetrics.backlogElementsOfSplit(splitId).getName();
    MetricName backlogBytesOfSplit = SourceMetrics.backlogBytesOfSplit(splitId).getName();
    MetricQueryResults metrics = result.metrics().queryMetrics(MetricsFilter.builder().build());
    Iterable<MetricResult<Long>> counters = metrics.counters();
    assertThat(counters, hasItem(attemptedMetricsResult(elementsRead.namespace(), elementsRead.name(), readStep, 1000L)));
    assertThat(counters, hasItem(attemptedMetricsResult(elementsReadBySplit.namespace(), elementsReadBySplit.name(), readStep, 1000L)));
    assertThat(counters, hasItem(attemptedMetricsResult(bytesRead.namespace(), bytesRead.name(), readStep, 12000L)));
    assertThat(counters, hasItem(attemptedMetricsResult(bytesReadBySplit.namespace(), bytesReadBySplit.name(), readStep, 12000L)));
    MetricQueryResults backlogElementsMetrics = result.metrics().queryMetrics(MetricsFilter.builder().addNameFilter(MetricNameFilter.named(backlogElementsOfSplit.namespace(), backlogElementsOfSplit.name())).build());
    // since gauge values may be inconsistent in some environments assert only on their existence.
    assertThat(backlogElementsMetrics.gauges(), IsIterableWithSize.<MetricResult<GaugeResult>>iterableWithSize(1));
    MetricQueryResults backlogBytesMetrics = result.metrics().queryMetrics(MetricsFilter.builder().addNameFilter(MetricNameFilter.named(backlogBytesOfSplit.namespace(), backlogBytesOfSplit.name())).build());
    // since gauge values may be inconsistent in some environments assert only on their existence.
    assertThat(backlogBytesMetrics.gauges(), IsIterableWithSize.<MetricResult<GaugeResult>>iterableWithSize(1));
}

Example 8 with MetricQueryResults

use of org.apache.beam.sdk.metrics.MetricQueryResults in project beam by apache.

the class DirectMetricsTest method testPartialScopeMatchingInMetricsQuery.

@SuppressWarnings("unchecked")
@Test
public void testPartialScopeMatchingInMetricsQuery() {
    metrics.updatePhysical(bundle1, MetricUpdates.create(ImmutableList.of(MetricUpdate.create(MetricKey.create("Top1/Outer1/Inner1", NAME1), 5L), MetricUpdate.create(MetricKey.create("Top1/Outer1/Inner2", NAME1), 8L)), ImmutableList.<MetricUpdate<DistributionData>>of(), ImmutableList.<MetricUpdate<GaugeData>>of()));
    metrics.updatePhysical(bundle1, MetricUpdates.create(ImmutableList.of(MetricUpdate.create(MetricKey.create("Top2/Outer1/Inner1", NAME1), 12L), MetricUpdate.create(MetricKey.create("Top1/Outer2/Inner2", NAME1), 18L)), ImmutableList.<MetricUpdate<DistributionData>>of(), ImmutableList.<MetricUpdate<GaugeData>>of()));
    MetricQueryResults results = metrics.queryMetrics(MetricsFilter.builder().addStep("Top1/Outer1").build());
    assertThat(results.counters(), containsInAnyOrder(attemptedMetricsResult("ns1", "name1", "Top1/Outer1/Inner1", 5L), attemptedMetricsResult("ns1", "name1", "Top1/Outer1/Inner2", 8L)));
    results = metrics.queryMetrics(MetricsFilter.builder().addStep("Inner2").build());
    assertThat(results.counters(), containsInAnyOrder(attemptedMetricsResult("ns1", "name1", "Top1/Outer1/Inner2", 8L), attemptedMetricsResult("ns1", "name1", "Top1/Outer2/Inner2", 18L)));
}

Example 9 with MetricQueryResults

use of org.apache.beam.sdk.metrics.MetricQueryResults in project beam by apache.

the class DirectMetricsTest method testApplyAttemptedCountersQueryOneNamespace.

@SuppressWarnings("unchecked")
@Test
public void testApplyAttemptedCountersQueryOneNamespace() {
    metrics.updatePhysical(bundle1, MetricUpdates.create(ImmutableList.of(MetricUpdate.create(MetricKey.create("step1", NAME1), 5L), MetricUpdate.create(MetricKey.create("step1", NAME3), 8L)), ImmutableList.<MetricUpdate<DistributionData>>of(), ImmutableList.<MetricUpdate<GaugeData>>of()));
    metrics.updatePhysical(bundle1, MetricUpdates.create(ImmutableList.of(MetricUpdate.create(MetricKey.create("step2", NAME1), 7L), MetricUpdate.create(MetricKey.create("step1", NAME3), 4L)), ImmutableList.<MetricUpdate<DistributionData>>of(), ImmutableList.<MetricUpdate<GaugeData>>of()));
    MetricQueryResults results = metrics.queryMetrics(MetricsFilter.builder().addNameFilter(inNamespace("ns1")).build());
    assertThat(results.counters(), containsInAnyOrder(attemptedMetricsResult("ns1", "name1", "step1", 5L), attemptedMetricsResult("ns1", "name1", "step2", 7L)));
    assertThat(results.counters(), containsInAnyOrder(committedMetricsResult("ns1", "name1", "step1", 0L), committedMetricsResult("ns1", "name1", "step2", 0L)));
}

Example 10 with MetricQueryResults

use of org.apache.beam.sdk.metrics.MetricQueryResults in project beam by apache.

the class StreamingSourceMetricsTest method testUnboundedSourceMetrics.

@Test
@Category(StreamingTest.class)
public void testUnboundedSourceMetrics() {
    final long numElements = 1000;
    pipeline.apply(// Use maxReadTime to force unbounded mode.
    GenerateSequence.from(0).to(numElements).withMaxReadTime(Duration.standardDays(1)));
    PipelineResult pipelineResult = pipeline.run();
    MetricQueryResults metrics = pipelineResult.metrics().queryMetrics(MetricsFilter.builder().addNameFilter(MetricNameFilter.named(ELEMENTS_READ.namespace(), ELEMENTS_READ.name())).build());
    assertThat(metrics.counters(), hasItem(attemptedMetricsResult(ELEMENTS_READ.namespace(), ELEMENTS_READ.name(), "Read(UnboundedCountingSource)", 1000L)));
}