Examples with Metric org.apache.kafka.common.Metric used on opensource projects

Example 26 with Metric

use of org.apache.kafka.common.Metric in project kafka by apache.

the class IntegrationTestUtils method waitForStandbyCompletion.

/**
 * Wait for streams to "finish" processing standbys, based on the (restore) consumer lag metric. Includes only the
 * restore consumer, for completion of active tasks see {@link #waitForCompletion}
 *
 * Caveats:
 * - Inputs must be finite, fully loaded, and flushed before this method is called
 * - expectedPartitions is the total number of partitions to watch the lag on, including both input and internal.
 *   It's somewhat ok to get this wrong, as the main failure case would be an immediate return due to the clients
 *   not being initialized, which you can avoid with any non-zero value. But it's probably better to get it right ;)
 */
public static void waitForStandbyCompletion(final KafkaStreams streams, final int expectedPartitions, final long timeoutMilliseconds) {
    final long start = System.currentTimeMillis();
    while (true) {
        int lagMetrics = 0;
        double totalLag = 0.0;
        for (final Metric metric : streams.metrics().values()) {
            if (metric.metricName().name().equals("records-lag")) {
                if (metric.metricName().tags().get("client-id").endsWith("restore-consumer")) {
                    lagMetrics++;
                    totalLag += ((Number) metric.metricValue()).doubleValue();
                }
            }
        }
        if (lagMetrics >= expectedPartitions && totalLag == 0.0) {
            return;
        }
        if (System.currentTimeMillis() - start >= timeoutMilliseconds) {
            throw new RuntimeException(String.format("Timed out waiting for completion. lagMetrics=[%s/%s] totalLag=[%s]", lagMetrics, expectedPartitions, totalLag));
        }
    }
}

Example 27 with Metric

use of org.apache.kafka.common.Metric in project kafka by apache.

the class KafkaConsumerTest method testMeasureCommitSyncDurationOnFailure.

@Test
public void testMeasureCommitSyncDurationOnFailure() {
    final KafkaConsumer<String, String> consumer = consumerWithPendingError(new MockTime(Duration.ofSeconds(1).toMillis()));
    try {
        consumer.commitSync(Collections.singletonMap(tp0, new OffsetAndMetadata(10L)));
    } catch (final RuntimeException e) {
    }
    final Metric metric = consumer.metrics().get(consumer.metrics.metricName("commit-sync-time-ns-total", "consumer-metrics"));
    assertTrue((Double) metric.metricValue() >= Duration.ofMillis(999).toNanos());
}

Example 28 with Metric

use of org.apache.kafka.common.Metric in project kafka by apache.

the class KafkaConsumerTest method testMeasureCommitSyncDuration.

@Test
public void testMeasureCommitSyncDuration() {
    Time time = new MockTime(Duration.ofSeconds(1).toMillis());
    SubscriptionState subscription = new SubscriptionState(new LogContext(), OffsetResetStrategy.EARLIEST);
    ConsumerMetadata metadata = createMetadata(subscription);
    MockClient client = new MockClient(time, metadata);
    initMetadata(client, Collections.singletonMap(topic, 2));
    Node node = metadata.fetch().nodes().get(0);
    KafkaConsumer<String, String> consumer = newConsumer(time, client, subscription, metadata, assignor, true, groupInstanceId);
    consumer.assign(singletonList(tp0));
    client.prepareResponseFrom(FindCoordinatorResponse.prepareResponse(Errors.NONE, groupId, node), node);
    Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
    client.prepareResponseFrom(offsetCommitResponse(Collections.singletonMap(tp0, Errors.NONE)), coordinator);
    consumer.commitSync(Collections.singletonMap(tp0, new OffsetAndMetadata(10L)));
    final Metric metric = consumer.metrics().get(consumer.metrics.metricName("commit-sync-time-ns-total", "consumer-metrics"));
    assertTrue((Double) metric.metricValue() >= Duration.ofMillis(999).toNanos());
}

Example 29 with Metric

use of org.apache.kafka.common.Metric in project kafka by apache.

the class KafkaConsumerTest method testMeasureCommittedDurationOnFailure.

@Test
public void testMeasureCommittedDurationOnFailure() {
    final KafkaConsumer<String, String> consumer = consumerWithPendingError(new MockTime(Duration.ofSeconds(1).toMillis()));
    try {
        consumer.committed(Collections.singleton(tp0));
    } catch (final RuntimeException e) {
    }
    final Metric metric = consumer.metrics().get(consumer.metrics.metricName("committed-time-ns-total", "consumer-metrics"));
    assertTrue((Double) metric.metricValue() >= Duration.ofMillis(999).toNanos());
}

Example 30 with Metric

use of org.apache.kafka.common.Metric in project kafka by apache.

the class ConsumerCoordinatorTest method testThreadSafeAssignedPartitionsMetric.

@Test
public void testThreadSafeAssignedPartitionsMetric() throws Exception {
    // Get the assigned-partitions metric
    final Metric metric = metrics.metric(new MetricName("assigned-partitions", consumerId + groupId + "-coordinator-metrics", "", Collections.emptyMap()));
    // Start polling the metric in the background
    final AtomicBoolean doStop = new AtomicBoolean();
    final AtomicReference<Exception> exceptionHolder = new AtomicReference<>();
    final AtomicInteger observedSize = new AtomicInteger();
    Thread poller = new Thread() {

        @Override
        public void run() {
            // Poll as fast as possible to reproduce ConcurrentModificationException
            while (!doStop.get()) {
                try {
                    int size = ((Double) metric.metricValue()).intValue();
                    observedSize.set(size);
                } catch (Exception e) {
                    exceptionHolder.set(e);
                    return;
                }
            }
        }
    };
    poller.start();
    // Assign two partitions to trigger a metric change that can lead to ConcurrentModificationException
    client.prepareResponse(groupCoordinatorResponse(node, Errors.NONE));
    coordinator.ensureCoordinatorReady(time.timer(Long.MAX_VALUE));
    // Change the assignment several times to increase likelihood of concurrent updates
    Set<TopicPartition> partitions = new HashSet<>();
    int totalPartitions = 10;
    for (int partition = 0; partition < totalPartitions; partition++) {
        partitions.add(new TopicPartition(topic1, partition));
        subscriptions.assignFromUser(partitions);
    }
    // Wait for the metric poller to observe the final assignment change or raise an error
    TestUtils.waitForCondition(() -> observedSize.get() == totalPartitions || exceptionHolder.get() != null, "Failed to observe expected assignment change");
    doStop.set(true);
    poller.join();
    assertNull(exceptionHolder.get(), "Failed fetching the metric at least once");
}