Examples with KafkaMetric org.apache.kafka.common.metrics.KafkaMetric used on opensource projects

Example 81 with KafkaMetric

use of org.apache.kafka.common.metrics.KafkaMetric in project apache-kafka-on-k8s by banzaicloud.

the class FetcherTest method testFetchResponseMetricsPartialResponse.

@Test
public void testFetchResponseMetricsPartialResponse() {
    subscriptions.assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 1);
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg));
    KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg));
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
    MemoryRecords records = builder.build();
    int expectedBytes = 0;
    for (Record record : records.records()) {
        if (record.offset() >= 1)
            expectedBytes += record.sizeInBytes();
    }
    fetchRecords(tp0, records, Errors.NONE, 100L, 0);
    assertEquals(expectedBytes, fetchSizeAverage.value(), EPSILON);
    assertEquals(2, recordsCountAverage.value(), EPSILON);
}

Example 82 with KafkaMetric

use of org.apache.kafka.common.metrics.KafkaMetric in project kafka by apache.

the class FetcherTest method testReadCommittedLagMetric.

@Test
public void testReadCommittedLagMetric() {
    buildFetcher(OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_COMMITTED);
    assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 0);
    MetricName maxLagMetric = metrics.metricInstance(metricsRegistry.recordsLagMax);
    Map<String, String> tags = new HashMap<>();
    tags.put("topic", tp0.topic());
    tags.put("partition", String.valueOf(tp0.partition()));
    MetricName partitionLagMetric = metrics.metricName("records-lag", metricGroup, tags);
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric recordsFetchLagMax = allMetrics.get(maxLagMetric);
    // recordsFetchLagMax should be initialized to NaN
    assertEquals(Double.NaN, (Double) recordsFetchLagMax.metricValue(), EPSILON);
    // recordsFetchLagMax should be lso - fetchOffset after receiving an empty FetchResponse
    fetchRecords(tidp0, MemoryRecords.EMPTY, Errors.NONE, 100L, 50L, 0);
    assertEquals(50, (Double) recordsFetchLagMax.metricValue(), EPSILON);
    KafkaMetric partitionLag = allMetrics.get(partitionLagMetric);
    assertEquals(50, (Double) partitionLag.metricValue(), EPSILON);
    // recordsFetchLagMax should be lso - offset of the last message after receiving a non-empty FetchResponse
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
    fetchRecords(tidp0, builder.build(), Errors.NONE, 200L, 150L, 0);
    assertEquals(147, (Double) recordsFetchLagMax.metricValue(), EPSILON);
    assertEquals(147, (Double) partitionLag.metricValue(), EPSILON);
    // verify de-registration of partition lag
    subscriptions.unsubscribe();
    fetcher.sendFetches();
    assertFalse(allMetrics.containsKey(partitionLagMetric));
}

Example 83 with KafkaMetric

use of org.apache.kafka.common.metrics.KafkaMetric in project kafka by apache.

the class FetcherTest method testFetchResponseMetricsPartialResponse.

@Test
public void testFetchResponseMetricsPartialResponse() {
    buildFetcher();
    assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 1);
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg));
    KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg));
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
    MemoryRecords records = builder.build();
    int expectedBytes = 0;
    for (Record record : records.records()) {
        if (record.offset() >= 1)
            expectedBytes += record.sizeInBytes();
    }
    fetchRecords(tidp0, records, Errors.NONE, 100L, 0);
    assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON);
    assertEquals(2, (Double) recordsCountAverage.metricValue(), EPSILON);
}

Example 84 with KafkaMetric

use of org.apache.kafka.common.metrics.KafkaMetric in project kafka by apache.

the class FetcherTest method testFetchResponseMetricsWithOnePartitionError.

@Test
public void testFetchResponseMetricsWithOnePartitionError() {
    buildFetcher();
    assignFromUser(mkSet(tp0, tp1));
    subscriptions.seek(tp0, 0);
    subscriptions.seek(tp1, 0);
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg));
    KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg));
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
    MemoryRecords records = builder.build();
    Map<TopicIdPartition, FetchResponseData.PartitionData> partitions = new HashMap<>();
    partitions.put(tidp0, new FetchResponseData.PartitionData().setPartitionIndex(tp0.partition()).setHighWatermark(100).setLogStartOffset(0).setRecords(records));
    partitions.put(tidp1, new FetchResponseData.PartitionData().setPartitionIndex(tp1.partition()).setErrorCode(Errors.OFFSET_OUT_OF_RANGE.code()).setHighWatermark(100).setLogStartOffset(0));
    assertEquals(1, fetcher.sendFetches());
    client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)));
    consumerClient.poll(time.timer(0));
    fetcher.collectFetch();
    int expectedBytes = 0;
    for (Record record : records.records()) expectedBytes += record.sizeInBytes();
    assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON);
    assertEquals(3, (Double) recordsCountAverage.metricValue(), EPSILON);
}

Example 85 with KafkaMetric

use of org.apache.kafka.common.metrics.KafkaMetric in project kafka by apache.

the class SenderTest method testQuotaMetrics.

/*
     * Send multiple requests. Verify that the client side quota metrics have the right values
     */
@SuppressWarnings("deprecation")
@Test
public void testQuotaMetrics() {
    MockSelector selector = new MockSelector(time);
    Sensor throttleTimeSensor = Sender.throttleTimeSensor(this.senderMetricsRegistry);
    Cluster cluster = TestUtils.singletonCluster("test", 1);
    Node node = cluster.nodes().get(0);
    NetworkClient client = new NetworkClient(selector, metadata, "mock", Integer.MAX_VALUE, 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, time, true, new ApiVersions(), throttleTimeSensor, logContext);
    ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse(400, ApiMessageType.ListenerType.ZK_BROKER);
    ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0);
    selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
    while (!client.ready(node, time.milliseconds())) {
        client.poll(1, time.milliseconds());
        // If a throttled response is received, advance the time to ensure progress.
        time.sleep(client.throttleDelayMs(node, time.milliseconds()));
    }
    selector.clear();
    for (int i = 1; i <= 3; i++) {
        int throttleTimeMs = 100 * i;
        ProduceRequest.Builder builder = ProduceRequest.forCurrentMagic(new ProduceRequestData().setTopicData(new ProduceRequestData.TopicProduceDataCollection()).setAcks((short) 1).setTimeoutMs(1000));
        ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true);
        client.send(request, time.milliseconds());
        client.poll(1, time.milliseconds());
        ProduceResponse response = produceResponse(tp0, i, Errors.NONE, throttleTimeMs);
        buffer = RequestTestUtils.serializeResponseWithHeader(response, ApiKeys.PRODUCE.latestVersion(), request.correlationId());
        selector.completeReceive(new NetworkReceive(node.idString(), buffer));
        client.poll(1, time.milliseconds());
        // If a throttled response is received, advance the time to ensure progress.
        time.sleep(client.throttleDelayMs(node, time.milliseconds()));
        selector.clear();
    }
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric avgMetric = allMetrics.get(this.senderMetricsRegistry.produceThrottleTimeAvg);
    KafkaMetric maxMetric = allMetrics.get(this.senderMetricsRegistry.produceThrottleTimeMax);
    // Throttle times are ApiVersions=400, Produce=(100, 200, 300)
    assertEquals(250, (Double) avgMetric.metricValue(), EPS);
    assertEquals(400, (Double) maxMetric.metricValue(), EPS);
    client.close();
}