Examples with KafkaMetric org.apache.kafka.common.metrics.KafkaMetric used on opensource projects

Example 21 with KafkaMetric

use of org.apache.kafka.common.metrics.KafkaMetric in project kafka by apache.

the class FetcherTest method testQuotaMetrics.

/*
     * Send multiple requests. Verify that the client side quota metrics have the right values
     */
@Test
public void testQuotaMetrics() {
    buildFetcher();
    MockSelector selector = new MockSelector(time);
    Sensor throttleTimeSensor = Fetcher.throttleTimeSensor(metrics, metricsRegistry);
    Cluster cluster = TestUtils.singletonCluster("test", 1);
    Node node = cluster.nodes().get(0);
    NetworkClient client = new NetworkClient(selector, metadata, "mock", Integer.MAX_VALUE, 1000, 1000, 64 * 1024, 64 * 1024, 1000, 10 * 1000, 127 * 1000, time, true, new ApiVersions(), throttleTimeSensor, new LogContext());
    ApiVersionsResponse apiVersionsResponse = ApiVersionsResponse.defaultApiVersionsResponse(400, ApiMessageType.ListenerType.ZK_BROKER);
    ByteBuffer buffer = RequestTestUtils.serializeResponseWithHeader(apiVersionsResponse, ApiKeys.API_VERSIONS.latestVersion(), 0);
    selector.delayedReceive(new DelayedReceive(node.idString(), new NetworkReceive(node.idString(), buffer)));
    while (!client.ready(node, time.milliseconds())) {
        client.poll(1, time.milliseconds());
        // If a throttled response is received, advance the time to ensure progress.
        time.sleep(client.throttleDelayMs(node, time.milliseconds()));
    }
    selector.clear();
    for (int i = 1; i <= 3; i++) {
        int throttleTimeMs = 100 * i;
        FetchRequest.Builder builder = FetchRequest.Builder.forConsumer(ApiKeys.FETCH.latestVersion(), 100, 100, new LinkedHashMap<>());
        builder.rackId("");
        ClientRequest request = client.newClientRequest(node.idString(), builder, time.milliseconds(), true);
        client.send(request, time.milliseconds());
        client.poll(1, time.milliseconds());
        FetchResponse response = fullFetchResponse(tidp0, nextRecords, Errors.NONE, i, throttleTimeMs);
        buffer = RequestTestUtils.serializeResponseWithHeader(response, ApiKeys.FETCH.latestVersion(), request.correlationId());
        selector.completeReceive(new NetworkReceive(node.idString(), buffer));
        client.poll(1, time.milliseconds());
        // If a throttled response is received, advance the time to ensure progress.
        time.sleep(client.throttleDelayMs(node, time.milliseconds()));
        selector.clear();
    }
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric avgMetric = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchThrottleTimeAvg));
    KafkaMetric maxMetric = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchThrottleTimeMax));
    // Throttle times are ApiVersions=400, Fetch=(100, 200, 300)
    assertEquals(250, (Double) avgMetric.metricValue(), EPSILON);
    assertEquals(400, (Double) maxMetric.metricValue(), EPSILON);
    client.close();
}

Example 22 with KafkaMetric

use of org.apache.kafka.common.metrics.KafkaMetric in project kafka by apache.

the class FetcherTest method testFetchResponseMetricsWithOnePartitionAtTheWrongOffset.

@Test
public void testFetchResponseMetricsWithOnePartitionAtTheWrongOffset() {
    buildFetcher();
    assignFromUser(mkSet(tp0, tp1));
    subscriptions.seek(tp0, 0);
    subscriptions.seek(tp1, 0);
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric fetchSizeAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.fetchSizeAvg));
    KafkaMetric recordsCountAverage = allMetrics.get(metrics.metricInstance(metricsRegistry.recordsPerRequestAvg));
    // send the fetch and then seek to a new offset
    assertEquals(1, fetcher.sendFetches());
    subscriptions.seek(tp1, 5);
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    for (int v = 0; v < 3; v++) builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
    MemoryRecords records = builder.build();
    Map<TopicIdPartition, FetchResponseData.PartitionData> partitions = new HashMap<>();
    partitions.put(tidp0, new FetchResponseData.PartitionData().setPartitionIndex(tp0.partition()).setHighWatermark(100).setLogStartOffset(0).setRecords(records));
    partitions.put(tidp1, new FetchResponseData.PartitionData().setPartitionIndex(tp1.partition()).setHighWatermark(100).setLogStartOffset(0).setRecords(MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord("val".getBytes()))));
    client.prepareResponse(FetchResponse.of(Errors.NONE, 0, INVALID_SESSION_ID, new LinkedHashMap<>(partitions)));
    consumerClient.poll(time.timer(0));
    fetcher.collectFetch();
    // we should have ignored the record at the wrong offset
    int expectedBytes = 0;
    for (Record record : records.records()) expectedBytes += record.sizeInBytes();
    assertEquals(expectedBytes, (Double) fetchSizeAverage.metricValue(), EPSILON);
    assertEquals(3, (Double) recordsCountAverage.metricValue(), EPSILON);
}

Example 23 with KafkaMetric

use of org.apache.kafka.common.metrics.KafkaMetric in project kafka by apache.

the class FetcherTest method testFetcherLeadMetric.

@Test
public void testFetcherLeadMetric() {
    buildFetcher();
    assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 0);
    MetricName minLeadMetric = metrics.metricInstance(metricsRegistry.recordsLeadMin);
    Map<String, String> tags = new HashMap<>(2);
    tags.put("topic", tp0.topic());
    tags.put("partition", String.valueOf(tp0.partition()));
    MetricName partitionLeadMetric = metrics.metricName("records-lead", metricGroup, "", tags);
    Map<MetricName, KafkaMetric> allMetrics = metrics.metrics();
    KafkaMetric recordsFetchLeadMin = allMetrics.get(minLeadMetric);
    // recordsFetchLeadMin should be initialized to NaN
    assertEquals(Double.NaN, (Double) recordsFetchLeadMin.metricValue(), EPSILON);
    // recordsFetchLeadMin should be position - logStartOffset after receiving an empty FetchResponse
    fetchRecords(tidp0, MemoryRecords.EMPTY, Errors.NONE, 100L, -1L, 0L, 0);
    assertEquals(0L, (Double) recordsFetchLeadMin.metricValue(), EPSILON);
    KafkaMetric partitionLead = allMetrics.get(partitionLeadMetric);
    assertEquals(0L, (Double) partitionLead.metricValue(), EPSILON);
    // recordsFetchLeadMin should be position - logStartOffset after receiving a non-empty FetchResponse
    MemoryRecordsBuilder builder = MemoryRecords.builder(ByteBuffer.allocate(1024), CompressionType.NONE, TimestampType.CREATE_TIME, 0L);
    for (int v = 0; v < 3; v++) {
        builder.appendWithOffset(v, RecordBatch.NO_TIMESTAMP, "key".getBytes(), ("value-" + v).getBytes());
    }
    fetchRecords(tidp0, builder.build(), Errors.NONE, 200L, -1L, 0L, 0);
    assertEquals(0L, (Double) recordsFetchLeadMin.metricValue(), EPSILON);
    assertEquals(3L, (Double) partitionLead.metricValue(), EPSILON);
    // verify de-registration of partition lag
    subscriptions.unsubscribe();
    fetcher.sendFetches();
    assertFalse(allMetrics.containsKey(partitionLeadMetric));
}

Example 24 with KafkaMetric

use of org.apache.kafka.common.metrics.KafkaMetric in project kafka by apache.

the class SelectorTest method testPartialSendAndReceiveReflectedInMetrics.

@Test
public void testPartialSendAndReceiveReflectedInMetrics() throws Exception {
    // We use a large payload to attempt to trigger the partial send and receive logic.
    int payloadSize = 20 * BUFFER_SIZE;
    String payload = TestUtils.randomString(payloadSize);
    String nodeId = "0";
    blockingConnect(nodeId);
    ByteBufferSend send = ByteBufferSend.sizePrefixed(ByteBuffer.wrap(payload.getBytes()));
    NetworkSend networkSend = new NetworkSend(nodeId, send);
    selector.send(networkSend);
    KafkaChannel channel = selector.channel(nodeId);
    KafkaMetric outgoingByteTotal = findUntaggedMetricByName("outgoing-byte-total");
    KafkaMetric incomingByteTotal = findUntaggedMetricByName("incoming-byte-total");
    TestUtils.waitForCondition(() -> {
        long bytesSent = send.size() - send.remaining();
        assertEquals(bytesSent, ((Double) outgoingByteTotal.metricValue()).longValue());
        NetworkReceive currentReceive = channel.currentReceive();
        if (currentReceive != null) {
            assertEquals(currentReceive.bytesRead(), ((Double) incomingByteTotal.metricValue()).intValue());
        }
        selector.poll(50);
        return !selector.completedReceives().isEmpty();
    }, "Failed to receive expected response");
    KafkaMetric requestTotal = findUntaggedMetricByName("request-total");
    assertEquals(1, ((Double) requestTotal.metricValue()).intValue());
    KafkaMetric responseTotal = findUntaggedMetricByName("response-total");
    assertEquals(1, ((Double) responseTotal.metricValue()).intValue());
}

Example 25 with KafkaMetric

use of org.apache.kafka.common.metrics.KafkaMetric in project kafka by apache.

the class SelectorTest method findUntaggedMetricByName.

private KafkaMetric findUntaggedMetricByName(String name) {
    MetricName metricName = new MetricName(name, METRIC_GROUP + "-metrics", "", new HashMap<>());
    KafkaMetric metric = metrics.metrics().get(metricName);
    assertNotNull(metric);
    return metric;
}