Examples with MetricConfig org.apache.kafka.common.metrics.MetricConfig used on opensource projects

Example 26 with MetricConfig

use of org.apache.kafka.common.metrics.MetricConfig in project kafka by apache.

the class MeterTest method testMeter.

@Test
public void testMeter() {
    Map<String, String> emptyTags = Collections.emptyMap();
    MetricName rateMetricName = new MetricName("rate", "test", "", emptyTags);
    MetricName totalMetricName = new MetricName("total", "test", "", emptyTags);
    Meter meter = new Meter(rateMetricName, totalMetricName);
    List<NamedMeasurable> stats = meter.stats();
    assertEquals(2, stats.size());
    NamedMeasurable total = stats.get(0);
    NamedMeasurable rate = stats.get(1);
    assertEquals(rateMetricName, rate.name());
    assertEquals(totalMetricName, total.name());
    Rate rateStat = (Rate) rate.stat();
    CumulativeSum totalStat = (CumulativeSum) total.stat();
    MetricConfig config = new MetricConfig();
    double nextValue = 0.0;
    double expectedTotal = 0.0;
    long now = 0;
    double intervalMs = 100;
    double delta = 5.0;
    // for time windows and that the total is cumulative.
    for (int i = 1; i <= 100; i++) {
        for (; now < i * 1000; now += intervalMs, nextValue += delta) {
            expectedTotal += nextValue;
            meter.record(config, nextValue, now);
        }
        assertEquals(expectedTotal, totalStat.measure(config, now), EPS);
        long windowSizeMs = rateStat.windowSize(config, now);
        long windowStartMs = Math.max(now - windowSizeMs, 0);
        double sampledTotal = 0.0;
        double prevValue = nextValue - delta;
        for (long timeMs = now - 100; timeMs >= windowStartMs; timeMs -= intervalMs, prevValue -= delta) sampledTotal += prevValue;
        assertEquals(sampledTotal * 1000 / windowSizeMs, rateStat.measure(config, now), EPS);
    }
}

Example 27 with MetricConfig

use of org.apache.kafka.common.metrics.MetricConfig in project kafka by apache.

the class KafkaStreams method getMetrics.

private static Metrics getMetrics(final StreamsConfig config, final Time time, final String clientId) {
    final MetricConfig metricConfig = new MetricConfig().samples(config.getInt(StreamsConfig.METRICS_NUM_SAMPLES_CONFIG)).recordLevel(Sensor.RecordingLevel.forName(config.getString(StreamsConfig.METRICS_RECORDING_LEVEL_CONFIG))).timeWindow(config.getLong(StreamsConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS);
    final List<MetricsReporter> reporters = config.getConfiguredInstances(StreamsConfig.METRIC_REPORTER_CLASSES_CONFIG, MetricsReporter.class, Collections.singletonMap(StreamsConfig.CLIENT_ID_CONFIG, clientId));
    final JmxReporter jmxReporter = new JmxReporter();
    jmxReporter.configure(config.originals());
    reporters.add(jmxReporter);
    final MetricsContext metricsContext = new KafkaMetricsContext(JMX_PREFIX, config.originalsWithPrefix(CommonClientConfigs.METRICS_CONTEXT_PREFIX));
    return new Metrics(metricConfig, reporters, time, metricsContext);
}

Example 28 with MetricConfig

use of org.apache.kafka.common.metrics.MetricConfig in project kafka by apache.

the class FetcherTest method testFetcherConcurrency.

@Test
public void testFetcherConcurrency() throws Exception {
    int numPartitions = 20;
    Set<TopicPartition> topicPartitions = new HashSet<>();
    for (int i = 0; i < numPartitions; i++) topicPartitions.add(new TopicPartition(topicName, i));
    LogContext logContext = new LogContext();
    buildDependencies(new MetricConfig(), Long.MAX_VALUE, new SubscriptionState(logContext, OffsetResetStrategy.EARLIEST), logContext);
    fetcher = new Fetcher<byte[], byte[]>(new LogContext(), consumerClient, minBytes, maxBytes, maxWaitMs, fetchSize, 2 * numPartitions, true, "", new ByteArrayDeserializer(), new ByteArrayDeserializer(), metadata, subscriptions, metrics, metricsRegistry, time, retryBackoffMs, requestTimeoutMs, IsolationLevel.READ_UNCOMMITTED, apiVersions) {

        @Override
        protected FetchSessionHandler sessionHandler(int id) {
            final FetchSessionHandler handler = super.sessionHandler(id);
            if (handler == null)
                return null;
            else {
                return new FetchSessionHandler(new LogContext(), id) {

                    @Override
                    public Builder newBuilder() {
                        verifySessionPartitions();
                        return handler.newBuilder();
                    }

                    @Override
                    public boolean handleResponse(FetchResponse response, short version) {
                        verifySessionPartitions();
                        return handler.handleResponse(response, version);
                    }

                    @Override
                    public void handleError(Throwable t) {
                        verifySessionPartitions();
                        handler.handleError(t);
                    }

                    // Verify that session partitions can be traversed safely.
                    private void verifySessionPartitions() {
                        try {
                            Field field = FetchSessionHandler.class.getDeclaredField("sessionPartitions");
                            field.setAccessible(true);
                            LinkedHashMap<?, ?> sessionPartitions = (LinkedHashMap<?, ?>) field.get(handler);
                            for (Map.Entry<?, ?> entry : sessionPartitions.entrySet()) {
                                // If `sessionPartitions` are modified on another thread, Thread.yield will increase the
                                // possibility of ConcurrentModificationException if appropriate synchronization is not used.
                                Thread.yield();
                            }
                        } catch (Exception e) {
                            throw new RuntimeException(e);
                        }
                    }
                };
            }
        }
    };
    MetadataResponse initialMetadataResponse = RequestTestUtils.metadataUpdateWithIds(1, singletonMap(topicName, numPartitions), tp -> validLeaderEpoch, topicIds);
    client.updateMetadata(initialMetadataResponse);
    fetchSize = 10000;
    assignFromUser(topicPartitions);
    topicPartitions.forEach(tp -> subscriptions.seek(tp, 0L));
    AtomicInteger fetchesRemaining = new AtomicInteger(1000);
    executorService = Executors.newSingleThreadExecutor();
    Future<?> future = executorService.submit(() -> {
        while (fetchesRemaining.get() > 0) {
            synchronized (consumerClient) {
                if (!client.requests().isEmpty()) {
                    ClientRequest request = client.requests().peek();
                    FetchRequest fetchRequest = (FetchRequest) request.requestBuilder().build();
                    LinkedHashMap<TopicIdPartition, FetchResponseData.PartitionData> responseMap = new LinkedHashMap<>();
                    for (Map.Entry<TopicIdPartition, FetchRequest.PartitionData> entry : fetchRequest.fetchData(topicNames).entrySet()) {
                        TopicIdPartition tp = entry.getKey();
                        long offset = entry.getValue().fetchOffset;
                        responseMap.put(tp, new FetchResponseData.PartitionData().setPartitionIndex(tp.topicPartition().partition()).setHighWatermark(offset + 2).setLastStableOffset(offset + 2).setLogStartOffset(0).setRecords(buildRecords(offset, 2, offset)));
                    }
                    client.respondToRequest(request, FetchResponse.of(Errors.NONE, 0, 123, responseMap));
                    consumerClient.poll(time.timer(0));
                }
            }
        }
        return fetchesRemaining.get();
    });
    Map<TopicPartition, Long> nextFetchOffsets = topicPartitions.stream().collect(Collectors.toMap(Function.identity(), t -> 0L));
    while (fetchesRemaining.get() > 0 && !future.isDone()) {
        if (fetcher.sendFetches() == 1) {
            synchronized (consumerClient) {
                consumerClient.poll(time.timer(0));
            }
        }
        if (fetcher.hasCompletedFetches()) {
            Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> fetchedRecords = fetchedRecords();
            if (!fetchedRecords.isEmpty()) {
                fetchesRemaining.decrementAndGet();
                fetchedRecords.forEach((tp, records) -> {
                    assertEquals(2, records.size());
                    long nextOffset = nextFetchOffsets.get(tp);
                    assertEquals(nextOffset, records.get(0).offset());
                    assertEquals(nextOffset + 1, records.get(1).offset());
                    nextFetchOffsets.put(tp, nextOffset + 2);
                });
            }
        }
    }
    assertEquals(0, future.get());
}

Example 29 with MetricConfig

use of org.apache.kafka.common.metrics.MetricConfig in project kafka by apache.

the class FetcherTest method testFetcherMetricsTemplates.

@Test
public void testFetcherMetricsTemplates() {
    Map<String, String> clientTags = Collections.singletonMap("client-id", "clientA");
    buildFetcher(new MetricConfig().tags(clientTags), OffsetResetStrategy.EARLIEST, new ByteArrayDeserializer(), new ByteArrayDeserializer(), Integer.MAX_VALUE, IsolationLevel.READ_UNCOMMITTED);
    // Fetch from topic to generate topic metrics
    assignFromUser(singleton(tp0));
    subscriptions.seek(tp0, 0);
    assertEquals(1, fetcher.sendFetches());
    client.prepareResponse(fullFetchResponse(tidp0, this.records, Errors.NONE, 100L, 0));
    consumerClient.poll(time.timer(0));
    assertTrue(fetcher.hasCompletedFetches());
    Map<TopicPartition, List<ConsumerRecord<byte[], byte[]>>> partitionRecords = fetchedRecords();
    assertTrue(partitionRecords.containsKey(tp0));
    // Create throttle metrics
    Fetcher.throttleTimeSensor(metrics, metricsRegistry);
    // Verify that all metrics except metrics-count have registered templates
    Set<MetricNameTemplate> allMetrics = new HashSet<>();
    for (MetricName n : metrics.metrics().keySet()) {
        String name = n.name().replaceAll(tp0.toString(), "{topic}-{partition}");
        if (!n.group().equals("kafka-metrics-count"))
            allMetrics.add(new MetricNameTemplate(name, n.group(), "", n.tags().keySet()));
    }
    TestUtils.checkEquals(allMetrics, new HashSet<>(metricsRegistry.getAllTemplates()), "metrics", "templates");
}

Example 30 with MetricConfig

use of org.apache.kafka.common.metrics.MetricConfig in project kafka by apache.

the class SenderTest method testSenderMetricsTemplates.

@Test
public void testSenderMetricsTemplates() throws Exception {
    metrics.close();
    Map<String, String> clientTags = Collections.singletonMap("client-id", "clientA");
    metrics = new Metrics(new MetricConfig().tags(clientTags));
    SenderMetricsRegistry metricsRegistry = new SenderMetricsRegistry(metrics);
    Sender sender = new Sender(logContext, client, metadata, this.accumulator, false, MAX_REQUEST_SIZE, ACKS_ALL, 1, metricsRegistry, time, REQUEST_TIMEOUT, RETRY_BACKOFF_MS, null, apiVersions);
    // Append a message so that topic metrics are created
    appendToAccumulator(tp0, 0L, "key", "value");
    // connect
    sender.runOnce();
    // send produce request
    sender.runOnce();
    client.respond(produceResponse(tp0, 0, Errors.NONE, 0));
    sender.runOnce();
    // Create throttle time metrics
    Sender.throttleTimeSensor(metricsRegistry);
    // Verify that all metrics except metrics-count have registered templates
    Set<MetricNameTemplate> allMetrics = new HashSet<>();
    for (MetricName n : metrics.metrics().keySet()) {
        if (!n.group().equals("kafka-metrics-count"))
            allMetrics.add(new MetricNameTemplate(n.name(), n.group(), "", n.tags().keySet()));
    }
    TestUtils.checkEquals(allMetrics, new HashSet<>(metricsRegistry.allTemplates()), "metrics", "templates");
}