Examples with RawMetricType com.linkedin.kafka.cruisecontrol.metricsreporter.metric.RawMetricType used on opensource projects

Example 6 with RawMetricType

use of com.linkedin.kafka.cruisecontrol.metricsreporter.metric.RawMetricType in project cruise-control by linkedin.

the class SamplingUtils method buildPartitionMetricSample.

/**
 * Create a {@link PartitionMetricSample}, record the relevant metrics for the given partition from the given topic on
 * broker that hosts the given number of leaders, and return the sample.
 *
 * @param cluster Kafka cluster.
 * @param leaderDistribution The leader count per topic/broker
 * @param tpDotNotHandled The original topic name that may contain dots.
 * @param brokerLoadById Load information for brokers by the broker id.
 * @param maxMetricTimestamp Maximum timestamp of the sampled metric during the sampling process.
 * @param cachedNumCoresByBroker Cached number of cores by broker.
 * @param skippedPartitionByBroker Number of skipped partition samples by broker ids.
 * @return Metric sample populated with topic and partition metrics, or {@code null} if sample generation is skipped.
 */
static PartitionMetricSample buildPartitionMetricSample(Cluster cluster, Map<Integer, Map<String, Integer>> leaderDistribution, TopicPartition tpDotNotHandled, Map<Integer, BrokerLoad> brokerLoadById, long maxMetricTimestamp, Map<Integer, Short> cachedNumCoresByBroker, Map<Integer, Integer> skippedPartitionByBroker) {
    Node leaderNode = cluster.leaderFor(tpDotNotHandled);
    if (leaderNode == null) {
        LOG.trace("Partition {} has no current leader.", tpDotNotHandled);
        skippedPartitionByBroker.merge(UNRECOGNIZED_BROKER_ID, 1, Integer::sum);
        return null;
    }
    int leaderId = leaderNode.id();
    // TODO: switch to linear regression model without computing partition level CPU usage.
    BrokerLoad brokerLoad = brokerLoadById.get(leaderId);
    TopicPartition tpWithDotHandled = partitionHandleDotInTopicName(tpDotNotHandled);
    if (skipBuildingPartitionMetricSample(tpDotNotHandled, tpWithDotHandled, leaderId, brokerLoad, cachedNumCoresByBroker)) {
        skippedPartitionByBroker.merge(leaderId, 1, Integer::sum);
        return null;
    }
    // Fill in all the common metrics.
    MetricDef commonMetricDef = KafkaMetricDef.commonMetricDef();
    PartitionMetricSample pms = new PartitionMetricSample(leaderId, tpDotNotHandled);
    int numLeaders = leaderDistribution.get(leaderId).get(tpDotNotHandled.topic());
    for (RawMetricType rawMetricType : RawMetricType.topicMetricTypes()) {
        double sampleValue = numLeaders == 0 ? 0 : (brokerLoad.topicMetrics(tpWithDotHandled.topic(), rawMetricType)) / numLeaders;
        MetricInfo metricInfo = commonMetricDef.metricInfo(KafkaMetricDef.forRawMetricType(rawMetricType).name());
        pms.record(metricInfo, sampleValue);
    }
    // Fill in disk and CPU utilization, which are not topic metric types.
    Double partitionSize = brokerLoad.partitionMetric(tpWithDotHandled.topic(), tpWithDotHandled.partition(), PARTITION_SIZE);
    if (partitionSize == null) {
        skippedPartitionByBroker.merge(leaderId, 1, Integer::sum);
        return null;
    }
    pms.record(commonMetricDef.metricInfo(KafkaMetricDef.DISK_USAGE.name()), partitionSize);
    Double estimatedLeaderCpuUtil = estimateLeaderCpuUtil(pms, brokerLoad, commonMetricDef, cachedNumCoresByBroker.get(leaderId));
    if (estimatedLeaderCpuUtil == null) {
        skippedPartitionByBroker.merge(leaderId, 1, Integer::sum);
        return null;
    }
    pms.record(commonMetricDef.metricInfo(KafkaMetricDef.CPU_USAGE.name()), estimatedLeaderCpuUtil);
    pms.close(maxMetricTimestamp);
    return pms;
}

Example 7 with RawMetricType

use of com.linkedin.kafka.cruisecontrol.metricsreporter.metric.RawMetricType in project cruise-control by linkedin.

the class CruiseControlMetricsProcessorTest method getCruiseControlMetrics.

/**
 * <ul>
 * <li>T1P0(B0): NW_IN = {@link #B0_TOPIC1_BYTES_IN} KB, NW_OUT = {@link #B0_TOPIC1_BYTES_OUT} KB,
 * size = {@link #T1P0_BYTES_SIZE} MB</li>
 * <li>T1P1(B1): NW_IN = {@link #B1_TOPIC1_BYTES_IN} KB, NW_OUT = {@link #B1_TOPIC1_BYTES_OUT} KB,
 * size = {@link #T1P1_BYTES_SIZE} MB</li>
 * <li>T2P0(B0): NW_IN = est. {@link #B0_TOPIC2_BYTES_IN}/2 KB, NW_OUT = est. {@link #B0_TOPIC2_BYTES_OUT}/2 KB,
 * size = {@link #T2P0_BYTES_SIZE} MB</li>
 * <li>T2P1(B0): NW_IN = est. {@link #B0_TOPIC2_BYTES_IN}/2 KB, NW_OUT = est. {@link #B0_TOPIC2_BYTES_OUT}/2 KB,
 * size = {@link #T2P1_BYTES_SIZE} MB</li>
 * <li>B0: CPU = {@link #B0_CPU}%</li>
 * <li>B1: CPU = {@link #B1_CPU}%</li>
 * </ul>
 * @return Cruise Control metrics.
 */
private Set<CruiseControlMetric> getCruiseControlMetrics() {
    Set<CruiseControlMetric> metrics = new HashSet<>();
    int i = 0;
    for (RawMetricType rawMetricType : RawMetricType.brokerMetricTypesDiffForVersion(BrokerMetricSample.MIN_SUPPORTED_VERSION)) {
        switch(rawMetricType) {
            case ALL_TOPIC_BYTES_IN:
                metrics.add(new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_IN, _time.milliseconds(), BROKER_ID_0, B0_ALL_TOPIC_BYTES_IN * BYTES_IN_KB));
                metrics.add(new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_IN, _time.milliseconds(), BROKER_ID_1, B1_ALL_TOPIC_BYTES_IN * BYTES_IN_KB));
                break;
            case ALL_TOPIC_BYTES_OUT:
                metrics.add(new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, B0_ALL_TOPIC_BYTES_OUT * BYTES_IN_KB));
                metrics.add(new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_1, B1_ALL_TOPIC_BYTES_OUT * BYTES_IN_KB));
                break;
            case BROKER_CPU_UTIL:
                metrics.add(new BrokerMetric(RawMetricType.BROKER_CPU_UTIL, _time.milliseconds(), BROKER_ID_0, B0_CPU));
                metrics.add(new BrokerMetric(RawMetricType.BROKER_CPU_UTIL, _time.milliseconds(), BROKER_ID_1, B1_CPU));
                break;
            default:
                metrics.add(new BrokerMetric(rawMetricType, _time.milliseconds(), BROKER_ID_0, i++ * BYTES_IN_MB));
                metrics.add(new BrokerMetric(rawMetricType, _time.milliseconds(), BROKER_ID_1, i++ * BYTES_IN_MB));
                break;
        }
    }
    for (RawMetricType rawMetricType : RawMetricType.topicMetricTypes()) {
        switch(rawMetricType) {
            case TOPIC_BYTES_IN:
                metrics.add(new TopicMetric(TOPIC_BYTES_IN, _time.milliseconds() + 1, BROKER_ID_0, TOPIC1, B0_TOPIC1_BYTES_IN * BYTES_IN_KB));
                metrics.add(new TopicMetric(TOPIC_BYTES_IN, _time.milliseconds() + 2, BROKER_ID_1, TOPIC1, B1_TOPIC1_BYTES_IN * BYTES_IN_KB));
                metrics.add(new TopicMetric(TOPIC_BYTES_IN, _time.milliseconds(), BROKER_ID_0, TOPIC2, B0_TOPIC2_BYTES_IN * BYTES_IN_KB));
                break;
            case TOPIC_BYTES_OUT:
                metrics.add(new TopicMetric(RawMetricType.TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC1, B0_TOPIC1_BYTES_OUT * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_1, TOPIC1, B1_TOPIC1_BYTES_OUT * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC2, B0_TOPIC2_BYTES_OUT * BYTES_IN_KB));
                break;
            case TOPIC_REPLICATION_BYTES_IN:
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_IN, _time.milliseconds(), BROKER_ID_1, TOPIC1, B1_TOPIC1_REPLICATION_BYTES_IN * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_IN, _time.milliseconds(), BROKER_ID_0, TOPIC1, B0_TOPIC1_REPLICATION_BYTES_IN * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_IN, _time.milliseconds(), BROKER_ID_1, TOPIC2, B1_TOPIC2_REPLICATION_BYTES_IN * BYTES_IN_KB));
                break;
            case TOPIC_REPLICATION_BYTES_OUT:
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC1, B0_TOPIC1_REPLICATION_BYTES_OUT * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_OUT, _time.milliseconds(), BROKER_ID_1, TOPIC1, B1_TOPIC1_REPLICATION_BYTES_OUT * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC2, B0_TOPIC2_REPLICATION_BYTES_OUT * BYTES_IN_KB));
                break;
            default:
                metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_0, TOPIC1, i * BYTES_IN_MB));
                metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_1, TOPIC1, i * BYTES_IN_MB));
                metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_0, TOPIC2, i * BYTES_IN_MB));
                metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_1, TOPIC2, i * BYTES_IN_MB));
                break;
        }
    }
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_0, TOPIC1, P0, T1P0_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_0, TOPIC1, P1, T1P1_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_0, TOPIC2, P0, T2P0_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_0, TOPIC2, P1, T2P1_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_1, TOPIC1, P0, T1P0_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_1, TOPIC1, P1, T1P1_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_1, TOPIC2, P0, T2P0_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_1, TOPIC2, P1, T2P1_BYTES_SIZE * BYTES_IN_MB));
    return metrics;
}

Example 8 with RawMetricType

use of com.linkedin.kafka.cruisecontrol.metricsreporter.metric.RawMetricType in project cruise-control by linkedin.

the class CruiseControlMetricsProcessorTest method testMissingTopicBytesInMetric.

@Test
public void testMissingTopicBytesInMetric() throws TimeoutException, BrokerCapacityResolutionException {
    CruiseControlMetricsProcessor processor = new CruiseControlMetricsProcessor(mockBrokerCapacityConfigResolver(), false);
    Set<CruiseControlMetric> metrics = getCruiseControlMetrics();
    Set<RawMetricType> metricTypeToExclude = new HashSet<>(Arrays.asList(TOPIC_BYTES_IN, TOPIC_BYTES_OUT, TOPIC_REPLICATION_BYTES_IN, TOPIC_REPLICATION_BYTES_OUT));
    for (CruiseControlMetric metric : metrics) {
        if (metricTypeToExclude.contains(metric.rawMetricType())) {
            TopicMetric tm = (TopicMetric) metric;
            if (tm.brokerId() == BROKER_ID_0 && tm.topic().equals(TOPIC1)) {
                continue;
            }
        }
        processor.addMetric(metric);
    }
    Cluster cluster = getCluster();
    MetricSampler.Samples samples = processor.process(cluster, TEST_PARTITIONS, MetricSampler.SamplingMode.ALL);
    assertEquals(4, samples.partitionMetricSamples().size());
    assertEquals(2, samples.brokerMetricSamples().size());
    for (PartitionMetricSample sample : samples.partitionMetricSamples()) {
        if (sample.entity().tp().equals(T1P0)) {
            // T1P0 should not have any IO or CPU usage.
            validatePartitionMetricSample(sample, _time.milliseconds() + 2, 0.0, 0.0, 0.0, T1P0_BYTES_SIZE);
        }
    }
}

Example 9 with RawMetricType

use of com.linkedin.kafka.cruisecontrol.metricsreporter.metric.RawMetricType in project cruise-control by linkedin.

the class PrometheusMetricSamplerTest method testGetSamplesSuccess.

@Test
public void testGetSamplesSuccess() throws Exception {
    Map<String, Object> config = new HashMap<>();
    config.put(PROMETHEUS_SERVER_ENDPOINT_CONFIG, "http://kafka-cluster-1.org:9090");
    addCapacityConfig(config);
    Set<String> topics = new HashSet<>(Arrays.asList(TEST_TOPIC, TEST_TOPIC_WITH_DOT));
    for (String topic : topics) {
        setUp();
        _prometheusMetricSampler.configure(config);
        MetricSamplerOptions metricSamplerOptions = buildMetricSamplerOptions(topic);
        _prometheusMetricSampler._prometheusAdapter = _prometheusAdapter;
        for (RawMetricType rawMetricType : _prometheusQueryMap.keySet()) {
            setupPrometheusAdapterMock(rawMetricType, buildBrokerResults(), buildTopicResults(topic), buildPartitionResults(topic));
        }
        replay(_prometheusAdapter);
        MetricSampler.Samples samples = _prometheusMetricSampler.getSamples(metricSamplerOptions);
        assertSamplesValid(samples, topic);
        verify(_prometheusAdapter);
    }
}

Example 10 with RawMetricType

use of com.linkedin.kafka.cruisecontrol.metricsreporter.metric.RawMetricType in project cruise-control by linkedin.

the class PrometheusMetricSamplerTest method testPrometheusQueryReturnsInvalidResults.

public void testPrometheusQueryReturnsInvalidResults(List<PrometheusQueryResult> brokerResults, List<PrometheusQueryResult> topicResults, List<PrometheusQueryResult> partitionResults) throws Exception {
    Map<String, Object> config = new HashMap<>();
    config.put(PROMETHEUS_SERVER_ENDPOINT_CONFIG, "http://kafka-cluster-1.org:9090");
    addCapacityConfig(config);
    _prometheusMetricSampler.configure(config);
    MetricSamplerOptions metricSamplerOptions = buildMetricSamplerOptions(TEST_TOPIC);
    _prometheusMetricSampler._prometheusAdapter = _prometheusAdapter;
    for (RawMetricType rawMetricType : _prometheusQueryMap.keySet()) {
        setupPrometheusAdapterMock(rawMetricType, brokerResults, topicResults, partitionResults);
    }
    replay(_prometheusAdapter);
    _prometheusMetricSampler.getSamples(metricSamplerOptions);
}