Examples with BrokerLoad com.linkedin.kafka.cruisecontrol.monitor.sampling.holder.BrokerLoad used on opensource projects

Example 1 with BrokerLoad

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.holder.BrokerLoad in project cruise-control by linkedin.

the class CruiseControlMetricsProcessor method addMetric.

void addMetric(CruiseControlMetric metric) {
    int brokerId = metric.brokerId();
    LOG.trace("Adding cruise control metric {}", metric);
    _maxMetricTimestamp = Math.max(metric.time(), _maxMetricTimestamp);
    _brokerLoad.compute(brokerId, (bid, load) -> {
        BrokerLoad brokerLoad = load == null ? new BrokerLoad() : load;
        brokerLoad.recordMetric(metric);
        return brokerLoad;
    });
}

Example 2 with BrokerLoad

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.holder.BrokerLoad in project cruise-control by linkedin.

the class SamplingUtils method buildBrokerMetricSample.

/**
 * Create a {@link BrokerMetricSample}, record the relevant metrics for the given broker, and return the sample.
 *
 * @param node Node hosting the broker.
 * @param brokerLoadById Load information for brokers by the broker id.
 * @param maxMetricTimestamp Maximum timestamp of the sampled metric during the sampling process.
 * @return Metric sample populated with broker metrics, or {@code null} if sample generation is skipped.
 */
static BrokerMetricSample buildBrokerMetricSample(Node node, Map<Integer, BrokerLoad> brokerLoadById, long maxMetricTimestamp) throws UnknownVersionException {
    BrokerLoad brokerLoad = brokerLoadById.get(node.id());
    if (skipBuildingBrokerMetricSample(brokerLoad, node.id())) {
        return null;
    }
    MetricDef brokerMetricDef = KafkaMetricDef.brokerMetricDef();
    BrokerMetricSample bms = new BrokerMetricSample(node.host(), node.id(), brokerLoad.brokerSampleDeserializationVersion());
    for (Map.Entry<Byte, Set<RawMetricType>> entry : RawMetricType.brokerMetricTypesDiffByVersion().entrySet()) {
        for (RawMetricType rawBrokerMetricType : entry.getValue()) {
            // We require the broker to report all the metric types (including nullable values). Otherwise we skip the broker.
            if (!brokerLoad.brokerMetricAvailable(rawBrokerMetricType)) {
                LOG.warn("{}broker {} because it does not have {} metrics (serde version {}) or the metrics are inconsistent.", SKIP_BUILDING_SAMPLE_PREFIX, node.id(), rawBrokerMetricType, entry.getKey());
                return null;
            } else {
                MetricInfo metricInfo = brokerMetricDef.metricInfo(KafkaMetricDef.forRawMetricType(rawBrokerMetricType).name());
                double metricValue = brokerLoad.brokerMetric(rawBrokerMetricType);
                bms.record(metricInfo, metricValue);
            }
        }
    }
    // Disk usage is not one of the broker raw metric type.
    bms.record(brokerMetricDef.metricInfo(KafkaMetricDef.DISK_USAGE.name()), brokerLoad.diskUsage());
    bms.close(maxMetricTimestamp);
    return bms;
}

Example 3 with BrokerLoad

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.holder.BrokerLoad in project cruise-control by linkedin.

the class SamplingUtils method buildPartitionMetricSample.

/**
 * Create a {@link PartitionMetricSample}, record the relevant metrics for the given partition from the given topic on
 * broker that hosts the given number of leaders, and return the sample.
 *
 * @param cluster Kafka cluster.
 * @param leaderDistribution The leader count per topic/broker
 * @param tpDotNotHandled The original topic name that may contain dots.
 * @param brokerLoadById Load information for brokers by the broker id.
 * @param maxMetricTimestamp Maximum timestamp of the sampled metric during the sampling process.
 * @param cachedNumCoresByBroker Cached number of cores by broker.
 * @param skippedPartitionByBroker Number of skipped partition samples by broker ids.
 * @return Metric sample populated with topic and partition metrics, or {@code null} if sample generation is skipped.
 */
static PartitionMetricSample buildPartitionMetricSample(Cluster cluster, Map<Integer, Map<String, Integer>> leaderDistribution, TopicPartition tpDotNotHandled, Map<Integer, BrokerLoad> brokerLoadById, long maxMetricTimestamp, Map<Integer, Short> cachedNumCoresByBroker, Map<Integer, Integer> skippedPartitionByBroker) {
    Node leaderNode = cluster.leaderFor(tpDotNotHandled);
    if (leaderNode == null) {
        LOG.trace("Partition {} has no current leader.", tpDotNotHandled);
        skippedPartitionByBroker.merge(UNRECOGNIZED_BROKER_ID, 1, Integer::sum);
        return null;
    }
    int leaderId = leaderNode.id();
    // TODO: switch to linear regression model without computing partition level CPU usage.
    BrokerLoad brokerLoad = brokerLoadById.get(leaderId);
    TopicPartition tpWithDotHandled = partitionHandleDotInTopicName(tpDotNotHandled);
    if (skipBuildingPartitionMetricSample(tpDotNotHandled, tpWithDotHandled, leaderId, brokerLoad, cachedNumCoresByBroker)) {
        skippedPartitionByBroker.merge(leaderId, 1, Integer::sum);
        return null;
    }
    // Fill in all the common metrics.
    MetricDef commonMetricDef = KafkaMetricDef.commonMetricDef();
    PartitionMetricSample pms = new PartitionMetricSample(leaderId, tpDotNotHandled);
    int numLeaders = leaderDistribution.get(leaderId).get(tpDotNotHandled.topic());
    for (RawMetricType rawMetricType : RawMetricType.topicMetricTypes()) {
        double sampleValue = numLeaders == 0 ? 0 : (brokerLoad.topicMetrics(tpWithDotHandled.topic(), rawMetricType)) / numLeaders;
        MetricInfo metricInfo = commonMetricDef.metricInfo(KafkaMetricDef.forRawMetricType(rawMetricType).name());
        pms.record(metricInfo, sampleValue);
    }
    // Fill in disk and CPU utilization, which are not topic metric types.
    Double partitionSize = brokerLoad.partitionMetric(tpWithDotHandled.topic(), tpWithDotHandled.partition(), PARTITION_SIZE);
    if (partitionSize == null) {
        skippedPartitionByBroker.merge(leaderId, 1, Integer::sum);
        return null;
    }
    pms.record(commonMetricDef.metricInfo(KafkaMetricDef.DISK_USAGE.name()), partitionSize);
    Double estimatedLeaderCpuUtil = estimateLeaderCpuUtil(pms, brokerLoad, commonMetricDef, cachedNumCoresByBroker.get(leaderId));
    if (estimatedLeaderCpuUtil == null) {
        skippedPartitionByBroker.merge(leaderId, 1, Integer::sum);
        return null;
    }
    pms.record(commonMetricDef.metricInfo(KafkaMetricDef.CPU_USAGE.name()), estimatedLeaderCpuUtil);
    pms.close(maxMetricTimestamp);
    return pms;
}