Examples with CruiseControlMetric com.linkedin.kafka.cruisecontrol.metricsreporter.metric.CruiseControlMetric used on opensource projects

Example 11 with CruiseControlMetric

use of com.linkedin.kafka.cruisecontrol.metricsreporter.metric.CruiseControlMetric in project cruise-control by linkedin.

the class CruiseControlMetricsReporterTest method testReportingMetrics.

@Test
public void testReportingMetrics() {
    Properties props = new Properties();
    props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers());
    props.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
    props.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, MetricSerde.class.getName());
    props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "testReportingMetrics");
    props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
    setSecurityConfigs(props, "consumer");
    Consumer<String, CruiseControlMetric> consumer = new KafkaConsumer<>(props);
    consumer.subscribe(Collections.singleton(TOPIC));
    long startMs = System.currentTimeMillis();
    HashSet<Integer> expectedMetricTypes = new HashSet<>(Arrays.asList((int) ALL_TOPIC_BYTES_IN.id(), (int) ALL_TOPIC_BYTES_OUT.id(), (int) TOPIC_BYTES_IN.id(), (int) TOPIC_BYTES_OUT.id(), (int) PARTITION_SIZE.id(), (int) BROKER_CPU_UTIL.id(), (int) ALL_TOPIC_REPLICATION_BYTES_IN.id(), (int) ALL_TOPIC_REPLICATION_BYTES_OUT.id(), (int) ALL_TOPIC_PRODUCE_REQUEST_RATE.id(), (int) ALL_TOPIC_FETCH_REQUEST_RATE.id(), (int) ALL_TOPIC_MESSAGES_IN_PER_SEC.id(), (int) TOPIC_PRODUCE_REQUEST_RATE.id(), (int) TOPIC_FETCH_REQUEST_RATE.id(), (int) TOPIC_MESSAGES_IN_PER_SEC.id(), (int) BROKER_PRODUCE_REQUEST_RATE.id(), (int) BROKER_CONSUMER_FETCH_REQUEST_RATE.id(), (int) BROKER_FOLLOWER_FETCH_REQUEST_RATE.id(), (int) BROKER_REQUEST_HANDLER_AVG_IDLE_PERCENT.id(), (int) BROKER_REQUEST_QUEUE_SIZE.id(), (int) BROKER_RESPONSE_QUEUE_SIZE.id(), (int) BROKER_PRODUCE_REQUEST_QUEUE_TIME_MS_MAX.id(), (int) BROKER_PRODUCE_REQUEST_QUEUE_TIME_MS_MEAN.id(), (int) BROKER_CONSUMER_FETCH_REQUEST_QUEUE_TIME_MS_MAX.id(), (int) BROKER_CONSUMER_FETCH_REQUEST_QUEUE_TIME_MS_MEAN.id(), (int) BROKER_FOLLOWER_FETCH_REQUEST_QUEUE_TIME_MS_MAX.id(), (int) BROKER_FOLLOWER_FETCH_REQUEST_QUEUE_TIME_MS_MEAN.id(), (int) BROKER_PRODUCE_TOTAL_TIME_MS_MAX.id(), (int) BROKER_PRODUCE_TOTAL_TIME_MS_MEAN.id(), (int) BROKER_CONSUMER_FETCH_TOTAL_TIME_MS_MAX.id(), (int) BROKER_CONSUMER_FETCH_TOTAL_TIME_MS_MEAN.id(), (int) BROKER_FOLLOWER_FETCH_TOTAL_TIME_MS_MAX.id(), (int) BROKER_FOLLOWER_FETCH_TOTAL_TIME_MS_MEAN.id(), (int) BROKER_PRODUCE_LOCAL_TIME_MS_MAX.id(), (int) BROKER_PRODUCE_LOCAL_TIME_MS_MEAN.id(), (int) BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MAX.id(), (int) BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_MEAN.id(), (int) BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MAX.id(), (int) BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_MEAN.id(), (int) BROKER_LOG_FLUSH_RATE.id(), (int) BROKER_LOG_FLUSH_TIME_MS_MAX.id(), (int) BROKER_LOG_FLUSH_TIME_MS_MEAN.id(), (int) BROKER_PRODUCE_REQUEST_QUEUE_TIME_MS_50TH.id(), (int) BROKER_PRODUCE_REQUEST_QUEUE_TIME_MS_999TH.id(), (int) BROKER_CONSUMER_FETCH_REQUEST_QUEUE_TIME_MS_50TH.id(), (int) BROKER_CONSUMER_FETCH_REQUEST_QUEUE_TIME_MS_999TH.id(), (int) BROKER_FOLLOWER_FETCH_REQUEST_QUEUE_TIME_MS_50TH.id(), (int) BROKER_FOLLOWER_FETCH_REQUEST_QUEUE_TIME_MS_999TH.id(), (int) BROKER_PRODUCE_TOTAL_TIME_MS_50TH.id(), (int) BROKER_PRODUCE_TOTAL_TIME_MS_999TH.id(), (int) BROKER_CONSUMER_FETCH_TOTAL_TIME_MS_50TH.id(), (int) BROKER_CONSUMER_FETCH_TOTAL_TIME_MS_999TH.id(), (int) BROKER_FOLLOWER_FETCH_TOTAL_TIME_MS_50TH.id(), (int) BROKER_FOLLOWER_FETCH_TOTAL_TIME_MS_999TH.id(), (int) BROKER_PRODUCE_LOCAL_TIME_MS_50TH.id(), (int) BROKER_PRODUCE_LOCAL_TIME_MS_999TH.id(), (int) BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_50TH.id(), (int) BROKER_CONSUMER_FETCH_LOCAL_TIME_MS_999TH.id(), (int) BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_50TH.id(), (int) BROKER_FOLLOWER_FETCH_LOCAL_TIME_MS_999TH.id(), (int) BROKER_LOG_FLUSH_TIME_MS_50TH.id(), (int) BROKER_LOG_FLUSH_TIME_MS_999TH.id()));
    Set<Integer> metricTypes = new HashSet<>();
    ConsumerRecords<String, CruiseControlMetric> records;
    while (metricTypes.size() < expectedMetricTypes.size() && System.currentTimeMillis() < startMs + 15000) {
        records = consumer.poll(Duration.ofMillis(10L));
        for (ConsumerRecord<String, CruiseControlMetric> record : records) {
            metricTypes.add((int) record.value().rawMetricType().id());
        }
    }
    assertEquals("Expected " + expectedMetricTypes + ", but saw " + metricTypes, expectedMetricTypes, metricTypes);
}

Example 12 with CruiseControlMetric

use of com.linkedin.kafka.cruisecontrol.metricsreporter.metric.CruiseControlMetric in project cruise-control by linkedin.

the class CruiseControlMetricsProcessorTest method testMissingBrokerCapacity.

@Test(expected = IllegalArgumentException.class)
public void testMissingBrokerCapacity() throws TimeoutException, BrokerCapacityResolutionException {
    Set<CruiseControlMetric> metrics = getCruiseControlMetrics();
    // All estimated.
    BrokerCapacityConfigResolver brokerCapacityConfigResolver = EasyMock.mock(BrokerCapacityConfigResolver.class);
    EasyMock.expect(brokerCapacityConfigResolver.capacityForBroker(EasyMock.anyString(), EasyMock.anyString(), EasyMock.anyInt(), EasyMock.anyLong(), EasyMock.anyBoolean())).andReturn(new BrokerCapacityInfo(Collections.emptyMap(), Collections.emptyMap(), MOCK_NUM_CPU_CORES)).anyTimes();
    EasyMock.replay(brokerCapacityConfigResolver);
    CruiseControlMetricsProcessor processor = new CruiseControlMetricsProcessor(brokerCapacityConfigResolver, false);
    for (CruiseControlMetric cruiseControlMetric : metrics) {
        processor.addMetric(cruiseControlMetric);
    }
    EasyMock.verify(brokerCapacityConfigResolver);
    Cluster cluster = getCluster();
    processor.process(cluster, TEST_PARTITIONS, MetricSampler.SamplingMode.ALL);
}

Example 13 with CruiseControlMetric

use of com.linkedin.kafka.cruisecontrol.metricsreporter.metric.CruiseControlMetric in project cruise-control by linkedin.

the class CruiseControlMetricsProcessorTest method getCruiseControlMetrics.

/**
 * <ul>
 * <li>T1P0(B0): NW_IN = {@link #B0_TOPIC1_BYTES_IN} KB, NW_OUT = {@link #B0_TOPIC1_BYTES_OUT} KB,
 * size = {@link #T1P0_BYTES_SIZE} MB</li>
 * <li>T1P1(B1): NW_IN = {@link #B1_TOPIC1_BYTES_IN} KB, NW_OUT = {@link #B1_TOPIC1_BYTES_OUT} KB,
 * size = {@link #T1P1_BYTES_SIZE} MB</li>
 * <li>T2P0(B0): NW_IN = est. {@link #B0_TOPIC2_BYTES_IN}/2 KB, NW_OUT = est. {@link #B0_TOPIC2_BYTES_OUT}/2 KB,
 * size = {@link #T2P0_BYTES_SIZE} MB</li>
 * <li>T2P1(B0): NW_IN = est. {@link #B0_TOPIC2_BYTES_IN}/2 KB, NW_OUT = est. {@link #B0_TOPIC2_BYTES_OUT}/2 KB,
 * size = {@link #T2P1_BYTES_SIZE} MB</li>
 * <li>B0: CPU = {@link #B0_CPU}%</li>
 * <li>B1: CPU = {@link #B1_CPU}%</li>
 * </ul>
 * @return Cruise Control metrics.
 */
private Set<CruiseControlMetric> getCruiseControlMetrics() {
    Set<CruiseControlMetric> metrics = new HashSet<>();
    int i = 0;
    for (RawMetricType rawMetricType : RawMetricType.brokerMetricTypesDiffForVersion(BrokerMetricSample.MIN_SUPPORTED_VERSION)) {
        switch(rawMetricType) {
            case ALL_TOPIC_BYTES_IN:
                metrics.add(new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_IN, _time.milliseconds(), BROKER_ID_0, B0_ALL_TOPIC_BYTES_IN * BYTES_IN_KB));
                metrics.add(new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_IN, _time.milliseconds(), BROKER_ID_1, B1_ALL_TOPIC_BYTES_IN * BYTES_IN_KB));
                break;
            case ALL_TOPIC_BYTES_OUT:
                metrics.add(new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, B0_ALL_TOPIC_BYTES_OUT * BYTES_IN_KB));
                metrics.add(new BrokerMetric(RawMetricType.ALL_TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_1, B1_ALL_TOPIC_BYTES_OUT * BYTES_IN_KB));
                break;
            case BROKER_CPU_UTIL:
                metrics.add(new BrokerMetric(RawMetricType.BROKER_CPU_UTIL, _time.milliseconds(), BROKER_ID_0, B0_CPU));
                metrics.add(new BrokerMetric(RawMetricType.BROKER_CPU_UTIL, _time.milliseconds(), BROKER_ID_1, B1_CPU));
                break;
            default:
                metrics.add(new BrokerMetric(rawMetricType, _time.milliseconds(), BROKER_ID_0, i++ * BYTES_IN_MB));
                metrics.add(new BrokerMetric(rawMetricType, _time.milliseconds(), BROKER_ID_1, i++ * BYTES_IN_MB));
                break;
        }
    }
    for (RawMetricType rawMetricType : RawMetricType.topicMetricTypes()) {
        switch(rawMetricType) {
            case TOPIC_BYTES_IN:
                metrics.add(new TopicMetric(TOPIC_BYTES_IN, _time.milliseconds() + 1, BROKER_ID_0, TOPIC1, B0_TOPIC1_BYTES_IN * BYTES_IN_KB));
                metrics.add(new TopicMetric(TOPIC_BYTES_IN, _time.milliseconds() + 2, BROKER_ID_1, TOPIC1, B1_TOPIC1_BYTES_IN * BYTES_IN_KB));
                metrics.add(new TopicMetric(TOPIC_BYTES_IN, _time.milliseconds(), BROKER_ID_0, TOPIC2, B0_TOPIC2_BYTES_IN * BYTES_IN_KB));
                break;
            case TOPIC_BYTES_OUT:
                metrics.add(new TopicMetric(RawMetricType.TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC1, B0_TOPIC1_BYTES_OUT * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_1, TOPIC1, B1_TOPIC1_BYTES_OUT * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC2, B0_TOPIC2_BYTES_OUT * BYTES_IN_KB));
                break;
            case TOPIC_REPLICATION_BYTES_IN:
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_IN, _time.milliseconds(), BROKER_ID_1, TOPIC1, B1_TOPIC1_REPLICATION_BYTES_IN * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_IN, _time.milliseconds(), BROKER_ID_0, TOPIC1, B0_TOPIC1_REPLICATION_BYTES_IN * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_IN, _time.milliseconds(), BROKER_ID_1, TOPIC2, B1_TOPIC2_REPLICATION_BYTES_IN * BYTES_IN_KB));
                break;
            case TOPIC_REPLICATION_BYTES_OUT:
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC1, B0_TOPIC1_REPLICATION_BYTES_OUT * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_OUT, _time.milliseconds(), BROKER_ID_1, TOPIC1, B1_TOPIC1_REPLICATION_BYTES_OUT * BYTES_IN_KB));
                metrics.add(new TopicMetric(RawMetricType.TOPIC_REPLICATION_BYTES_OUT, _time.milliseconds(), BROKER_ID_0, TOPIC2, B0_TOPIC2_REPLICATION_BYTES_OUT * BYTES_IN_KB));
                break;
            default:
                metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_0, TOPIC1, i * BYTES_IN_MB));
                metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_1, TOPIC1, i * BYTES_IN_MB));
                metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_0, TOPIC2, i * BYTES_IN_MB));
                metrics.add(new TopicMetric(rawMetricType, _time.milliseconds(), BROKER_ID_1, TOPIC2, i * BYTES_IN_MB));
                break;
        }
    }
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_0, TOPIC1, P0, T1P0_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_0, TOPIC1, P1, T1P1_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_0, TOPIC2, P0, T2P0_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_0, TOPIC2, P1, T2P1_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_1, TOPIC1, P0, T1P0_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_1, TOPIC1, P1, T1P1_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_1, TOPIC2, P0, T2P0_BYTES_SIZE * BYTES_IN_MB));
    metrics.add(new PartitionMetric(RawMetricType.PARTITION_SIZE, _time.milliseconds(), BROKER_ID_1, TOPIC2, P1, T2P1_BYTES_SIZE * BYTES_IN_MB));
    return metrics;
}

Example 14 with CruiseControlMetric

use of com.linkedin.kafka.cruisecontrol.metricsreporter.metric.CruiseControlMetric in project cruise-control by linkedin.

the class CruiseControlMetricsProcessorTest method testMissingTopicBytesInMetric.

@Test
public void testMissingTopicBytesInMetric() throws TimeoutException, BrokerCapacityResolutionException {
    CruiseControlMetricsProcessor processor = new CruiseControlMetricsProcessor(mockBrokerCapacityConfigResolver(), false);
    Set<CruiseControlMetric> metrics = getCruiseControlMetrics();
    Set<RawMetricType> metricTypeToExclude = new HashSet<>(Arrays.asList(TOPIC_BYTES_IN, TOPIC_BYTES_OUT, TOPIC_REPLICATION_BYTES_IN, TOPIC_REPLICATION_BYTES_OUT));
    for (CruiseControlMetric metric : metrics) {
        if (metricTypeToExclude.contains(metric.rawMetricType())) {
            TopicMetric tm = (TopicMetric) metric;
            if (tm.brokerId() == BROKER_ID_0 && tm.topic().equals(TOPIC1)) {
                continue;
            }
        }
        processor.addMetric(metric);
    }
    Cluster cluster = getCluster();
    MetricSampler.Samples samples = processor.process(cluster, TEST_PARTITIONS, MetricSampler.SamplingMode.ALL);
    assertEquals(4, samples.partitionMetricSamples().size());
    assertEquals(2, samples.brokerMetricSamples().size());
    for (PartitionMetricSample sample : samples.partitionMetricSamples()) {
        if (sample.entity().tp().equals(T1P0)) {
            // T1P0 should not have any IO or CPU usage.
            validatePartitionMetricSample(sample, _time.milliseconds() + 2, 0.0, 0.0, 0.0, T1P0_BYTES_SIZE);
        }
    }
}