Examples with PartitionEntity com.linkedin.kafka.cruisecontrol.monitor.sampling.PartitionEntity used on opensource projects

Example 11 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.PartitionEntity in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method testExcludeInvalidMetricSample.

@Test
public void testExcludeInvalidMetricSample() throws NotEnoughValidWindowsException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
    MetricDef metricDef = KafkaCruiseControlMetricDef.metricDef();
    populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
    // Set the leader to be node 1, which is different from the leader in the metadata.
    PartitionMetricSample sampleWithDifferentLeader = new PartitionMetricSample(1, TP);
    sampleWithDifferentLeader.record(metricDef.metricInfo(DISK_USAGE.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_IN.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_OUT.name()), 10000);
    sampleWithDifferentLeader.close(0);
    // Only populate the CPU metric
    PartitionMetricSample incompletePartitionMetricSample = new PartitionMetricSample(0, TP);
    incompletePartitionMetricSample.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
    incompletePartitionMetricSample.close(0);
    metricSampleAggregator.addSample(sampleWithDifferentLeader);
    metricSampleAggregator.addSample(incompletePartitionMetricSample);
    // Check the snapshot value and make sure the metric samples above are excluded.
    Map<PartitionEntity, ValuesAndExtrapolations> snapshotsForPartition = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress()).valuesAndExtrapolations();
    ValuesAndExtrapolations snapshots = snapshotsForPartition.get(PE);
    for (Resource resource : Resource.values()) {
        int metricId = KafkaCruiseControlMetricDef.resourceToMetricId(resource);
        double expectedValue = resource == Resource.DISK ? MIN_SAMPLES_PER_WINDOW - 1 : (MIN_SAMPLES_PER_WINDOW - 1) / 2.0;
        assertEquals("The utilization for " + resource + " should be " + expectedValue, expectedValue, snapshots.metricValues().valuesFor(metricId).get(NUM_WINDOWS - 1), 0);
    }
}

Example 12 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.PartitionEntity in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method testTooManyFlaws.

@Test
public void testTooManyFlaws() throws NotEnoughValidWindowsException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
    // Only give two samples to the aggregator.
    CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 2, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator, PE, 3, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
    MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress());
    assertTrue(result.valuesAndExtrapolations().isEmpty());
}

Example 13 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.PartitionEntity in project cruise-control by linkedin.

the class KafkaMetricSampleAggregator method allPartitions.

private Set<PartitionEntity> allPartitions(Cluster cluster) {
    Set<PartitionEntity> allPartitions = new HashSet<>();
    for (Node node : cluster.nodes()) {
        for (PartitionInfo partitionInfo : cluster.partitionsForNode(node.id())) {
            TopicPartition tp = new TopicPartition(partitionInfo.topic(), partitionInfo.partition());
            PartitionEntity partitionEntity = new PartitionEntity(tp);
            allPartitions.add(_identityEntityMap.computeIfAbsent(partitionEntity, k -> partitionEntity));
        }
    }
    return allPartitions;
}

Example 14 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.PartitionEntity in project cruise-control by linkedin.

the class LoadMonitor method getMonitoredPartitionsPercentage.

private double getMonitoredPartitionsPercentage() {
    MetadataClient.ClusterAndGeneration clusterAndGeneration = _metadataClient.refreshMetadata();
    Cluster kafkaCluster = clusterAndGeneration.cluster();
    MetricSampleAggregationResult<String, PartitionEntity> metricSampleAggregationResult;
    try {
        metricSampleAggregationResult = _metricSampleAggregator.aggregate(clusterAndGeneration, System.currentTimeMillis(), new OperationProgress());
    } catch (NotEnoughValidWindowsException e) {
        return 0.0;
    }
    Map<PartitionEntity, ValuesAndExtrapolations> partitionLoads = metricSampleAggregationResult.valuesAndExtrapolations();
    AtomicInteger numPartitionsWithExtrapolations = new AtomicInteger(0);
    partitionLoads.values().forEach(valuesAndExtrapolations -> {
        if (!valuesAndExtrapolations.extrapolations().isEmpty()) {
            numPartitionsWithExtrapolations.incrementAndGet();
        }
    });
    _numPartitionsWithExtrapolations = numPartitionsWithExtrapolations.get();
    int totalNumPartitions = MonitorUtils.totalNumPartitions(kafkaCluster);
    return totalNumPartitions > 0 ? metricSampleAggregationResult.completeness().validEntityRatio() : 0.0;
}