Examples with PartitionEntity com.linkedin.kafka.cruisecontrol.monitor.sampling.holder.PartitionEntity used on opensource projects

Example 6 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.holder.PartitionEntity in project cruise-control by linkedin.

the class KafkaPartitionMetricSampleAggregatorTest method setupScenario3.

/**
 * Three topics with 2 partitions each.
 * T0P1 missing window 18000 (index=17), 19000 (index=18)
 * T1P1 missing window 6000 (index=5), 7000 (index=6)
 * Other partitions have all data.
 * @return Setup scenario #3
 */
private TestContext setupScenario3() {
    TopicPartition t0p1 = new TopicPartition(TOPIC0, 1);
    TopicPartition t1p0 = new TopicPartition("TOPIC1", 0);
    TopicPartition t1p1 = new TopicPartition("TOPIC1", 1);
    TopicPartition t2p0 = new TopicPartition("TOPIC2", 0);
    TopicPartition t2p1 = new TopicPartition("TOPIC2", 1);
    List<TopicPartition> allPartitions = Arrays.asList(TP, t0p1, t1p0, t1p1, t2p0, t2p1);
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(allPartitions);
    KafkaPartitionMetricSampleAggregator aggregator = new KafkaPartitionMetricSampleAggregator(config, metadata);
    for (TopicPartition tp : Arrays.asList(TP, t1p0, t2p0, t2p1)) {
        populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, aggregator, tp);
    }
    // Let t0p1 miss the second and the third latest window.
    populateSampleAggregator(NUM_WINDOWS - 3, MIN_SAMPLES_PER_WINDOW, aggregator, t0p1);
    CruiseControlUnitTestUtils.populateSampleAggregator(2, MIN_SAMPLES_PER_WINDOW, aggregator, new PartitionEntity(t0p1), NUM_WINDOWS - 1, WINDOW_MS, KafkaMetricDef.commonMetricDef());
    // let t1p1 miss another earlier window
    populateSampleAggregator(5, MIN_SAMPLES_PER_WINDOW, aggregator, t1p1);
    CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 6, MIN_SAMPLES_PER_WINDOW, aggregator, new PartitionEntity(t1p1), 7, WINDOW_MS, KafkaMetricDef.commonMetricDef());
    return new TestContext(metadata, aggregator);
}

Example 7 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.holder.PartitionEntity in project cruise-control by linkedin.

the class KafkaPartitionMetricSampleAggregatorTest method testAggregate.

@Test
public void testAggregate() throws NotEnoughValidWindowsException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaPartitionMetricSampleAggregator metricSampleAggregator = new KafkaPartitionMetricSampleAggregator(config, metadata);
    populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
    MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(metadata.fetch(), Long.MAX_VALUE, new OperationProgress());
    Map<PartitionEntity, ValuesAndExtrapolations> valuesAndExtrapolations = result.valuesAndExtrapolations();
    assertEquals("The windows should only have one partition", 1, valuesAndExtrapolations.size());
    ValuesAndExtrapolations partitionValuesAndExtrapolations = valuesAndExtrapolations.get(PE);
    assertNotNull(partitionValuesAndExtrapolations);
    assertEquals(NUM_WINDOWS, partitionValuesAndExtrapolations.metricValues().length());
    for (int i = 0; i < NUM_WINDOWS; i++) {
        assertEquals((NUM_WINDOWS - i) * WINDOW_MS, result.valuesAndExtrapolations().get(PE).window(i));
        for (Resource resource : Resource.cachedValues()) {
            Collection<Short> metricIds = KafkaMetricDef.resourceToMetricIds(resource);
            double expectedValue = (resource == Resource.DISK ? (NUM_WINDOWS - 1 - i) * 10 + MIN_SAMPLES_PER_WINDOW - 1 : (NUM_WINDOWS - 1 - i) * 10 + (MIN_SAMPLES_PER_WINDOW - 1) / 2.0) / (resource == Resource.CPU ? UNIT_INTERVAL_TO_PERCENTAGE : 1.0) * metricIds.size();
            assertEquals("The utilization for " + resource + " should be " + expectedValue, expectedValue, partitionValuesAndExtrapolations.metricValues().valuesForGroup(resource.name(), KafkaMetricDef.commonMetricDef(), true).get(i), 0.01);
        }
    }
    // Verify the metric completeness checker state
    MetadataClient.ClusterAndGeneration clusterAndGeneration = new MetadataClient.ClusterAndGeneration(metadata.fetch(), 1);
    assertEquals(NUM_WINDOWS, metricSampleAggregator.validWindows(clusterAndGeneration.cluster(), 1.0).size());
    Map<Long, Float> monitoredPercentages = metricSampleAggregator.validPartitionRatioByWindows(clusterAndGeneration.cluster());
    for (double percentage : monitoredPercentages.values()) {
        assertEquals(1.0, percentage, 0.0);
    }
    assertEquals(NUM_WINDOWS, metricSampleAggregator.availableWindows().size());
}

Example 8 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.holder.PartitionEntity in project cruise-control by linkedin.

the class KafkaPartitionMetricSampleAggregator method allPartitions.

private Set<PartitionEntity> allPartitions(Cluster cluster) {
    Set<PartitionEntity> allPartitions = new HashSet<>();
    for (String topic : cluster.topics()) {
        for (PartitionInfo partitionInfo : cluster.partitionsForTopic(topic)) {
            TopicPartition tp = new TopicPartition(partitionInfo.topic(), partitionInfo.partition());
            PartitionEntity partitionEntity = new PartitionEntity(tp);
            allPartitions.add(identity(partitionEntity));
        }
    }
    return allPartitions;
}

Example 9 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.holder.PartitionEntity in project cruise-control by linkedin.

the class LoadMonitor method getMonitoredPartitionsPercentage.

private double getMonitoredPartitionsPercentage() {
    MetadataClient.ClusterAndGeneration clusterAndGeneration = refreshClusterAndGeneration();
    Cluster kafkaCluster = clusterAndGeneration.cluster();
    MetricSampleAggregationResult<String, PartitionEntity> metricSampleAggregationResult;
    try {
        metricSampleAggregationResult = _partitionMetricSampleAggregator.aggregate(kafkaCluster, _time.milliseconds(), new OperationProgress());
    } catch (NotEnoughValidWindowsException e) {
        LOG.debug("Not enough valid windows to get monitored partitions. {}", e.getMessage());
        return 0.0;
    }
    Map<PartitionEntity, ValuesAndExtrapolations> partitionLoads = metricSampleAggregationResult.valuesAndExtrapolations();
    AtomicInteger numPartitionsWithExtrapolations = new AtomicInteger(0);
    partitionLoads.values().forEach(valuesAndExtrapolations -> {
        if (!valuesAndExtrapolations.extrapolations().isEmpty()) {
            numPartitionsWithExtrapolations.incrementAndGet();
        }
    });
    _numPartitionsWithExtrapolations = numPartitionsWithExtrapolations.get();
    _totalNumPartitions = MonitorUtils.totalNumPartitions(kafkaCluster);
    return _totalNumPartitions > 0 ? metricSampleAggregationResult.validEntityRatioOfCompleteness() : 0.0;
}

Example 10 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.holder.PartitionEntity in project cruise-control by linkedin.

the class KafkaPartitionMetricSampleAggregatorTest method setupScenario2.

/**
 * Two topics with 2 partitions each.
 * T1P1 misses window 6000 (index=5), 7000 (index=6) and 20000 (index=19)
 * Other partitions has full data.
 * @return Setup scenario #2
 */
private TestContext setupScenario2() {
    TopicPartition t0p1 = new TopicPartition(TOPIC0, 1);
    TopicPartition t1p0 = new TopicPartition("TOPIC1", 0);
    TopicPartition t1p1 = new TopicPartition("TOPIC1", 1);
    List<TopicPartition> allPartitions = Arrays.asList(TP, t0p1, t1p0, t1p1);
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(allPartitions);
    KafkaPartitionMetricSampleAggregator aggregator = new KafkaPartitionMetricSampleAggregator(config, metadata);
    for (TopicPartition tp : Arrays.asList(TP, t0p1, t1p0)) {
        populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, aggregator, tp);
    }
    // Let t1p1 miss two consecutive windows and the most recent window.
    populateSampleAggregator(5, MIN_SAMPLES_PER_WINDOW, aggregator, t1p1);
    CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 8, MIN_SAMPLES_PER_WINDOW, aggregator, new PartitionEntity(t1p1), 7, WINDOW_MS, KafkaMetricDef.commonMetricDef());
    return new TestContext(metadata, aggregator);
}