Examples with PartitionEntity com.linkedin.kafka.cruisecontrol.monitor.sampling.PartitionEntity used on opensource projects

Example 6 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.PartitionEntity in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method testFallbackToAvgAvailable.

@Test
public void testFallbackToAvgAvailable() throws NotEnoughValidWindowsException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
    // Only give two sample to the aggregator.
    CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator, PE, 2, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
    MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress());
    assertTrue(result.valuesAndExtrapolations().isEmpty());
    populateSampleAggregator(2, MIN_SAMPLES_PER_WINDOW - 2, metricSampleAggregator);
    result = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress());
    int numSnapshots = result.valuesAndExtrapolations().get(PE).metricValues().length();
    assertEquals(NUM_WINDOWS, numSnapshots);
    int numExtrapolationss = 0;
    for (Map.Entry<Integer, Extrapolation> entry : result.valuesAndExtrapolations().get(PE).extrapolations().entrySet()) {
        assertEquals(Extrapolation.AVG_AVAILABLE, entry.getValue());
        numExtrapolationss++;
    }
    assertEquals(2, numExtrapolationss);
}

Example 7 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.PartitionEntity in project cruise-control by linkedin.

the class LoadMonitor method partitionSampleExtrapolations.

private Map<TopicPartition, List<SampleExtrapolation>> partitionSampleExtrapolations(Map<PartitionEntity, ValuesAndExtrapolations> valuesAndExtrapolations) {
    Map<TopicPartition, List<SampleExtrapolation>> sampleExtrapolations = new HashMap<>();
    for (Map.Entry<PartitionEntity, ValuesAndExtrapolations> entry : valuesAndExtrapolations.entrySet()) {
        TopicPartition tp = entry.getKey().tp();
        Map<Integer, Extrapolation> extrapolations = entry.getValue().extrapolations();
        if (extrapolations.isEmpty()) {
            List<SampleExtrapolation> extrapolationForPartition = sampleExtrapolations.computeIfAbsent(tp, p -> new ArrayList<>());
            extrapolations.forEach((t, imputation) -> extrapolationForPartition.add(new SampleExtrapolation(t, imputation)));
        }
    }
    return sampleExtrapolations;
}

Example 8 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.PartitionEntity in project cruise-control by linkedin.

the class LoadMonitor method state.

/**
 * Get the state of the load monitor.
 */
public LoadMonitorState state(OperationProgress operationProgress) {
    LoadMonitorTaskRunner.LoadMonitorTaskRunnerState state = _loadMonitorTaskRunner.state();
    MetadataClient.ClusterAndGeneration clusterAndGeneration = _metadataClient.refreshMetadata();
    Cluster kafkaCluster = clusterAndGeneration.cluster();
    int totalNumPartitions = MonitorUtils.totalNumPartitions(kafkaCluster);
    double minMonitoredPartitionsPercentage = _defaultModelCompletenessRequirements.minMonitoredPartitionsPercentage();
    // Get the window to monitored partitions percentage mapping.
    SortedMap<Long, Float> validPartitionRatio = _metricSampleAggregator.partitionCoverageByWindows(clusterAndGeneration);
    // Get valid snapshot window number and populate the monitored partition map.
    // We do this primarily because the checker and aggregator are not always synchronized.
    SortedSet<Long> validWindows = _metricSampleAggregator.validWindows(clusterAndGeneration, minMonitoredPartitionsPercentage);
    int numValidSnapshotWindows = validWindows.size();
    // Get the number of valid partitions and sample extrapolations.
    int numValidPartitions = 0;
    Map<TopicPartition, List<SampleExtrapolation>> extrapolations = Collections.emptyMap();
    if (_metricSampleAggregator.numAvailableWindows() >= _numWindows) {
        try {
            MetricSampleAggregationResult<String, PartitionEntity> metricSampleAggregationResult = _metricSampleAggregator.aggregate(clusterAndGeneration, Long.MAX_VALUE, operationProgress);
            Map<PartitionEntity, ValuesAndExtrapolations> loads = metricSampleAggregationResult.valuesAndExtrapolations();
            extrapolations = partitionSampleExtrapolations(metricSampleAggregationResult.valuesAndExtrapolations());
            numValidPartitions = loads.size();
        } catch (Exception e) {
            LOG.warn("Received exception when trying to get the load monitor state", e);
        }
    }
    switch(state) {
        case NOT_STARTED:
            return LoadMonitorState.notStarted();
        case RUNNING:
            return LoadMonitorState.running(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, extrapolations);
        case SAMPLING:
            return LoadMonitorState.sampling(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, extrapolations);
        case PAUSED:
            return LoadMonitorState.paused(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, extrapolations);
        case BOOTSTRAPPING:
            double bootstrapProgress = _loadMonitorTaskRunner.bootStrapProgress();
            // Handle the race between querying the state and getting the progress.
            return LoadMonitorState.bootstrapping(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, bootstrapProgress >= 0 ? bootstrapProgress : 1.0, extrapolations);
        case TRAINING:
            return LoadMonitorState.training(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, extrapolations);
        case LOADING:
            return LoadMonitorState.loading(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, _loadMonitorTaskRunner.sampleLoadingProgress());
        default:
            throw new IllegalStateException("Should never be here.");
    }
}

Example 9 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.PartitionEntity in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method setupScenario2.

/**
 * Two topics with 2 partitions each.
 * T1P1 misses window 6000 (index=5), 7000 (index=6) and 20000 (index=19)
 * Other partitions has full data.
 */
private TestContext setupScenario2() {
    TopicPartition t0p1 = new TopicPartition(TOPIC, 1);
    TopicPartition t1p0 = new TopicPartition("TOPIC1", 0);
    TopicPartition t1p1 = new TopicPartition("TOPIC1", 1);
    List<TopicPartition> allPartitions = Arrays.asList(TP, t0p1, t1p0, t1p1);
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(allPartitions);
    KafkaMetricSampleAggregator aggregator = new KafkaMetricSampleAggregator(config, metadata);
    for (TopicPartition tp : Arrays.asList(TP, t0p1, t1p0)) {
        populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, aggregator, tp);
    }
    // Let t1p1 miss two consecutive windows and the most recent window.
    populateSampleAggregator(5, MIN_SAMPLES_PER_WINDOW, aggregator, t1p1);
    CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 8, MIN_SAMPLES_PER_WINDOW, aggregator, new PartitionEntity(t1p1), 7, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
    return new TestContext(metadata, aggregator);
}

Example 10 with PartitionEntity

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.PartitionEntity in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method setupScenario3.

/**
 * Three topics with 2 partitions each.
 * T0P1 missing window 18000 (index=17), 19000 (index=18)
 * T1P1 missing window 6000 (index=5), 7000 (index=6)
 * Other partitions have all data.
 */
private TestContext setupScenario3() {
    TopicPartition t0p1 = new TopicPartition(TOPIC, 1);
    TopicPartition t1p0 = new TopicPartition("TOPIC1", 0);
    TopicPartition t1p1 = new TopicPartition("TOPIC1", 1);
    TopicPartition t2p0 = new TopicPartition("TOPIC2", 0);
    TopicPartition t2p1 = new TopicPartition("TOPIC2", 1);
    List<TopicPartition> allPartitions = Arrays.asList(TP, t0p1, t1p0, t1p1, t2p0, t2p1);
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(allPartitions);
    KafkaMetricSampleAggregator aggregator = new KafkaMetricSampleAggregator(config, metadata);
    for (TopicPartition tp : Arrays.asList(TP, t1p0, t2p0, t2p1)) {
        populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, aggregator, tp);
    }
    // Let t0p1 miss the second and the third latest window.
    populateSampleAggregator(NUM_WINDOWS - 3, MIN_SAMPLES_PER_WINDOW, aggregator, t0p1);
    CruiseControlUnitTestUtils.populateSampleAggregator(2, MIN_SAMPLES_PER_WINDOW, aggregator, new PartitionEntity(t0p1), NUM_WINDOWS - 1, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
    // let t1p1 miss another earlier window
    populateSampleAggregator(5, MIN_SAMPLES_PER_WINDOW, aggregator, t1p1);
    CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 6, MIN_SAMPLES_PER_WINDOW, aggregator, new PartitionEntity(t1p1), 7, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
    return new TestContext(metadata, aggregator);
}