Examples with ValuesAndExtrapolations com.linkedin.cruisecontrol.monitor.sampling.aggregator.ValuesAndExtrapolations used on opensource projects

Example 1 with ValuesAndExtrapolations

use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.ValuesAndExtrapolations in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method testRecentSnapshot.

@Test
public void testRecentSnapshot() throws NotEnoughValidWindowsException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
    populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
    MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), Long.MAX_VALUE, new OperationProgress());
    Map<PartitionEntity, ValuesAndExtrapolations> snapshotsForPartition = result.valuesAndExtrapolations();
    assertEquals("The snapshots should only have one partition", 1, snapshotsForPartition.size());
    ValuesAndExtrapolations snapshots = snapshotsForPartition.get(PE);
    assertNotNull(snapshots);
    assertEquals(NUM_WINDOWS, snapshots.metricValues().length());
    for (int i = 0; i < NUM_WINDOWS; i++) {
        assertEquals((NUM_WINDOWS - i) * WINDOW_MS, result.valuesAndExtrapolations().get(PE).window(i));
        for (Resource resource : Resource.values()) {
            double expectedValue = resource == Resource.DISK ? (NUM_WINDOWS - 1 - i) * 10 + MIN_SAMPLES_PER_WINDOW - 1 : (NUM_WINDOWS - 1 - i) * 10 + (MIN_SAMPLES_PER_WINDOW - 1) / 2.0;
            assertEquals("The utilization for " + resource + " should be " + expectedValue, expectedValue, snapshots.metricValues().valuesFor(KafkaCruiseControlMetricDef.resourceToMetricId(resource)).get(i), 0);
        }
    }
    // Verify the metric completeness checker state
    MetadataClient.ClusterAndGeneration clusterAndGeneration = new MetadataClient.ClusterAndGeneration(metadata.fetch(), 1);
    assertEquals(NUM_WINDOWS, metricSampleAggregator.validWindows(clusterAndGeneration, 1.0).size());
    Map<Long, Float> monitoredPercentages = metricSampleAggregator.partitionCoverageByWindows(clusterAndGeneration);
    for (double percentage : monitoredPercentages.values()) {
        assertEquals(1.0, percentage, 0.0);
    }
    assertEquals(NUM_WINDOWS, metricSampleAggregator.availableWindows().size());
}

Example 2 with ValuesAndExtrapolations

use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.ValuesAndExtrapolations in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method testSnapshotWithUpdatedCluster.

@Test
public void testSnapshotWithUpdatedCluster() throws NotEnoughValidWindowsException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
    populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
    TopicPartition tp1 = new TopicPartition(TOPIC, 1);
    Cluster cluster = getCluster(Arrays.asList(TP, tp1));
    metadata.update(cluster, Collections.emptySet(), 1);
    Map<PartitionEntity, ValuesAndExtrapolations> snapshotsForPartition = metricSampleAggregator.aggregate(clusterAndGeneration(cluster), Long.MAX_VALUE, new OperationProgress()).valuesAndExtrapolations();
    assertTrue("tp1 should not be included because recent snapshot does not include all topics", snapshotsForPartition.isEmpty());
    ModelCompletenessRequirements requirements = new ModelCompletenessRequirements(1, 0.0, true);
    MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(cluster), -1, Long.MAX_VALUE, requirements, new OperationProgress());
    snapshotsForPartition = result.valuesAndExtrapolations();
    assertNotNull("tp1 should be included because includeAllTopics is set to true", snapshotsForPartition.get(new PartitionEntity(tp1)));
    Map<Integer, Extrapolation> extrapolationss = snapshotsForPartition.get(new PartitionEntity(tp1)).extrapolations();
    assertEquals(NUM_WINDOWS, extrapolationss.size());
    for (int i = 0; i < NUM_WINDOWS; i++) {
        assertEquals(Extrapolation.NO_VALID_EXTRAPOLATION, extrapolationss.get(i));
    }
}

Example 3 with ValuesAndExtrapolations

use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.ValuesAndExtrapolations in project cruise-control by linkedin.

the class LoadMonitor method partitionSampleExtrapolations.

private Map<TopicPartition, List<SampleExtrapolation>> partitionSampleExtrapolations(Map<PartitionEntity, ValuesAndExtrapolations> valuesAndExtrapolations) {
    Map<TopicPartition, List<SampleExtrapolation>> sampleExtrapolations = new HashMap<>();
    for (Map.Entry<PartitionEntity, ValuesAndExtrapolations> entry : valuesAndExtrapolations.entrySet()) {
        TopicPartition tp = entry.getKey().tp();
        Map<Integer, Extrapolation> extrapolations = entry.getValue().extrapolations();
        if (extrapolations.isEmpty()) {
            List<SampleExtrapolation> extrapolationForPartition = sampleExtrapolations.computeIfAbsent(tp, p -> new ArrayList<>());
            extrapolations.forEach((t, imputation) -> extrapolationForPartition.add(new SampleExtrapolation(t, imputation)));
        }
    }
    return sampleExtrapolations;
}

Example 4 with ValuesAndExtrapolations

use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.ValuesAndExtrapolations in project cruise-control by linkedin.

the class LoadMonitor method state.

/**
 * Get the state of the load monitor.
 */
public LoadMonitorState state(OperationProgress operationProgress) {
    LoadMonitorTaskRunner.LoadMonitorTaskRunnerState state = _loadMonitorTaskRunner.state();
    MetadataClient.ClusterAndGeneration clusterAndGeneration = _metadataClient.refreshMetadata();
    Cluster kafkaCluster = clusterAndGeneration.cluster();
    int totalNumPartitions = MonitorUtils.totalNumPartitions(kafkaCluster);
    double minMonitoredPartitionsPercentage = _defaultModelCompletenessRequirements.minMonitoredPartitionsPercentage();
    // Get the window to monitored partitions percentage mapping.
    SortedMap<Long, Float> validPartitionRatio = _metricSampleAggregator.partitionCoverageByWindows(clusterAndGeneration);
    // Get valid snapshot window number and populate the monitored partition map.
    // We do this primarily because the checker and aggregator are not always synchronized.
    SortedSet<Long> validWindows = _metricSampleAggregator.validWindows(clusterAndGeneration, minMonitoredPartitionsPercentage);
    int numValidSnapshotWindows = validWindows.size();
    // Get the number of valid partitions and sample extrapolations.
    int numValidPartitions = 0;
    Map<TopicPartition, List<SampleExtrapolation>> extrapolations = Collections.emptyMap();
    if (_metricSampleAggregator.numAvailableWindows() >= _numWindows) {
        try {
            MetricSampleAggregationResult<String, PartitionEntity> metricSampleAggregationResult = _metricSampleAggregator.aggregate(clusterAndGeneration, Long.MAX_VALUE, operationProgress);
            Map<PartitionEntity, ValuesAndExtrapolations> loads = metricSampleAggregationResult.valuesAndExtrapolations();
            extrapolations = partitionSampleExtrapolations(metricSampleAggregationResult.valuesAndExtrapolations());
            numValidPartitions = loads.size();
        } catch (Exception e) {
            LOG.warn("Received exception when trying to get the load monitor state", e);
        }
    }
    switch(state) {
        case NOT_STARTED:
            return LoadMonitorState.notStarted();
        case RUNNING:
            return LoadMonitorState.running(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, extrapolations);
        case SAMPLING:
            return LoadMonitorState.sampling(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, extrapolations);
        case PAUSED:
            return LoadMonitorState.paused(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, extrapolations);
        case BOOTSTRAPPING:
            double bootstrapProgress = _loadMonitorTaskRunner.bootStrapProgress();
            // Handle the race between querying the state and getting the progress.
            return LoadMonitorState.bootstrapping(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, bootstrapProgress >= 0 ? bootstrapProgress : 1.0, extrapolations);
        case TRAINING:
            return LoadMonitorState.training(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, extrapolations);
        case LOADING:
            return LoadMonitorState.loading(numValidSnapshotWindows, validPartitionRatio, numValidPartitions, totalNumPartitions, _loadMonitorTaskRunner.sampleLoadingProgress());
        default:
            throw new IllegalStateException("Should never be here.");
    }
}

Example 5 with ValuesAndExtrapolations

use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.ValuesAndExtrapolations in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method testExcludeInvalidMetricSample.

@Test
public void testExcludeInvalidMetricSample() throws NotEnoughValidWindowsException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
    MetricDef metricDef = KafkaCruiseControlMetricDef.metricDef();
    populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
    // Set the leader to be node 1, which is different from the leader in the metadata.
    PartitionMetricSample sampleWithDifferentLeader = new PartitionMetricSample(1, TP);
    sampleWithDifferentLeader.record(metricDef.metricInfo(DISK_USAGE.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_IN.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_OUT.name()), 10000);
    sampleWithDifferentLeader.close(0);
    // Only populate the CPU metric
    PartitionMetricSample incompletePartitionMetricSample = new PartitionMetricSample(0, TP);
    incompletePartitionMetricSample.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
    incompletePartitionMetricSample.close(0);
    metricSampleAggregator.addSample(sampleWithDifferentLeader);
    metricSampleAggregator.addSample(incompletePartitionMetricSample);
    // Check the snapshot value and make sure the metric samples above are excluded.
    Map<PartitionEntity, ValuesAndExtrapolations> snapshotsForPartition = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress()).valuesAndExtrapolations();
    ValuesAndExtrapolations snapshots = snapshotsForPartition.get(PE);
    for (Resource resource : Resource.values()) {
        int metricId = KafkaCruiseControlMetricDef.resourceToMetricId(resource);
        double expectedValue = resource == Resource.DISK ? MIN_SAMPLES_PER_WINDOW - 1 : (MIN_SAMPLES_PER_WINDOW - 1) / 2.0;
        assertEquals("The utilization for " + resource + " should be " + expectedValue, expectedValue, snapshots.metricValues().valuesFor(metricId).get(NUM_WINDOWS - 1), 0);
    }
}