use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class KafkaMetricSampleAggregatorTest method testFallbackToAvgAdjacent.
@Test
public void testFallbackToAvgAdjacent() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
TopicPartition anotherTopicPartition = new TopicPartition("AnotherTopic", 1);
PartitionEntity anotherPartitionEntity = new PartitionEntity(anotherTopicPartition);
Metadata metadata = getMetadata(Arrays.asList(TP, anotherTopicPartition));
KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
// Only give one sample to the aggregator for previous period.
populateSampleAggregator(NUM_WINDOWS, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
// Create let (NUM_SNAPSHOT + 1) have enough samples.
CruiseControlUnitTestUtils.populateSampleAggregator(1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator, PE, NUM_WINDOWS, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
// Let a snapshot window exist but not containing samples for partition 0
CruiseControlUnitTestUtils.populateSampleAggregator(1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator, anotherPartitionEntity, NUM_WINDOWS + 1, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
// Let the rest of the snapshot has enough samples.
CruiseControlUnitTestUtils.populateSampleAggregator(2, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator, PE, NUM_WINDOWS + 2, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS * 2, new OperationProgress());
int numSnapshots = result.valuesAndExtrapolations().get(PE).metricValues().length();
assertEquals(NUM_WINDOWS, numSnapshots);
int numExtrapolationss = 0;
for (Map.Entry<Integer, Extrapolation> entry : result.valuesAndExtrapolations().get(PE).extrapolations().entrySet()) {
assertEquals(Extrapolation.AVG_ADJACENT, entry.getValue());
numExtrapolationss++;
}
assertEquals(1, numExtrapolationss);
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class KafkaMetricSampleAggregatorTest method testSnapshotWithPartitionExtrapolations.
@Test
public void testSnapshotWithPartitionExtrapolations() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
TopicPartition tp1 = new TopicPartition(TOPIC, 1);
Cluster cluster = getCluster(Arrays.asList(TP, tp1));
PartitionEntity pe1 = new PartitionEntity(tp1);
metadata.update(cluster, Collections.emptySet(), 1);
populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
// Populate partition 1 but leave 1 hole at NUM_SNAPSHOT'th window.
CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 2, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator, pe1, 0, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
CruiseControlUnitTestUtils.populateSampleAggregator(2, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator, pe1, NUM_WINDOWS - 1, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(cluster), Long.MAX_VALUE, new OperationProgress());
assertEquals(2, result.valuesAndExtrapolations().size());
assertTrue(result.valuesAndExtrapolations().get(PE).extrapolations().isEmpty());
assertEquals(1, result.valuesAndExtrapolations().get(pe1).extrapolations().size());
assertTrue(result.valuesAndExtrapolations().get(pe1).extrapolations().containsKey(1));
assertEquals((NUM_WINDOWS - 1) * WINDOW_MS, result.valuesAndExtrapolations().get(pe1).window(1));
assertEquals(Extrapolation.AVG_ADJACENT, result.valuesAndExtrapolations().get(pe1).extrapolations().get(1));
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class KafkaMetricSampleAggregatorTest method testRecentSnapshot.
@Test
public void testRecentSnapshot() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), Long.MAX_VALUE, new OperationProgress());
Map<PartitionEntity, ValuesAndExtrapolations> snapshotsForPartition = result.valuesAndExtrapolations();
assertEquals("The snapshots should only have one partition", 1, snapshotsForPartition.size());
ValuesAndExtrapolations snapshots = snapshotsForPartition.get(PE);
assertNotNull(snapshots);
assertEquals(NUM_WINDOWS, snapshots.metricValues().length());
for (int i = 0; i < NUM_WINDOWS; i++) {
assertEquals((NUM_WINDOWS - i) * WINDOW_MS, result.valuesAndExtrapolations().get(PE).window(i));
for (Resource resource : Resource.values()) {
double expectedValue = resource == Resource.DISK ? (NUM_WINDOWS - 1 - i) * 10 + MIN_SAMPLES_PER_WINDOW - 1 : (NUM_WINDOWS - 1 - i) * 10 + (MIN_SAMPLES_PER_WINDOW - 1) / 2.0;
assertEquals("The utilization for " + resource + " should be " + expectedValue, expectedValue, snapshots.metricValues().valuesFor(KafkaCruiseControlMetricDef.resourceToMetricId(resource)).get(i), 0);
}
}
// Verify the metric completeness checker state
MetadataClient.ClusterAndGeneration clusterAndGeneration = new MetadataClient.ClusterAndGeneration(metadata.fetch(), 1);
assertEquals(NUM_WINDOWS, metricSampleAggregator.validWindows(clusterAndGeneration, 1.0).size());
Map<Long, Float> monitoredPercentages = metricSampleAggregator.partitionCoverageByWindows(clusterAndGeneration);
for (double percentage : monitoredPercentages.values()) {
assertEquals(1.0, percentage, 0.0);
}
assertEquals(NUM_WINDOWS, metricSampleAggregator.availableWindows().size());
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class KafkaMetricSampleAggregatorTest method testSnapshotWithUpdatedCluster.
@Test
public void testSnapshotWithUpdatedCluster() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
TopicPartition tp1 = new TopicPartition(TOPIC, 1);
Cluster cluster = getCluster(Arrays.asList(TP, tp1));
metadata.update(cluster, Collections.emptySet(), 1);
Map<PartitionEntity, ValuesAndExtrapolations> snapshotsForPartition = metricSampleAggregator.aggregate(clusterAndGeneration(cluster), Long.MAX_VALUE, new OperationProgress()).valuesAndExtrapolations();
assertTrue("tp1 should not be included because recent snapshot does not include all topics", snapshotsForPartition.isEmpty());
ModelCompletenessRequirements requirements = new ModelCompletenessRequirements(1, 0.0, true);
MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(cluster), -1, Long.MAX_VALUE, requirements, new OperationProgress());
snapshotsForPartition = result.valuesAndExtrapolations();
assertNotNull("tp1 should be included because includeAllTopics is set to true", snapshotsForPartition.get(new PartitionEntity(tp1)));
Map<Integer, Extrapolation> extrapolationss = snapshotsForPartition.get(new PartitionEntity(tp1)).extrapolations();
assertEquals(NUM_WINDOWS, extrapolationss.size());
for (int i = 0; i < NUM_WINDOWS; i++) {
assertEquals(Extrapolation.NO_VALID_EXTRAPOLATION, extrapolationss.get(i));
}
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class KafkaMetricSampleAggregatorTest method testFallbackToAvgAvailable.
@Test
public void testFallbackToAvgAvailable() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
// Only give two sample to the aggregator.
CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator, PE, 2, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress());
assertTrue(result.valuesAndExtrapolations().isEmpty());
populateSampleAggregator(2, MIN_SAMPLES_PER_WINDOW - 2, metricSampleAggregator);
result = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress());
int numSnapshots = result.valuesAndExtrapolations().get(PE).metricValues().length();
assertEquals(NUM_WINDOWS, numSnapshots);
int numExtrapolationss = 0;
for (Map.Entry<Integer, Extrapolation> entry : result.valuesAndExtrapolations().get(PE).extrapolations().entrySet()) {
assertEquals(Extrapolation.AVG_AVAILABLE, entry.getValue());
numExtrapolationss++;
}
assertEquals(2, numExtrapolationss);
}
Aggregations