Examples with KafkaCruiseControlConfig com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig used on opensource projects

Example 21 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method setupScenario3.

/**
 * Three topics with 2 partitions each.
 * T0P1 missing window 18000 (index=17), 19000 (index=18)
 * T1P1 missing window 6000 (index=5), 7000 (index=6)
 * Other partitions have all data.
 */
private TestContext setupScenario3() {
    TopicPartition t0p1 = new TopicPartition(TOPIC, 1);
    TopicPartition t1p0 = new TopicPartition("TOPIC1", 0);
    TopicPartition t1p1 = new TopicPartition("TOPIC1", 1);
    TopicPartition t2p0 = new TopicPartition("TOPIC2", 0);
    TopicPartition t2p1 = new TopicPartition("TOPIC2", 1);
    List<TopicPartition> allPartitions = Arrays.asList(TP, t0p1, t1p0, t1p1, t2p0, t2p1);
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(allPartitions);
    KafkaMetricSampleAggregator aggregator = new KafkaMetricSampleAggregator(config, metadata);
    for (TopicPartition tp : Arrays.asList(TP, t1p0, t2p0, t2p1)) {
        populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, aggregator, tp);
    }
    // Let t0p1 miss the second and the third latest window.
    populateSampleAggregator(NUM_WINDOWS - 3, MIN_SAMPLES_PER_WINDOW, aggregator, t0p1);
    CruiseControlUnitTestUtils.populateSampleAggregator(2, MIN_SAMPLES_PER_WINDOW, aggregator, new PartitionEntity(t0p1), NUM_WINDOWS - 1, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
    // let t1p1 miss another earlier window
    populateSampleAggregator(5, MIN_SAMPLES_PER_WINDOW, aggregator, t1p1);
    CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 6, MIN_SAMPLES_PER_WINDOW, aggregator, new PartitionEntity(t1p1), 7, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
    return new TestContext(metadata, aggregator);
}

Example 22 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method testNotEnoughSnapshots.

@Test
public void testNotEnoughSnapshots() {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
    populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
    try {
        // Only 4 snapshots has smaller timestamp than the timestamp we passed in.
        ModelCompletenessRequirements requirements = new ModelCompletenessRequirements(NUM_WINDOWS, 0.0, false);
        metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), -1L, (NUM_WINDOWS - 1) * WINDOW_MS - 1, requirements, new OperationProgress());
        fail("Should throw NotEnoughValidWindowsException");
    } catch (NotEnoughValidWindowsException nse) {
    // let it go
    }
}

Example 23 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method testExcludeInvalidMetricSample.

@Test
public void testExcludeInvalidMetricSample() throws NotEnoughValidWindowsException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
    MetricDef metricDef = KafkaCruiseControlMetricDef.metricDef();
    populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
    // Set the leader to be node 1, which is different from the leader in the metadata.
    PartitionMetricSample sampleWithDifferentLeader = new PartitionMetricSample(1, TP);
    sampleWithDifferentLeader.record(metricDef.metricInfo(DISK_USAGE.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_IN.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_OUT.name()), 10000);
    sampleWithDifferentLeader.close(0);
    // Only populate the CPU metric
    PartitionMetricSample incompletePartitionMetricSample = new PartitionMetricSample(0, TP);
    incompletePartitionMetricSample.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
    incompletePartitionMetricSample.close(0);
    metricSampleAggregator.addSample(sampleWithDifferentLeader);
    metricSampleAggregator.addSample(incompletePartitionMetricSample);
    // Check the snapshot value and make sure the metric samples above are excluded.
    Map<PartitionEntity, ValuesAndExtrapolations> snapshotsForPartition = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress()).valuesAndExtrapolations();
    ValuesAndExtrapolations snapshots = snapshotsForPartition.get(PE);
    for (Resource resource : Resource.values()) {
        int metricId = KafkaCruiseControlMetricDef.resourceToMetricId(resource);
        double expectedValue = resource == Resource.DISK ? MIN_SAMPLES_PER_WINDOW - 1 : (MIN_SAMPLES_PER_WINDOW - 1) / 2.0;
        assertEquals("The utilization for " + resource + " should be " + expectedValue, expectedValue, snapshots.metricValues().valuesFor(metricId).get(NUM_WINDOWS - 1), 0);
    }
}

Example 24 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method setupScenario1.

/**
 * Two topics with 2 partitions each. No data missing.
 */
private TestContext setupScenario1() {
    TopicPartition t0p1 = new TopicPartition(TOPIC, 1);
    TopicPartition t1p0 = new TopicPartition("TOPIC1", 0);
    TopicPartition t1p1 = new TopicPartition("TOPIC1", 1);
    List<TopicPartition> allPartitions = Arrays.asList(TP, t0p1, t1p0, t1p1);
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(allPartitions);
    KafkaMetricSampleAggregator aggregator = new KafkaMetricSampleAggregator(config, metadata);
    for (TopicPartition tp : allPartitions) {
        populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, aggregator, tp);
    }
    return new TestContext(metadata, aggregator);
}

Example 25 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method testTooManyFlaws.

@Test
public void testTooManyFlaws() throws NotEnoughValidWindowsException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
    // Only give two samples to the aggregator.
    CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 2, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator, PE, 3, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
    MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress());
    assertTrue(result.valuesAndExtrapolations().isEmpty());
}