Examples with MetricFetcherManager com.linkedin.kafka.cruisecontrol.monitor.sampling.MetricFetcherManager used on opensource projects

Example 1 with MetricFetcherManager

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.MetricFetcherManager in project cruise-control by linkedin.

the class LoadMonitorTaskRunnerTest method testSimpleFetch.

@Test
public void testSimpleFetch() throws InterruptedException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = new Metadata(10, 10, false);
    MetadataClient metadataClient = new MetadataClient(config, metadata, -1L, TIME);
    MockMetricSampleAggregator mockMetricSampleAggregator = new MockMetricSampleAggregator(config, metadata);
    List<MetricSampler> samplers = new ArrayList<>();
    MetricRegistry dropwizardMetricRegistry = new MetricRegistry();
    for (int i = 0; i < NUM_METRIC_FETCHERS; i++) {
        samplers.add(new MockSampler(0));
    }
    MetricFetcherManager fetcherManager = new MetricFetcherManager(config, mockMetricSampleAggregator, metadataClient, METRIC_DEF, TIME, dropwizardMetricRegistry, samplers);
    LoadMonitorTaskRunner loadMonitorTaskRunner = new LoadMonitorTaskRunner(config, fetcherManager, mockMetricSampleAggregator, metadataClient, TIME);
    while (metadata.fetch().topics().size() < NUM_TOPICS) {
        Thread.sleep(10);
        metadataClient.refreshMetadata();
    }
    loadMonitorTaskRunner.start(true);
    Set<TopicPartition> partitionsToSample = new HashSet<>();
    for (int i = 0; i < NUM_TOPICS; i++) {
        for (int j = 0; j < NUM_PARTITIONS; j++) {
            partitionsToSample.add(new TopicPartition("topic-" + i, j));
        }
    }
    long startMs = System.currentTimeMillis();
    BlockingQueue<PartitionMetricSample> sampleQueue = mockMetricSampleAggregator.metricSampleQueue();
    while (!partitionsToSample.isEmpty() && System.currentTimeMillis() < startMs + 10000) {
        PartitionMetricSample sample = sampleQueue.poll();
        if (sample != null) {
            assertTrue("The topic partition should have been sampled and sampled only once.", partitionsToSample.contains(sample.entity().tp()));
            partitionsToSample.remove(sample.entity().tp());
        }
    }
    assertTrue("Did not see sample for partitions " + Arrays.toString(partitionsToSample.toArray()), partitionsToSample.isEmpty());
    fetcherManager.shutdown();
    assertTrue(sampleQueue.isEmpty());
}

Example 2 with MetricFetcherManager

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.MetricFetcherManager in project cruise-control by linkedin.

the class LoadMonitorTaskRunnerTest method testSamplingError.

@Test
public void testSamplingError() {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = new Metadata(10, 10, false);
    MetadataClient metadataClient = new MetadataClient(config, metadata, -1L, TIME);
    MockMetricSampleAggregator mockMetricSampleAggregator = new MockMetricSampleAggregator(config, metadata);
    List<MetricSampler> samplers = new ArrayList<>();
    MetricRegistry dropwizardMetricRegistry = new MetricRegistry();
    for (int i = 0; i < NUM_METRIC_FETCHERS; i++) {
        samplers.add(new MockSampler(i));
    }
    MetricFetcherManager fetcherManager = new MetricFetcherManager(config, mockMetricSampleAggregator, metadataClient, METRIC_DEF, TIME, dropwizardMetricRegistry, samplers);
    LoadMonitorTaskRunner loadMonitorTaskRunner = new LoadMonitorTaskRunner(config, fetcherManager, mockMetricSampleAggregator, metadataClient, TIME);
    while (metadata.fetch().topics().size() < 100) {
        metadataClient.refreshMetadata();
    }
    loadMonitorTaskRunner.start(true);
    int numSamples = 0;
    long startMs = System.currentTimeMillis();
    BlockingQueue<PartitionMetricSample> sampleQueue = mockMetricSampleAggregator.metricSampleQueue();
    while (numSamples < (NUM_PARTITIONS * NUM_TOPICS) * 10 && System.currentTimeMillis() < startMs + 10000) {
        PartitionMetricSample sample = sampleQueue.poll();
        if (sample != null) {
            numSamples++;
        }
    }
    // We should have NUM_METRIC_FETCHER rounds of sampling. The first round only has one metric fetcher returns
    // samples, two fetchers return samples for the second round, three for the third and four for the forth round.
    // So the first round only has 1/4 of the total samples, then 2/4, 3/4 and all the samples.
    int expectedNumSamples = 0;
    for (int i = 0; i < NUM_METRIC_FETCHERS; i++) {
        expectedNumSamples += (NUM_TOPICS * NUM_PARTITIONS) * (i + 1) / NUM_METRIC_FETCHERS;
    }
    assertEquals("Only see " + numSamples + " samples. Expecting " + expectedNumSamples + " samples", expectedNumSamples, numSamples);
    fetcherManager.shutdown();
}