Examples with MetadataClient com.linkedin.kafka.cruisecontrol.common.MetadataClient used on opensource projects

Example 1 with MetadataClient

use of com.linkedin.kafka.cruisecontrol.common.MetadataClient in project cruise-control by linkedin.

the class LoadMonitorTaskRunnerTest method testSimpleFetch.

@Test
public void testSimpleFetch() throws InterruptedException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = new Metadata(10, 10, false);
    MetadataClient metadataClient = new MetadataClient(config, metadata, -1L, TIME);
    MockMetricSampleAggregator mockMetricSampleAggregator = new MockMetricSampleAggregator(config, metadata);
    List<MetricSampler> samplers = new ArrayList<>();
    MetricRegistry dropwizardMetricRegistry = new MetricRegistry();
    for (int i = 0; i < NUM_METRIC_FETCHERS; i++) {
        samplers.add(new MockSampler(0));
    }
    MetricFetcherManager fetcherManager = new MetricFetcherManager(config, mockMetricSampleAggregator, metadataClient, METRIC_DEF, TIME, dropwizardMetricRegistry, samplers);
    LoadMonitorTaskRunner loadMonitorTaskRunner = new LoadMonitorTaskRunner(config, fetcherManager, mockMetricSampleAggregator, metadataClient, TIME);
    while (metadata.fetch().topics().size() < NUM_TOPICS) {
        Thread.sleep(10);
        metadataClient.refreshMetadata();
    }
    loadMonitorTaskRunner.start(true);
    Set<TopicPartition> partitionsToSample = new HashSet<>();
    for (int i = 0; i < NUM_TOPICS; i++) {
        for (int j = 0; j < NUM_PARTITIONS; j++) {
            partitionsToSample.add(new TopicPartition("topic-" + i, j));
        }
    }
    long startMs = System.currentTimeMillis();
    BlockingQueue<PartitionMetricSample> sampleQueue = mockMetricSampleAggregator.metricSampleQueue();
    while (!partitionsToSample.isEmpty() && System.currentTimeMillis() < startMs + 10000) {
        PartitionMetricSample sample = sampleQueue.poll();
        if (sample != null) {
            assertTrue("The topic partition should have been sampled and sampled only once.", partitionsToSample.contains(sample.entity().tp()));
            partitionsToSample.remove(sample.entity().tp());
        }
    }
    assertTrue("Did not see sample for partitions " + Arrays.toString(partitionsToSample.toArray()), partitionsToSample.isEmpty());
    fetcherManager.shutdown();
    assertTrue(sampleQueue.isEmpty());
}

Example 2 with MetadataClient

use of com.linkedin.kafka.cruisecontrol.common.MetadataClient in project cruise-control by linkedin.

the class LoadMonitorTaskRunnerTest method testSamplingError.

@Test
public void testSamplingError() {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = new Metadata(10, 10, false);
    MetadataClient metadataClient = new MetadataClient(config, metadata, -1L, TIME);
    MockMetricSampleAggregator mockMetricSampleAggregator = new MockMetricSampleAggregator(config, metadata);
    List<MetricSampler> samplers = new ArrayList<>();
    MetricRegistry dropwizardMetricRegistry = new MetricRegistry();
    for (int i = 0; i < NUM_METRIC_FETCHERS; i++) {
        samplers.add(new MockSampler(i));
    }
    MetricFetcherManager fetcherManager = new MetricFetcherManager(config, mockMetricSampleAggregator, metadataClient, METRIC_DEF, TIME, dropwizardMetricRegistry, samplers);
    LoadMonitorTaskRunner loadMonitorTaskRunner = new LoadMonitorTaskRunner(config, fetcherManager, mockMetricSampleAggregator, metadataClient, TIME);
    while (metadata.fetch().topics().size() < 100) {
        metadataClient.refreshMetadata();
    }
    loadMonitorTaskRunner.start(true);
    int numSamples = 0;
    long startMs = System.currentTimeMillis();
    BlockingQueue<PartitionMetricSample> sampleQueue = mockMetricSampleAggregator.metricSampleQueue();
    while (numSamples < (NUM_PARTITIONS * NUM_TOPICS) * 10 && System.currentTimeMillis() < startMs + 10000) {
        PartitionMetricSample sample = sampleQueue.poll();
        if (sample != null) {
            numSamples++;
        }
    }
    // We should have NUM_METRIC_FETCHER rounds of sampling. The first round only has one metric fetcher returns
    // samples, two fetchers return samples for the second round, three for the third and four for the forth round.
    // So the first round only has 1/4 of the total samples, then 2/4, 3/4 and all the samples.
    int expectedNumSamples = 0;
    for (int i = 0; i < NUM_METRIC_FETCHERS; i++) {
        expectedNumSamples += (NUM_TOPICS * NUM_PARTITIONS) * (i + 1) / NUM_METRIC_FETCHERS;
    }
    assertEquals("Only see " + numSamples + " samples. Expecting " + expectedNumSamples + " samples", expectedNumSamples, numSamples);
    fetcherManager.shutdown();
}

Example 3 with MetadataClient

use of com.linkedin.kafka.cruisecontrol.common.MetadataClient in project cruise-control by linkedin.

the class LoadMonitorTest method prepareContext.

private TestContext prepareContext() {
    // Create mock metadata client.
    Metadata metadata = getMetadata(Arrays.asList(T0P0, T0P1, T1P0, T1P1));
    MetadataClient mockMetadataClient = EasyMock.mock(MetadataClient.class);
    EasyMock.expect(mockMetadataClient.cluster()).andReturn(metadata.fetch()).anyTimes();
    EasyMock.expect(mockMetadataClient.clusterAndGeneration()).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
    EasyMock.expect(mockMetadataClient.metadata()).andReturn(metadata).anyTimes();
    EasyMock.expect(mockMetadataClient.refreshMetadata()).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
    EasyMock.expect(mockMetadataClient.refreshMetadata(anyLong())).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
    EasyMock.replay(mockMetadataClient);
    // create load monitor.
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    props.put(KafkaCruiseControlConfig.NUM_METRICS_WINDOWS_CONFIG, Integer.toString(NUM_WINDOWS));
    props.put(KafkaCruiseControlConfig.MIN_SAMPLES_PER_METRICS_WINDOW_CONFIG, Integer.toString(MIN_SAMPLES_PER_WINDOW));
    props.put(KafkaCruiseControlConfig.METRICS_WINDOW_MS_CONFIG, Long.toString(WINDOW_MS));
    props.put("cleanup.policy", DEFAULT_CLEANUP_POLICY);
    props.put(KafkaCruiseControlConfig.SAMPLE_STORE_CLASS_CONFIG, NoopSampleStore.class.getName());
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(props);
    LoadMonitor loadMonitor = new LoadMonitor(config, mockMetadataClient, _time, new MetricRegistry(), METRIC_DEF);
    KafkaMetricSampleAggregator aggregator = loadMonitor.aggregator();
    ModelParameters.init(config);
    loadMonitor.startUp();
    while (loadMonitor.state(new OperationProgress()).state() != LoadMonitorTaskRunner.LoadMonitorTaskRunnerState.RUNNING) {
        try {
            Thread.sleep(1);
        } catch (InterruptedException e) {
        // let it go.
        }
    }
    return new TestContext(loadMonitor, aggregator, config, metadata);
}