Examples with KafkaCruiseControlConfig com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig used on opensource projects

Example 26 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class LoadMonitorTaskRunnerTest method testSamplingError.

@Test
public void testSamplingError() {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = new Metadata(10, 10, false);
    MetadataClient metadataClient = new MetadataClient(config, metadata, -1L, TIME);
    MockMetricSampleAggregator mockMetricSampleAggregator = new MockMetricSampleAggregator(config, metadata);
    List<MetricSampler> samplers = new ArrayList<>();
    MetricRegistry dropwizardMetricRegistry = new MetricRegistry();
    for (int i = 0; i < NUM_METRIC_FETCHERS; i++) {
        samplers.add(new MockSampler(i));
    }
    MetricFetcherManager fetcherManager = new MetricFetcherManager(config, mockMetricSampleAggregator, metadataClient, METRIC_DEF, TIME, dropwizardMetricRegistry, samplers);
    LoadMonitorTaskRunner loadMonitorTaskRunner = new LoadMonitorTaskRunner(config, fetcherManager, mockMetricSampleAggregator, metadataClient, TIME);
    while (metadata.fetch().topics().size() < 100) {
        metadataClient.refreshMetadata();
    }
    loadMonitorTaskRunner.start(true);
    int numSamples = 0;
    long startMs = System.currentTimeMillis();
    BlockingQueue<PartitionMetricSample> sampleQueue = mockMetricSampleAggregator.metricSampleQueue();
    while (numSamples < (NUM_PARTITIONS * NUM_TOPICS) * 10 && System.currentTimeMillis() < startMs + 10000) {
        PartitionMetricSample sample = sampleQueue.poll();
        if (sample != null) {
            numSamples++;
        }
    }
    // We should have NUM_METRIC_FETCHER rounds of sampling. The first round only has one metric fetcher returns
    // samples, two fetchers return samples for the second round, three for the third and four for the forth round.
    // So the first round only has 1/4 of the total samples, then 2/4, 3/4 and all the samples.
    int expectedNumSamples = 0;
    for (int i = 0; i < NUM_METRIC_FETCHERS; i++) {
        expectedNumSamples += (NUM_TOPICS * NUM_PARTITIONS) * (i + 1) / NUM_METRIC_FETCHERS;
    }
    assertEquals("Only see " + numSamples + " samples. Expecting " + expectedNumSamples + " samples", expectedNumSamples, numSamples);
    fetcherManager.shutdown();
}

Example 27 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class ExecutorTest method executeAndVerifyProposals.

private void executeAndVerifyProposals(ZkUtils zkUtils, Collection<ExecutionProposal> proposalsToExecute, Collection<ExecutionProposal> proposalsToCheck) {
    KafkaCruiseControlConfig configs = new KafkaCruiseControlConfig(getExecutorProperties());
    Executor executor = new Executor(configs, new SystemTime(), new MetricRegistry());
    executor.executeProposals(proposalsToExecute, Collections.emptySet(), EasyMock.mock(LoadMonitor.class));
    Map<TopicPartition, Integer> replicationFactors = new HashMap<>();
    for (ExecutionProposal proposal : proposalsToCheck) {
        int replicationFactor = zkUtils.getReplicasForPartition(proposal.topic(), proposal.partitionId()).size();
        replicationFactors.put(new TopicPartition(proposal.topic(), proposal.partitionId()), replicationFactor);
    }
    long now = System.currentTimeMillis();
    while (executor.state().state() != ExecutorState.State.NO_TASK_IN_PROGRESS && System.currentTimeMillis() < now + 30000) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
    if (executor.state().state() != ExecutorState.State.NO_TASK_IN_PROGRESS) {
        fail("The execution did not finish in 30 seconds.");
    }
    for (ExecutionProposal proposal : proposalsToCheck) {
        TopicPartition tp = new TopicPartition(proposal.topic(), proposal.partitionId());
        int expectedReplicationFactor = replicationFactors.get(tp);
        assertEquals("Replication factor for partition " + tp + " should be " + expectedReplicationFactor, expectedReplicationFactor, zkUtils.getReplicasForPartition(tp.topic(), tp.partition()).size());
        if (proposal.hasReplicaAction()) {
            for (int brokerId : proposal.newReplicas()) {
                assertTrue("The partition should have moved for " + tp, zkUtils.getReplicasForPartition(tp.topic(), tp.partition()).contains(brokerId));
            }
        }
        assertEquals("The leader should have moved for " + tp, proposal.newLeader(), zkUtils.getLeaderForPartition(tp.topic(), tp.partition()).get());
    }
}

Example 28 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class LoadMonitorTest method prepareContext.

private TestContext prepareContext() {
    // Create mock metadata client.
    Metadata metadata = getMetadata(Arrays.asList(T0P0, T0P1, T1P0, T1P1));
    MetadataClient mockMetadataClient = EasyMock.mock(MetadataClient.class);
    EasyMock.expect(mockMetadataClient.cluster()).andReturn(metadata.fetch()).anyTimes();
    EasyMock.expect(mockMetadataClient.clusterAndGeneration()).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
    EasyMock.expect(mockMetadataClient.metadata()).andReturn(metadata).anyTimes();
    EasyMock.expect(mockMetadataClient.refreshMetadata()).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
    EasyMock.expect(mockMetadataClient.refreshMetadata(anyLong())).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
    EasyMock.replay(mockMetadataClient);
    // create load monitor.
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    props.put(KafkaCruiseControlConfig.NUM_METRICS_WINDOWS_CONFIG, Integer.toString(NUM_WINDOWS));
    props.put(KafkaCruiseControlConfig.MIN_SAMPLES_PER_METRICS_WINDOW_CONFIG, Integer.toString(MIN_SAMPLES_PER_WINDOW));
    props.put(KafkaCruiseControlConfig.METRICS_WINDOW_MS_CONFIG, Long.toString(WINDOW_MS));
    props.put("cleanup.policy", DEFAULT_CLEANUP_POLICY);
    props.put(KafkaCruiseControlConfig.SAMPLE_STORE_CLASS_CONFIG, NoopSampleStore.class.getName());
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(props);
    LoadMonitor loadMonitor = new LoadMonitor(config, mockMetadataClient, _time, new MetricRegistry(), METRIC_DEF);
    KafkaMetricSampleAggregator aggregator = loadMonitor.aggregator();
    ModelParameters.init(config);
    loadMonitor.startUp();
    while (loadMonitor.state(new OperationProgress()).state() != LoadMonitorTaskRunner.LoadMonitorTaskRunnerState.RUNNING) {
        try {
            Thread.sleep(1);
        } catch (InterruptedException e) {
        // let it go.
        }
    }
    return new TestContext(loadMonitor, aggregator, config, metadata);
}

Example 29 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class KafkaAssignerDiskUsageDistributionGoal method configure.

@Override
public void configure(Map<String, ?> configs) {
    _balancingConstraint = new BalancingConstraint(new KafkaCruiseControlConfig(configs, false));
    String minMonitoredPartitionPercentageString = (String) configs.get(KafkaCruiseControlConfig.MIN_VALID_PARTITION_RATIO_CONFIG);
    if (minMonitoredPartitionPercentageString != null && !minMonitoredPartitionPercentageString.isEmpty()) {
        _minMonitoredPartitionPercentage = Double.parseDouble(minMonitoredPartitionPercentageString);
    }
}

Example 30 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class KafkaCruiseControlMain method main.

public static void main(String[] args) throws Exception {
    // Check the command line arguments.
    if (args.length == 0) {
        printErrorMessageAndDie();
    }
    // Load all the properties.
    Properties props = new Properties();
    try (InputStream propStream = new FileInputStream(args[0])) {
        props.load(propStream);
    }
    int port = 9090;
    if (args.length > 1) {
        try {
            port = Integer.parseInt(args[1]);
        } catch (Exception e) {
            printErrorMessageAndDie();
        }
    }
    MetricRegistry dropwizardMetricsRegistry = new MetricRegistry();
    JmxReporter jmxReporter = JmxReporter.forRegistry(dropwizardMetricsRegistry).inDomain(METRIC_DOMAIN).build();
    jmxReporter.start();
    AsyncKafkaCruiseControl kafkaCruiseControl = new AsyncKafkaCruiseControl(new KafkaCruiseControlConfig(props), dropwizardMetricsRegistry);
    Server server = new Server(port);
    ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
    context.setContextPath("/");
    server.setHandler(context);
    // Placeholder for any static content
    DefaultServlet defaultServlet = new DefaultServlet();
    ServletHolder holderWebapp = new ServletHolder("default", defaultServlet);
    holderWebapp.setInitParameter("org.eclipse.jetty.servlet.Default.dirAllowed", "false");
    holderWebapp.setInitParameter("resourceBase", "./cruise-control-ui/dist/");
    context.addServlet(holderWebapp, "/*");
    // Kafka Cruise Control servlet data
    KafkaCruiseControlServlet kafkaCruiseControlServlet = new KafkaCruiseControlServlet(kafkaCruiseControl, 10000L, 60000L, dropwizardMetricsRegistry);
    ServletHolder servletHolder = new ServletHolder(kafkaCruiseControlServlet);
    context.addServlet(servletHolder, "/kafkacruisecontrol/*");
    Runtime.getRuntime().addShutdownHook(new Thread() {

        @Override
        public void run() {
            kafkaCruiseControl.shutdown();
            jmxReporter.close();
        }
    });
    kafkaCruiseControl.startUp();
    server.start();
    System.out.println("Application directory: " + System.getProperty("user.dir"));
    System.out.println("Kafka Cruise Control started.");
}