Examples with BrokerCapacityInfo com.linkedin.kafka.cruisecontrol.config.BrokerCapacityInfo used on opensource projects

Example 1 with BrokerCapacityInfo

use of com.linkedin.kafka.cruisecontrol.config.BrokerCapacityInfo in project cruise-control by linkedin.

the class LoadMonitor method populateClusterCapacity.

private void populateClusterCapacity(boolean populateReplicaPlacementInfo, boolean allowCapacityEstimation, ClusterModel clusterModel, Cluster cluster) throws TimeoutException, BrokerCapacityResolutionException {
    // Create the racks and brokers.
    // If broker capacity is allowed to estimate broker capacity, shuffle nodes before getting their capacity from the
    // capacity resolver. This is good for the capacity resolver to estimate the capacity of the nodes, for which the
    // capacity retrieval has failed.
    // The use case for this estimation is that if the capacity of one of the nodes is not available (e.g. due to some
    // 3rd party service issue), the capacity resolver may want to use the capacity of a peer node as the capacity for
    // that node.
    // To this end, Cruise Control handles the case that the first node is problematic so the capacity resolver does
    // not have the chance to get the capacity for the other nodes.
    // Shuffling the node order helps, as the problematic node is unlikely to always be the first node in the list.
    List<Node> shuffledNodes = allowCapacityEstimation ? new ArrayList<>(cluster.nodes()) : cluster.nodes();
    if (allowCapacityEstimation) {
        Collections.shuffle(shuffledNodes);
    }
    for (Node node : shuffledNodes) {
        // If the rack is not specified, we use the host info as rack info.
        String rack = getRackHandleNull(node);
        clusterModel.createRack(rack);
        BrokerCapacityInfo brokerCapacity;
        try {
            brokerCapacity = _brokerCapacityConfigResolver.capacityForBroker(rack, node.host(), node.id(), BROKER_CAPACITY_FETCH_TIMEOUT_MS, allowCapacityEstimation);
            LOG.debug("Capacity of broker {}: {}, (LogDir: {}, Cores: {}).", node.id(), brokerCapacity.capacity(), brokerCapacity.diskCapacityByLogDir(), brokerCapacity.numCpuCores());
            if (populateReplicaPlacementInfo && brokerCapacity.diskCapacityByLogDir() == null) {
                throw new IllegalStateException(String.format("Missing disk capacity information for logDirs on broker %d. " + "Are you trying to use a JBOD feature on a non-JBOD Kafka deployment?", node.id()));
            }
        } catch (TimeoutException | BrokerCapacityResolutionException e) {
            String errorMessage = String.format("Unable to retrieve capacity for broker %d. This may be caused by churn in " + "the cluster, please retry.", node.id());
            LOG.warn(errorMessage, e);
            throw e;
        }
        clusterModel.createBroker(rack, node.host(), node.id(), brokerCapacity, populateReplicaPlacementInfo);
    }
}

Example 2 with BrokerCapacityInfo

use of com.linkedin.kafka.cruisecontrol.config.BrokerCapacityInfo in project cruise-control by linkedin.

the class SortedReplicasTest method generateBroker.

private static Broker generateBroker(int numReplicas) {
    Rack rack = new Rack("rack");
    Host host = new Host("host", rack);
    Broker broker = new Broker(host, 0, new BrokerCapacityInfo(TestConstants.BROKER_CAPACITY), false);
    for (int i = 0; i < numReplicas; i++) {
        Replica r = new Replica(new TopicPartition(TOPIC0, i), broker, i % 3 == 0);
        broker.addReplica(r);
    }
    return broker;
}

Example 3 with BrokerCapacityInfo

use of com.linkedin.kafka.cruisecontrol.config.BrokerCapacityInfo in project cruise-control by linkedin.

the class CruiseControlMetricsProcessorTest method testWithCpuCapacityEstimation.

@Test
public void testWithCpuCapacityEstimation() throws TimeoutException, BrokerCapacityResolutionException {
    Set<CruiseControlMetric> metrics = getCruiseControlMetrics();
    // All estimated.
    BrokerCapacityConfigResolver brokerCapacityConfigResolverAllEstimated = EasyMock.mock(BrokerCapacityConfigResolver.class);
    EasyMock.expect(brokerCapacityConfigResolverAllEstimated.capacityForBroker(EasyMock.anyString(), EasyMock.anyString(), EasyMock.anyInt(), EasyMock.anyLong(), EasyMock.eq(false))).andThrow(new BrokerCapacityResolutionException("Unable to resolve capacity.")).anyTimes();
    EasyMock.replay(brokerCapacityConfigResolverAllEstimated);
    CruiseControlMetricsProcessor processor = new CruiseControlMetricsProcessor(brokerCapacityConfigResolverAllEstimated, false);
    for (CruiseControlMetric cruiseControlMetric : metrics) {
        processor.addMetric(cruiseControlMetric);
    }
    Cluster cluster = getCluster();
    processor.process(cluster, TEST_PARTITIONS, MetricSampler.SamplingMode.ALL);
    for (Node node : cluster.nodes()) {
        assertNull(processor.cachedNumCoresByBroker().get(node.id()));
    }
    // Capacity resolver unable to retrieve broker capacity.
    BrokerCapacityConfigResolver brokerCapacityConfigResolverTimeout = EasyMock.mock(BrokerCapacityConfigResolver.class);
    EasyMock.expect(brokerCapacityConfigResolverTimeout.capacityForBroker(EasyMock.anyString(), EasyMock.anyString(), EasyMock.anyInt(), EasyMock.anyLong(), EasyMock.anyBoolean())).andThrow(new TimeoutException("Unable to resolve capacity.")).anyTimes();
    EasyMock.replay(brokerCapacityConfigResolverTimeout);
    processor = new CruiseControlMetricsProcessor(brokerCapacityConfigResolverTimeout, false);
    for (CruiseControlMetric cruiseControlMetric : metrics) {
        processor.addMetric(cruiseControlMetric);
    }
    cluster = getCluster();
    processor.process(cluster, TEST_PARTITIONS, MetricSampler.SamplingMode.ALL);
    for (Node node : cluster.nodes()) {
        assertNull(processor.cachedNumCoresByBroker().get(node.id()));
    }
    // Some estimated.
    BrokerCapacityConfigResolver brokerCapacityConfigResolverSomeEstimated = EasyMock.mock(BrokerCapacityConfigResolver.class);
    EasyMock.expect(brokerCapacityConfigResolverSomeEstimated.capacityForBroker(EasyMock.anyString(), EasyMock.anyString(), EasyMock.eq(BROKER_ID_1), EasyMock.anyLong(), EasyMock.anyBoolean())).andThrow(new TimeoutException("Unable to resolve capacity.")).anyTimes();
    EasyMock.expect(brokerCapacityConfigResolverSomeEstimated.capacityForBroker(EasyMock.anyString(), EasyMock.anyString(), EasyMock.eq(BROKER_ID_0), EasyMock.anyLong(), EasyMock.anyBoolean())).andReturn(new BrokerCapacityInfo(EMPTY_BROKER_CAPACITY, Collections.emptyMap(), MOCK_NUM_CPU_CORES)).anyTimes();
    EasyMock.replay(brokerCapacityConfigResolverSomeEstimated);
    processor = new CruiseControlMetricsProcessor(brokerCapacityConfigResolverSomeEstimated, false);
    for (CruiseControlMetric metric : metrics) {
        processor.addMetric(metric);
    }
    processor.process(cluster, TEST_PARTITIONS, MetricSampler.SamplingMode.ALL);
    assertEquals(MOCK_NUM_CPU_CORES, (short) processor.cachedNumCoresByBroker().get(BROKER_ID_0));
    assertNull(processor.cachedNumCoresByBroker().get(BROKER_ID_1));
    EasyMock.verify(brokerCapacityConfigResolverTimeout, brokerCapacityConfigResolverSomeEstimated, brokerCapacityConfigResolverAllEstimated);
}

Example 4 with BrokerCapacityInfo

use of com.linkedin.kafka.cruisecontrol.config.BrokerCapacityInfo in project cruise-control by linkedin.

the class PreferredLeaderElectionGoalTest method createClusterModel.

private ClusterModelAndInfo createClusterModel(boolean skipClusterInfoGeneration, boolean populateDiskInfo) {
    ClusterModel clusterModel = new ClusterModel(new ModelGeneration(0, 0), 1.0);
    for (int i = 0; i < NUM_RACKS; i++) {
        clusterModel.createRack("r" + i);
    }
    BrokerCapacityInfo commonBrokerCapacityInfo = populateDiskInfo ? new BrokerCapacityInfo(TestConstants.BROKER_CAPACITY, null, TestConstants.DISK_CAPACITY) : new BrokerCapacityInfo(TestConstants.BROKER_CAPACITY);
    int i = 0;
    for (; i < 2; i++) {
        clusterModel.createBroker("r0", "h" + i, i, commonBrokerCapacityInfo, populateDiskInfo);
    }
    for (int j = 1; j < NUM_RACKS; j++, i++) {
        clusterModel.createBroker("r" + j, "h" + i, i, commonBrokerCapacityInfo, populateDiskInfo);
    }
    createReplicaAndSetLoad(clusterModel, "r0", 0, logdir(populateDiskInfo, 0, 0), T0P0, 0, true);
    createReplicaAndSetLoad(clusterModel, "r0", 1, logdir(populateDiskInfo, 0, 1), T0P1, 0, true);
    createReplicaAndSetLoad(clusterModel, "r1", 2, logdir(populateDiskInfo, 0, 2), T0P2, 0, true);
    createReplicaAndSetLoad(clusterModel, "r2", 3, logdir(populateDiskInfo, 0, 3), T1P0, 0, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, logdir(populateDiskInfo, 0, 4), T1P1, 0, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, logdir(populateDiskInfo, 0, 0), T1P2, 0, false);
    createReplicaAndSetLoad(clusterModel, "r0", 1, logdir(populateDiskInfo, 0, 1), T2P0, 0, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, logdir(populateDiskInfo, 0, 2), T2P1, 0, false);
    createReplicaAndSetLoad(clusterModel, "r2", 3, logdir(populateDiskInfo, 0, 3), T2P2, 0, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, logdir(populateDiskInfo, 1, 4), T0P0, 1, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, logdir(populateDiskInfo, 1, 2), T0P1, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, logdir(populateDiskInfo, 1, 0), T0P2, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 1, logdir(populateDiskInfo, 1, 1), T1P0, 1, true);
    createReplicaAndSetLoad(clusterModel, "r2", 3, logdir(populateDiskInfo, 1, 3), T1P1, 1, true);
    createReplicaAndSetLoad(clusterModel, "r3", 4, logdir(populateDiskInfo, 1, 4), T1P2, 1, true);
    createReplicaAndSetLoad(clusterModel, "r1", 2, logdir(populateDiskInfo, 1, 2), T2P0, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, logdir(populateDiskInfo, 1, 0), T2P1, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 1, logdir(populateDiskInfo, 1, 1), T2P2, 1, false);
    createReplicaAndSetLoad(clusterModel, "r2", 3, logdir(populateDiskInfo, 2, 3), T0P0, 2, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, logdir(populateDiskInfo, 2, 4), T0P1, 2, false);
    createReplicaAndSetLoad(clusterModel, "r2", 3, logdir(populateDiskInfo, 2, 3), T0P2, 2, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, logdir(populateDiskInfo, 2, 2), T1P0, 2, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, logdir(populateDiskInfo, 2, 0), T1P1, 2, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, logdir(populateDiskInfo, 2, 2), T1P2, 2, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, logdir(populateDiskInfo, 2, 4), T2P0, 2, true);
    createReplicaAndSetLoad(clusterModel, "r2", 3, logdir(populateDiskInfo, 2, 3), T2P1, 2, true);
    createReplicaAndSetLoad(clusterModel, "r3", 4, logdir(populateDiskInfo, 2, 4), T2P2, 2, true);
    Cluster cluster = null;
    if (!skipClusterInfoGeneration) {
        Node[] nodes = new Node[NUM_RACKS + 1];
        for (i = 0; i < NUM_RACKS + 1; i++) {
            nodes[i] = new Node(i, "h" + i, 100);
        }
        List<PartitionInfo> partitions = new ArrayList<>(9);
        // Make topic1 and topic2's partitions under replicated.
        partitions.add(new PartitionInfo(T0P0.topic(), T0P0.partition(), nodes[0], new Node[] { nodes[0], nodes[4], nodes[3] }, new Node[] { nodes[0], nodes[4], nodes[3] }));
        partitions.add(new PartitionInfo(T0P1.topic(), T0P1.partition(), nodes[1], new Node[] { nodes[1], nodes[2], nodes[4] }, new Node[] { nodes[1], nodes[2], nodes[4] }));
        partitions.add(new PartitionInfo(T0P2.topic(), T0P2.partition(), nodes[2], new Node[] { nodes[2], nodes[0], nodes[3] }, new Node[] { nodes[2], nodes[0], nodes[3] }));
        partitions.add(new PartitionInfo(T1P0.topic(), T1P0.partition(), nodes[1], new Node[] { nodes[1], nodes[3] }, new Node[] { nodes[1], nodes[3], nodes[2] }));
        partitions.add(new PartitionInfo(T1P1.topic(), T1P1.partition(), nodes[3], new Node[] { nodes[3], nodes[4] }, new Node[] { nodes[3], nodes[4], nodes[0] }));
        partitions.add(new PartitionInfo(T1P2.topic(), T1P2.partition(), nodes[4], new Node[] { nodes[4], nodes[2] }, new Node[] { nodes[4], nodes[2], nodes[0] }));
        partitions.add(new PartitionInfo(T2P0.topic(), T2P0.partition(), nodes[4], new Node[] { nodes[4], nodes[2] }, new Node[] { nodes[4], nodes[2], nodes[1] }));
        partitions.add(new PartitionInfo(T2P1.topic(), T2P1.partition(), nodes[3], new Node[] { nodes[3], nodes[0] }, new Node[] { nodes[3], nodes[0], nodes[2] }));
        partitions.add(new PartitionInfo(T2P2.topic(), T2P2.partition(), nodes[4], new Node[] { nodes[4], nodes[1] }, new Node[] { nodes[4], nodes[1], nodes[3] }));
        cluster = new Cluster("id", Arrays.asList(nodes), partitions, Collections.emptySet(), Collections.emptySet());
    }
    return new ClusterModelAndInfo(clusterModel, cluster);
}

Example 5 with BrokerCapacityInfo

use of com.linkedin.kafka.cruisecontrol.config.BrokerCapacityInfo in project cruise-control by linkedin.

the class DeterministicCluster method getHomogeneousCluster.

/**
 * Creates a deterministic cluster with the given number of racks and the broker distribution.
 *
 * @param rackByBroker Racks by broker ids.
 * @param brokerCapacity Alive broker capacity.
 * @param diskCapacityByLogDir Disk capacity for each broker.
 * @return Cluster with the specified number of racks and broker distribution.
 */
public static ClusterModel getHomogeneousCluster(Map<Integer, Integer> rackByBroker, Map<Resource, Double> brokerCapacity, Map<String, Double> diskCapacityByLogDir) {
    // Sanity checks.
    if (rackByBroker.size() <= 0 || brokerCapacity.get(Resource.CPU) < 0 || brokerCapacity.get(Resource.DISK) < 0 || brokerCapacity.get(Resource.NW_IN) < 0 || brokerCapacity.get(Resource.NW_OUT) < 0) {
        throw new IllegalArgumentException("Deterministic cluster generation failed due to bad input.");
    }
    // Create cluster.
    ClusterModel cluster = new ClusterModel(new ModelGeneration(0, 0L), 1.0);
    // Create racks and add them to cluster.
    Set<Integer> racks = new HashSet<>();
    for (int rackId : rackByBroker.values()) {
        if (racks.add(rackId)) {
            cluster.createRack(Integer.toString(rackId));
        }
    }
    BrokerCapacityInfo commonBrokerCapacityInfo = new BrokerCapacityInfo(brokerCapacity, diskCapacityByLogDir);
    // Create brokers and assign a broker to each rack.
    rackByBroker.forEach((key, value) -> cluster.createBroker(value.toString(), Integer.toString(key), key, commonBrokerCapacityInfo, diskCapacityByLogDir != null));
    return cluster;
}