Examples with ModelGeneration com.linkedin.kafka.cruisecontrol.monitor.ModelGeneration used on opensource projects

Example 1 with ModelGeneration

use of com.linkedin.kafka.cruisecontrol.monitor.ModelGeneration in project cruise-control by linkedin.

the class DeterministicCluster method getHomogeneousDeterministicCluster.

/**
 * Creates a deterministic cluster with the given number of racks and the broker distribution.
 *
 * @param numRacks                Number of racks in ToR architecture.
 * @param orderedRackIdsOfBrokers Specifies the rack id for each broker.
 * @param brokerCapacity          Healthy broker capacity.
 * @return Cluster with the specified number of racks and broker distribution.
 */
public static ClusterModel getHomogeneousDeterministicCluster(int numRacks, List<Integer> orderedRackIdsOfBrokers, Map<Resource, Double> brokerCapacity) {
    int numBrokers = orderedRackIdsOfBrokers.size();
    // Sanity checks.
    if (numRacks > numBrokers || numBrokers <= 0 || numRacks <= 0 || brokerCapacity.get(Resource.CPU) < 0 || brokerCapacity.get(Resource.DISK) < 0 || brokerCapacity.get(Resource.NW_IN) < 0 || brokerCapacity.get(Resource.NW_OUT) < 0) {
        throw new IllegalArgumentException("Deterministic cluster generation failed due to bad input.");
    }
    // Create cluster.
    ClusterModel cluster = new ClusterModel(new ModelGeneration(0, 0L), 1.0);
    // Create racks and add them to cluster.
    for (int i = 0; i < numRacks; i++) {
        cluster.createRack(Integer.toString(i));
    }
    // Create brokers and assign a broker to each rack.
    int brokerId = 0;
    for (Integer rackId : orderedRackIdsOfBrokers) {
        cluster.createBroker(rackId.toString(), Integer.toString(brokerId), brokerId, brokerCapacity);
        brokerId++;
    }
    return cluster;
}

Example 2 with ModelGeneration

use of com.linkedin.kafka.cruisecontrol.monitor.ModelGeneration in project cruise-control by linkedin.

the class PreferredLeaderElectionGoalTest method createClusterModel.

private ClusterModel createClusterModel() {
    final int numRacks = 4;
    ClusterModel clusterModel = new ClusterModel(new ModelGeneration(0, 0), 1.0);
    for (int i = 0; i < numRacks; i++) {
        clusterModel.createRack("r" + i);
    }
    int i = 0;
    for (; i < 2; i++) {
        clusterModel.createBroker("r0", "h" + i, i, TestConstants.BROKER_CAPACITY);
    }
    for (int j = 1; j < numRacks; j++, i++) {
        clusterModel.createBroker("r" + j, "h" + i, i, TestConstants.BROKER_CAPACITY);
    }
    createReplicaAndSetLoad(clusterModel, "r0", 0, T0P0, 0, true);
    createReplicaAndSetLoad(clusterModel, "r0", 1, T0P1, 0, true);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T0P2, 0, true);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T1P0, 0, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T1P1, 0, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, T1P2, 0, false);
    createReplicaAndSetLoad(clusterModel, "r0", 1, T2P0, 0, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T2P1, 0, false);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T2P2, 0, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T0P0, 1, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T0P1, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, T0P2, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 1, T1P0, 1, true);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T1P1, 1, true);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T1P2, 1, true);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T2P0, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, T2P1, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 1, T2P2, 1, false);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T0P0, 2, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T0P1, 2, false);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T0P2, 2, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T1P0, 2, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, T1P1, 2, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T1P2, 2, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T2P0, 2, true);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T2P1, 2, true);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T2P2, 2, true);
    return clusterModel;
}

Example 3 with ModelGeneration

use of com.linkedin.kafka.cruisecontrol.monitor.ModelGeneration in project cruise-control by linkedin.

the class RandomClusterTest method testNewBrokers.

/**
 * This test first creates a random cluster, balance it. Then add two new brokers, balance the cluster again.
 */
public void testNewBrokers() throws Exception {
    ClusterModel clusterModel = rebalance();
    ClusterModel clusterWithNewBroker = new ClusterModel(new ModelGeneration(0, 0L), 1.0);
    for (Broker b : clusterModel.brokers()) {
        clusterWithNewBroker.createRack(b.rack().id());
        Map<Resource, Double> brokerCapacity = new HashMap<>();
        for (Resource r : Resource.cachedValues()) {
            brokerCapacity.put(r, b.capacityFor(r));
        }
        clusterWithNewBroker.createBroker(b.rack().id(), Integer.toString(b.id()), b.id(), brokerCapacity);
    }
    for (Map.Entry<String, List<Partition>> entry : clusterModel.getPartitionsByTopic().entrySet()) {
        for (Partition p : entry.getValue()) {
            int index = 0;
            for (Replica r : p.replicas()) {
                clusterWithNewBroker.createReplica(r.broker().rack().id(), r.broker().id(), p.topicPartition(), index++, r.isLeader());
            }
        }
    }
    for (Broker b : clusterModel.brokers()) {
        for (Replica replica : b.replicas()) {
            AggregatedMetricValues aggregatedMetricValues = clusterModel.broker(b.id()).replica(replica.topicPartition()).load().loadByWindows();
            clusterWithNewBroker.setReplicaLoad(b.rack().id(), b.id(), replica.topicPartition(), aggregatedMetricValues, clusterModel.load().windows());
        }
    }
    for (int i = 1; i < 3; i++) {
        clusterWithNewBroker.createBroker(Integer.toString(i), Integer.toString(i + clusterModel.brokers().size() - 1), i + clusterModel.brokers().size() - 1, TestConstants.BROKER_CAPACITY);
        clusterWithNewBroker.setBrokerState(i + clusterModel.brokers().size() - 1, Broker.State.NEW);
    }
    assertTrue("Random Cluster Test failed to improve the existing state with new brokers.", OptimizationVerifier.executeGoalsFor(_balancingConstraint, clusterWithNewBroker, _goalNameByPriority, _verifications));
}

Example 4 with ModelGeneration

use of com.linkedin.kafka.cruisecontrol.monitor.ModelGeneration in project cruise-control by linkedin.

the class KafkaAssignerDiskUsageDistributionGoalTest method createClusterModel.

/**
 * The replica distribution is as below.
 *
 * L - Leader
 * F - Follower
 * S - Secondary Follower
 *
 *         r0             r1         r2          r3
 *        /  \             |          |           |
 *      b0    b1          b2         b3          b4
 *   T0P0(L)  T0P1(L)   T0P2(L)    T1P0(L)     T1P1(L)
 *   T1P2(L)  T2P0(L)   T2P1(L)    T2P2(L)     T0P0(F)
 *   T0P2(F)  T1P0(F)   T0P1(F)    T1P1(F)     T1P2(F)
 *   T2P1(F)  T2P2(F)   T2P0(F)    T0P0(S)     T0P1(S)
 *   T1P1(S)            T1P0(S)    T0P2(S)     T2P0(S)
 *                      T1P2(S)    T2P1(S)     T2P2(S)
 * The sizes of each broker are:
 * b0: 190
 * b1: 260
 * b2: 360
 * b3: 250
 * b4: 290
 *
 * The average broker size should be: 270
 */
private ClusterModel createClusterModel() {
    Map<TopicPartition, Float> partitionSize = new HashMap<>();
    partitionSize.put(T0P0, 10f);
    partitionSize.put(T0P1, 90f);
    partitionSize.put(T0P2, 20f);
    partitionSize.put(T1P0, 80f);
    partitionSize.put(T1P1, 30f);
    partitionSize.put(T1P2, 70f);
    partitionSize.put(T2P0, 40f);
    partitionSize.put(T2P1, 60f);
    partitionSize.put(T2P2, 50f);
    final int numRacks = 4;
    ClusterModel clusterModel = new ClusterModel(new ModelGeneration(0, 0), 1.0);
    for (int i = 0; i < numRacks; i++) {
        clusterModel.createRack("r" + i);
    }
    int i = 0;
    for (; i < 2; i++) {
        clusterModel.createBroker("r0", "h" + i, i, TestConstants.BROKER_CAPACITY);
    }
    for (int j = 1; j < numRacks; j++, i++) {
        clusterModel.createBroker("r" + j, "h" + i, i, TestConstants.BROKER_CAPACITY);
    }
    clusterModel.createReplica("r0", 0, T0P0, 0, true);
    clusterModel.createReplica("r0", 0, T1P2, 0, true);
    clusterModel.createReplica("r0", 1, T0P1, 0, true);
    clusterModel.createReplica("r0", 1, T2P0, 0, true);
    clusterModel.createReplica("r1", 2, T0P2, 0, true);
    clusterModel.createReplica("r1", 2, T2P1, 0, true);
    clusterModel.createReplica("r2", 3, T1P0, 0, true);
    clusterModel.createReplica("r2", 3, T2P2, 0, true);
    clusterModel.createReplica("r3", 4, T1P1, 0, true);
    clusterModel.createReplica("r0", 0, T0P2, 1, false);
    clusterModel.createReplica("r0", 0, T2P1, 1, false);
    clusterModel.createReplica("r0", 1, T1P0, 1, false);
    clusterModel.createReplica("r0", 1, T2P2, 1, false);
    clusterModel.createReplica("r1", 2, T0P1, 1, false);
    clusterModel.createReplica("r1", 2, T2P0, 1, false);
    clusterModel.createReplica("r2", 3, T1P1, 1, false);
    clusterModel.createReplica("r3", 4, T0P0, 1, false);
    clusterModel.createReplica("r3", 4, T1P2, 1, false);
    clusterModel.createReplica("r0", 0, T1P1, 2, false);
    clusterModel.createReplica("r1", 2, T1P0, 2, false);
    clusterModel.createReplica("r1", 2, T1P2, 2, false);
    clusterModel.createReplica("r2", 3, T0P0, 2, false);
    clusterModel.createReplica("r2", 3, T0P2, 2, false);
    clusterModel.createReplica("r2", 3, T2P1, 2, false);
    clusterModel.createReplica("r3", 4, T0P1, 2, false);
    clusterModel.createReplica("r3", 4, T2P0, 2, false);
    clusterModel.createReplica("r3", 4, T2P2, 2, false);
    List<Long> windows = Collections.singletonList(1L);
    clusterModel.setReplicaLoad("r0", 0, T0P0, getAggregatedMetricValues(partitionSize.get(T0P0)), windows);
    clusterModel.setReplicaLoad("r0", 0, T1P2, getAggregatedMetricValues(partitionSize.get(T1P2)), windows);
    clusterModel.setReplicaLoad("r0", 0, T0P2, getAggregatedMetricValues(partitionSize.get(T0P2)), windows);
    clusterModel.setReplicaLoad("r0", 0, T2P1, getAggregatedMetricValues(partitionSize.get(T2P1)), windows);
    clusterModel.setReplicaLoad("r0", 0, T1P1, getAggregatedMetricValues(partitionSize.get(T1P1)), windows);
    clusterModel.setReplicaLoad("r0", 1, T0P1, getAggregatedMetricValues(partitionSize.get(T0P1)), windows);
    clusterModel.setReplicaLoad("r0", 1, T2P0, getAggregatedMetricValues(partitionSize.get(T2P0)), windows);
    clusterModel.setReplicaLoad("r0", 1, T1P0, getAggregatedMetricValues(partitionSize.get(T1P0)), windows);
    clusterModel.setReplicaLoad("r0", 1, T2P2, getAggregatedMetricValues(partitionSize.get(T2P2)), windows);
    clusterModel.setReplicaLoad("r1", 2, T0P2, getAggregatedMetricValues(partitionSize.get(T0P2)), windows);
    clusterModel.setReplicaLoad("r1", 2, T2P1, getAggregatedMetricValues(partitionSize.get(T2P1)), windows);
    clusterModel.setReplicaLoad("r1", 2, T0P1, getAggregatedMetricValues(partitionSize.get(T0P1)), windows);
    clusterModel.setReplicaLoad("r1", 2, T2P0, getAggregatedMetricValues(partitionSize.get(T2P0)), windows);
    clusterModel.setReplicaLoad("r1", 2, T1P0, getAggregatedMetricValues(partitionSize.get(T1P0)), windows);
    clusterModel.setReplicaLoad("r1", 2, T1P2, getAggregatedMetricValues(partitionSize.get(T1P2)), windows);
    clusterModel.setReplicaLoad("r2", 3, T1P0, getAggregatedMetricValues(partitionSize.get(T1P0)), windows);
    clusterModel.setReplicaLoad("r2", 3, T2P2, getAggregatedMetricValues(partitionSize.get(T2P2)), windows);
    clusterModel.setReplicaLoad("r2", 3, T1P1, getAggregatedMetricValues(partitionSize.get(T1P1)), windows);
    clusterModel.setReplicaLoad("r2", 3, T0P0, getAggregatedMetricValues(partitionSize.get(T0P0)), windows);
    clusterModel.setReplicaLoad("r2", 3, T0P2, getAggregatedMetricValues(partitionSize.get(T0P2)), windows);
    clusterModel.setReplicaLoad("r2", 3, T2P1, getAggregatedMetricValues(partitionSize.get(T2P1)), windows);
    clusterModel.setReplicaLoad("r3", 4, T1P1, getAggregatedMetricValues(partitionSize.get(T1P1)), windows);
    clusterModel.setReplicaLoad("r3", 4, T0P0, getAggregatedMetricValues(partitionSize.get(T0P0)), windows);
    clusterModel.setReplicaLoad("r3", 4, T1P2, getAggregatedMetricValues(partitionSize.get(T1P2)), windows);
    clusterModel.setReplicaLoad("r3", 4, T0P1, getAggregatedMetricValues(partitionSize.get(T0P1)), windows);
    clusterModel.setReplicaLoad("r3", 4, T2P0, getAggregatedMetricValues(partitionSize.get(T2P0)), windows);
    clusterModel.setReplicaLoad("r3", 4, T2P2, getAggregatedMetricValues(partitionSize.get(T2P2)), windows);
    return clusterModel;
}

Example 5 with ModelGeneration

use of com.linkedin.kafka.cruisecontrol.monitor.ModelGeneration in project cruise-control by linkedin.

the class RandomCluster method generate.

/**
 * Create a random cluster with the given number of racks, brokers and broker capacity.
 *
 * @param clusterProperties Cluster properties specifying number of racks and brokers.
 * @return Cluster with the specified number of racks and brokers.
 */
public static ClusterModel generate(Map<ClusterProperty, Number> clusterProperties) {
    int numRacks = clusterProperties.get(ClusterProperty.NUM_RACKS).intValue();
    int numBrokers = clusterProperties.get(ClusterProperty.NUM_BROKERS).intValue();
    BrokerCapacityConfigFileResolver configFileResolver = new BrokerCapacityConfigFileResolver();
    configFileResolver.configure(Collections.singletonMap(BrokerCapacityConfigFileResolver.CAPACITY_CONFIG_FILE, RandomCluster.class.getClassLoader().getResource("DefaultCapacityConfig.json").getFile()));
    if (numRacks > numBrokers || numBrokers <= 0 || numRacks <= 0) {
        throw new IllegalArgumentException("Random cluster generation failed due to bad input.");
    }
    // Create cluster.
    ClusterModel cluster = new ClusterModel(new ModelGeneration(0, 0L), 1.0);
    // Create racks and add them to cluster.
    for (int i = 0; i < numRacks; i++) {
        cluster.createRack(Integer.toString(i));
    }
    // Create brokers and assign a broker to each rack.
    for (int i = 0; i < numRacks; i++) {
        cluster.createBroker(Integer.toString(i), Integer.toString(i), i, configFileResolver.capacityForBroker("", "", i));
    }
    // Assign the rest of the brokers over racks randomly.
    for (int i = numRacks; i < numBrokers; i++) {
        int randomRackId = uniformlyRandom(0, numRacks - 1, TestConstants.SEED_BASE + i);
        cluster.createBroker(Integer.toString(randomRackId), Integer.toString(i), i, configFileResolver.capacityForBroker("", "", i));
    }
    return cluster;
}