Examples with ClusterModel com.linkedin.kafka.cruisecontrol.model.ClusterModel used on opensource projects

Example 31 with ClusterModel

use of com.linkedin.kafka.cruisecontrol.model.ClusterModel in project cruise-control by linkedin.

the class KafkaAssignerDiskUsageDistributionGoalTest method testFindReplicaToSwapWith.

@Test
public void testFindReplicaToSwapWith() {
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    props.setProperty(KafkaCruiseControlConfig.MAX_REPLICAS_PER_BROKER_CONFIG, Long.toString(10L));
    props.setProperty(KafkaCruiseControlConfig.DISK_BALANCE_THRESHOLD_CONFIG, "1.05");
    BalancingConstraint balancingConstraint = new BalancingConstraint(new KafkaCruiseControlConfig(props));
    KafkaAssignerDiskUsageDistributionGoal goal = new KafkaAssignerDiskUsageDistributionGoal(balancingConstraint);
    ClusterModel clusterModel = createClusterModel();
    Broker b2 = clusterModel.broker(2);
    Replica r = b2.replica(T0P1);
    assertNull(goal.findReplicaToSwapWith(r, sortedReplicaAscend(clusterModel.broker(1)), 30, 10, 90, clusterModel));
    // The replicas on broker 3 are of the following sizes
    // T0P0: 10
    // T0P2: 20
    // T1P1: 30
    // T2P2: 50
    // T2P1: 60
    // T1P0: 80
    // Only T0P0 and T1P1 are eligible to swap with r.
    findReplicaToSwapWithAndVerify(Arrays.asList(-1.0, 5.0, 10.0, 20.0, 21.0, 60.0, 100.0), Arrays.asList(T0P0, T0P0, T0P0, T0P0, T1P1, T1P1, T1P1), 9, 90, r, 3, clusterModel, goal);
    findReplicaToSwapWithAndVerify(Arrays.asList(-1.0, 5.0, 10.0, 20.0, 21.0, 60.0, 100.0), Arrays.asList(T1P1, T1P1, T1P1, T1P1, T1P1, T1P1, T1P1), 10, 31, r, 3, clusterModel, goal);
    findReplicaToSwapWithAndVerify(Arrays.asList(-1.0, 5.0, 10.0, 20.0, 21.0, 60.0, 100.0), Arrays.asList(T0P0, T0P0, T0P0, T0P0, T0P0, T0P0, T0P0), 9, 30, r, 3, clusterModel, goal);
    findReplicaToSwapWithAndVerify(Arrays.asList(-1.0, 5.0, 10.0, 20.0, 21.0, 60.0, 100.0), Arrays.asList(null, null, null, null, null, null, null), 10, 30, r, 3, clusterModel, goal);
}

Example 32 with ClusterModel

use of com.linkedin.kafka.cruisecontrol.model.ClusterModel in project cruise-control by linkedin.

the class LoadMonitorTest method testClusterModelWithInvalidPartitionAndInsufficientSnapshotWindows.

// Not enough snapshot windows and some partitions are missing from all snapshot windows.
@Test
public void testClusterModelWithInvalidPartitionAndInsufficientSnapshotWindows() throws NotEnoughValidWindowsException {
    TestContext context = prepareContext();
    LoadMonitor loadMonitor = context.loadmonitor();
    KafkaMetricSampleAggregator aggregator = context.aggregator();
    ModelCompletenessRequirements requirements1 = new ModelCompletenessRequirements(1, 1.0, false);
    ModelCompletenessRequirements requirements2 = new ModelCompletenessRequirements(1, 0.5, false);
    ModelCompletenessRequirements requirements3 = new ModelCompletenessRequirements(2, 1.0, false);
    ModelCompletenessRequirements requirements4 = new ModelCompletenessRequirements(2, 0.5, false);
    // populate the metrics aggregator.
    // two samples for each partition except T1P1
    CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T0P0, 0, WINDOW_MS, METRIC_DEF);
    CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T0P1, 0, WINDOW_MS, METRIC_DEF);
    CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T1P0, 0, WINDOW_MS, METRIC_DEF);
    try {
        loadMonitor.clusterModel(-1, Long.MAX_VALUE, requirements1, new OperationProgress());
        fail("Should have thrown NotEnoughValidWindowsException.");
    } catch (NotEnoughValidWindowsException nevwe) {
    // let it go
    }
    ClusterModel clusterModel = loadMonitor.clusterModel(-1L, Long.MAX_VALUE, requirements2, new OperationProgress());
    assertNull(clusterModel.partition(T1P0));
    assertNull(clusterModel.partition(T1P1));
    assertEquals(1, clusterModel.partition(T0P0).leader().load().numWindows());
    assertEquals(3, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.DISK), 0.0);
    assertEquals(1.5, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.CPU), 0.0);
    assertEquals(1.5, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.NW_IN), 0.0);
    assertEquals(1.5, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.NW_OUT), 0.0);
    try {
        loadMonitor.clusterModel(-1L, Long.MAX_VALUE, requirements3, new OperationProgress());
        fail("Should have thrown NotEnoughValidWindowsException.");
    } catch (NotEnoughValidWindowsException nevwe) {
    // let it go
    }
    try {
        loadMonitor.clusterModel(-1L, Long.MAX_VALUE, requirements4, new OperationProgress());
        fail("Should have thrown NotEnoughValidWindowsException.");
    } catch (NotEnoughValidWindowsException nevwe) {
    // let it go
    }
}

Example 33 with ClusterModel

use of com.linkedin.kafka.cruisecontrol.model.ClusterModel in project cruise-control by linkedin.

the class DeterministicCluster method mediumClusterModel.

/**
 * Generates a medium test cluster.
 * <p>
 * <li>Number of Partitions: 12.</li>
 * <li>Topics: A, B, C, D</li>
 * <li>Replication factor/Topic: A:2, B:2, C:2, D:2</li>
 * <li>Partitions/Topic: A: 6, B:2, C:2, D:2</li>
 *
 * @return A medium test cluster.
 */
public static ClusterModel mediumClusterModel(Map<Resource, Double> brokerCapacity) {
    List<Integer> orderedRackIdsOfBrokers = Arrays.asList(0, 0, 1);
    ClusterModel cluster = getHomogeneousDeterministicCluster(2, orderedRackIdsOfBrokers, brokerCapacity);
    // Create topic partition.
    TopicPartition pInfoA0 = new TopicPartition("A", 0);
    TopicPartition pInfoA1 = new TopicPartition("A", 1);
    TopicPartition pInfoA2 = new TopicPartition("A", 2);
    TopicPartition pInfoB0 = new TopicPartition("B", 0);
    TopicPartition pInfoC0 = new TopicPartition("C", 0);
    TopicPartition pInfoD0 = new TopicPartition("D", 0);
    // Create replicas for TopicA.
    cluster.createReplica("0", 1, pInfoA0, 0, true);
    cluster.createReplica("0", 0, pInfoA0, 1, false);
    cluster.createReplica("0", 0, pInfoA1, 0, true);
    cluster.createReplica("1", 2, pInfoA1, 1, false);
    cluster.createReplica("0", 0, pInfoA2, 0, true);
    cluster.createReplica("1", 2, pInfoA2, 1, false);
    // Create replicas for TopicB.
    cluster.createReplica("0", 1, pInfoB0, 0, true);
    cluster.createReplica("1", 2, pInfoB0, 1, false);
    // Create replicas for TopicC.
    cluster.createReplica("1", 2, pInfoC0, 0, true);
    cluster.createReplica("0", 1, pInfoC0, 1, false);
    // Create replicas for TopicD.
    cluster.createReplica("0", 1, pInfoD0, 0, true);
    cluster.createReplica("1", 2, pInfoD0, 1, false);
    // Create snapshots and push them to the cluster.
    List<Long> windows = Collections.singletonList(1L);
    cluster.setReplicaLoad("0", 0, pInfoA0, createLoad(5.0, 5.0, 0.0, 4.0), windows);
    cluster.setReplicaLoad("0", 0, pInfoA1, createLoad(5.0, 3.0, 10.0, 8.0), windows);
    cluster.setReplicaLoad("0", 0, pInfoA2, createLoad(5.0, 2.0, 10.0, 6.0), windows);
    cluster.setReplicaLoad("0", 1, pInfoB0, createLoad(5.0, 4.0, 10.0, 7.0), windows);
    cluster.setReplicaLoad("0", 1, pInfoC0, createLoad(5.0, 6.0, 0.0, 4.0), windows);
    cluster.setReplicaLoad("0", 1, pInfoD0, createLoad(5.0, 5.0, 10.0, 6.0), windows);
    cluster.setReplicaLoad("0", 1, pInfoA0, createLoad(5.0, 4.0, 10.0, 10.0), windows);
    cluster.setReplicaLoad("1", 2, pInfoB0, createLoad(2.0, 2.0, 0.0, 5.0), windows);
    cluster.setReplicaLoad("1", 2, pInfoC0, createLoad(1.0, 8.0, 10.0, 4.0), windows);
    cluster.setReplicaLoad("1", 2, pInfoD0, createLoad(2.0, 8.0, 0.0, 7.0), windows);
    cluster.setReplicaLoad("1", 2, pInfoA1, createLoad(3.0, 4.0, 0.0, 6.0), windows);
    cluster.setReplicaLoad("1", 2, pInfoA2, createLoad(4.0, 5.0, 0.0, 3.0), windows);
    return cluster;
}

Example 34 with ClusterModel

use of com.linkedin.kafka.cruisecontrol.model.ClusterModel in project cruise-control by linkedin.

the class RandomCluster method generate.

/**
 * Create a random cluster with the given number of racks, brokers and broker capacity.
 *
 * @param clusterProperties Cluster properties specifying number of racks and brokers.
 * @return Cluster with the specified number of racks and brokers.
 */
public static ClusterModel generate(Map<ClusterProperty, Number> clusterProperties) {
    int numRacks = clusterProperties.get(ClusterProperty.NUM_RACKS).intValue();
    int numBrokers = clusterProperties.get(ClusterProperty.NUM_BROKERS).intValue();
    BrokerCapacityConfigFileResolver configFileResolver = new BrokerCapacityConfigFileResolver();
    configFileResolver.configure(Collections.singletonMap(BrokerCapacityConfigFileResolver.CAPACITY_CONFIG_FILE, RandomCluster.class.getClassLoader().getResource("DefaultCapacityConfig.json").getFile()));
    if (numRacks > numBrokers || numBrokers <= 0 || numRacks <= 0) {
        throw new IllegalArgumentException("Random cluster generation failed due to bad input.");
    }
    // Create cluster.
    ClusterModel cluster = new ClusterModel(new ModelGeneration(0, 0L), 1.0);
    // Create racks and add them to cluster.
    for (int i = 0; i < numRacks; i++) {
        cluster.createRack(Integer.toString(i));
    }
    // Create brokers and assign a broker to each rack.
    for (int i = 0; i < numRacks; i++) {
        cluster.createBroker(Integer.toString(i), Integer.toString(i), i, configFileResolver.capacityForBroker("", "", i));
    }
    // Assign the rest of the brokers over racks randomly.
    for (int i = numRacks; i < numBrokers; i++) {
        int randomRackId = uniformlyRandom(0, numRacks - 1, TestConstants.SEED_BASE + i);
        cluster.createBroker(Integer.toString(randomRackId), Integer.toString(i), i, configFileResolver.capacityForBroker("", "", i));
    }
    return cluster;
}

Example 35 with ClusterModel

use of com.linkedin.kafka.cruisecontrol.model.ClusterModel in project cruise-control by linkedin.

the class LoadMonitor method clusterModel.

/**
 * Get the most recent cluster load model before the given timestamp.
 *
 * @param now The current time in millisecond.
 * @param requirements the load requirements for getting the cluster model.
 * @param operationProgress the progress to report.
 * @return A cluster model with the configured number of windows whose timestamp is before given timestamp.
 */
public ClusterModel clusterModel(long now, ModelCompletenessRequirements requirements, OperationProgress operationProgress) throws NotEnoughValidWindowsException {
    ClusterModel clusterModel = clusterModel(-1L, now, requirements, operationProgress);
    // Micro optimization: put the broker stats construction out of the lock.
    ClusterModel.BrokerStats brokerStats = clusterModel.brokerStats();
    // update the cached brokerLoadStats
    synchronized (this) {
        _cachedBrokerLoadStats = brokerStats;
        _cachedBrokerLoadGeneration = clusterModel.generation();
    }
    return clusterModel;
}