Examples with ClusterModel com.linkedin.kafka.cruisecontrol.model.ClusterModel used on opensource projects

Example 21 with ClusterModel

use of com.linkedin.kafka.cruisecontrol.model.ClusterModel in project cruise-control by linkedin.

the class ExcludedTopicsTest method unbalanced.

// two racks, three brokers, two partitions, one replica.
private static ClusterModel unbalanced() {
    List<Integer> orderedRackIdsOfBrokers = Arrays.asList(0, 0, 1);
    ClusterModel cluster = DeterministicCluster.getHomogeneousDeterministicCluster(2, orderedRackIdsOfBrokers, TestConstants.BROKER_CAPACITY);
    // Create topic partition.
    TopicPartition pInfoT10 = new TopicPartition("T1", 0);
    TopicPartition pInfoT20 = new TopicPartition("T2", 0);
    // Create replicas for topic: T1.
    cluster.createReplica("0", 0, pInfoT10, 0, true);
    cluster.createReplica("0", 0, pInfoT20, 0, true);
    AggregatedMetricValues aggregatedMetricValues = KafkaCruiseControlUnitTestUtils.getAggregatedMetricValues(TestConstants.TYPICAL_CPU_CAPACITY / 2, TestConstants.LARGE_BROKER_CAPACITY / 2, TestConstants.MEDIUM_BROKER_CAPACITY / 2, TestConstants.LARGE_BROKER_CAPACITY / 2);
    // Create snapshots and push them to the cluster.
    cluster.setReplicaLoad("0", 0, pInfoT10, aggregatedMetricValues, Collections.singletonList(1L));
    cluster.setReplicaLoad("0", 0, pInfoT20, aggregatedMetricValues, Collections.singletonList(1L));
    return cluster;
}

Example 22 with ClusterModel

use of com.linkedin.kafka.cruisecontrol.model.ClusterModel in project cruise-control by linkedin.

the class ExcludedTopicsTest method unbalanced2.

// two racks, three brokers, six partitions, one replica.
private static ClusterModel unbalanced2() {
    ClusterModel cluster = unbalanced();
    // Create topic partition.
    TopicPartition pInfoT30 = new TopicPartition("T1", 1);
    TopicPartition pInfoT40 = new TopicPartition("T2", 1);
    TopicPartition pInfoT50 = new TopicPartition("T1", 2);
    TopicPartition pInfoT60 = new TopicPartition("T2", 2);
    // Create replicas for topic: T1.
    cluster.createReplica("0", 1, pInfoT30, 0, true);
    cluster.createReplica("0", 0, pInfoT40, 0, true);
    cluster.createReplica("0", 0, pInfoT50, 0, true);
    cluster.createReplica("0", 0, pInfoT60, 0, true);
    AggregatedMetricValues aggregatedMetricValues = KafkaCruiseControlUnitTestUtils.getAggregatedMetricValues(TestConstants.LARGE_BROKER_CAPACITY / 2, TestConstants.LARGE_BROKER_CAPACITY / 2, TestConstants.MEDIUM_BROKER_CAPACITY / 2, TestConstants.LARGE_BROKER_CAPACITY / 2);
    // Create snapshots and push them to the cluster.
    cluster.setReplicaLoad("0", 1, pInfoT30, aggregatedMetricValues, Collections.singletonList(1L));
    cluster.setReplicaLoad("0", 0, pInfoT40, aggregatedMetricValues, Collections.singletonList(1L));
    cluster.setReplicaLoad("0", 0, pInfoT50, aggregatedMetricValues, Collections.singletonList(1L));
    cluster.setReplicaLoad("0", 0, pInfoT60, aggregatedMetricValues, Collections.singletonList(1L));
    return cluster;
}

Example 23 with ClusterModel

use of com.linkedin.kafka.cruisecontrol.model.ClusterModel in project cruise-control by linkedin.

the class OfflineProposalGenerator method main.

public static void main(String[] argv) throws Exception {
    // TODO: probably need to save this in the original model file
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(props);
    ModelUtils.init(config);
    ModelParameters.init(config);
    BalancingConstraint balancingConstraint = new BalancingConstraint(config);
    long start = System.currentTimeMillis();
    ClusterModel clusterModel = clusterModelFromFile(argv[0]);
    long end = System.currentTimeMillis();
    double duration = (end - start) / 1000.0;
    System.out.println("Model loaded in " + duration + "s.");
    ClusterModelStats origStats = clusterModel.getClusterStats(balancingConstraint);
    String loadBeforeOptimization = clusterModel.brokerStats().toString();
    // Instantiate the components.
    GoalOptimizer goalOptimizer = new GoalOptimizer(config, null, new SystemTime(), new MetricRegistry());
    start = System.currentTimeMillis();
    GoalOptimizer.OptimizerResult optimizerResult = goalOptimizer.optimizations(clusterModel, new OperationProgress());
    end = System.currentTimeMillis();
    duration = (end - start) / 1000.0;
    String loadAfterOptimization = clusterModel.brokerStats().toString();
    System.out.println("Optimize goals in " + duration + "s.");
    System.out.println(optimizerResult.goalProposals().size());
    System.out.println(loadBeforeOptimization);
    System.out.println(loadAfterOptimization);
    ClusterModelStats optimizedStats = clusterModel.getClusterStats(balancingConstraint);
    double[] testStatistics = AnalyzerUtils.testDifference(origStats.utilizationMatrix(), optimizedStats.utilizationMatrix());
    System.out.println(Arrays.stream(RawAndDerivedResource.values()).map(x -> x.toString()).collect(Collectors.joining(", ")));
    System.out.println(Arrays.stream(testStatistics).boxed().map(pValue -> Double.toString(pValue)).collect(Collectors.joining(", ")));
}

Example 24 with ClusterModel

use of com.linkedin.kafka.cruisecontrol.model.ClusterModel in project cruise-control by linkedin.

the class PreferredLeaderElectionGoalTest method createClusterModel.

private ClusterModel createClusterModel() {
    final int numRacks = 4;
    ClusterModel clusterModel = new ClusterModel(new ModelGeneration(0, 0), 1.0);
    for (int i = 0; i < numRacks; i++) {
        clusterModel.createRack("r" + i);
    }
    int i = 0;
    for (; i < 2; i++) {
        clusterModel.createBroker("r0", "h" + i, i, TestConstants.BROKER_CAPACITY);
    }
    for (int j = 1; j < numRacks; j++, i++) {
        clusterModel.createBroker("r" + j, "h" + i, i, TestConstants.BROKER_CAPACITY);
    }
    createReplicaAndSetLoad(clusterModel, "r0", 0, T0P0, 0, true);
    createReplicaAndSetLoad(clusterModel, "r0", 1, T0P1, 0, true);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T0P2, 0, true);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T1P0, 0, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T1P1, 0, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, T1P2, 0, false);
    createReplicaAndSetLoad(clusterModel, "r0", 1, T2P0, 0, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T2P1, 0, false);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T2P2, 0, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T0P0, 1, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T0P1, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, T0P2, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 1, T1P0, 1, true);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T1P1, 1, true);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T1P2, 1, true);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T2P0, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, T2P1, 1, false);
    createReplicaAndSetLoad(clusterModel, "r0", 1, T2P2, 1, false);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T0P0, 2, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T0P1, 2, false);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T0P2, 2, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T1P0, 2, false);
    createReplicaAndSetLoad(clusterModel, "r0", 0, T1P1, 2, false);
    createReplicaAndSetLoad(clusterModel, "r1", 2, T1P2, 2, false);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T2P0, 2, true);
    createReplicaAndSetLoad(clusterModel, "r2", 3, T2P1, 2, true);
    createReplicaAndSetLoad(clusterModel, "r3", 4, T2P2, 2, true);
    return clusterModel;
}

Example 25 with ClusterModel

use of com.linkedin.kafka.cruisecontrol.model.ClusterModel in project cruise-control by linkedin.

the class PreferredLeaderElectionGoalTest method testOptimize.

@Test
public void testOptimize() throws KafkaCruiseControlException {
    ClusterModel clusterModel = createClusterModel();
    PreferredLeaderElectionGoal goal = new PreferredLeaderElectionGoal();
    goal.optimize(clusterModel, Collections.emptySet(), Collections.emptySet());
    for (String t : Arrays.asList(TOPIC0, TOPIC1, TOPIC2)) {
        for (int p = 0; p < 3; p++) {
            List<Replica> replicas = clusterModel.partition(new TopicPartition(t, p)).replicas();
            for (int i = 0; i < 3; i++) {
                // only the first replica should be leader.
                assertEquals(i == 0, replicas.get(i).isLeader());
            }
        }
    }
}