Examples with AggregatedMetricValues com.linkedin.cruisecontrol.monitor.sampling.aggregator.AggregatedMetricValues used on opensource projects

Example 6 with AggregatedMetricValues

use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.AggregatedMetricValues in project cruise-control by linkedin.

the class RandomClusterTest method testNewBrokers.

/**
 * This test first creates a random cluster, balance it. Then add two new brokers, balance the cluster again.
 */
public void testNewBrokers() throws Exception {
    ClusterModel clusterModel = rebalance();
    ClusterModel clusterWithNewBroker = new ClusterModel(new ModelGeneration(0, 0L), 1.0);
    for (Broker b : clusterModel.brokers()) {
        clusterWithNewBroker.createRack(b.rack().id());
        Map<Resource, Double> brokerCapacity = new HashMap<>();
        for (Resource r : Resource.cachedValues()) {
            brokerCapacity.put(r, b.capacityFor(r));
        }
        clusterWithNewBroker.createBroker(b.rack().id(), Integer.toString(b.id()), b.id(), brokerCapacity);
    }
    for (Map.Entry<String, List<Partition>> entry : clusterModel.getPartitionsByTopic().entrySet()) {
        for (Partition p : entry.getValue()) {
            int index = 0;
            for (Replica r : p.replicas()) {
                clusterWithNewBroker.createReplica(r.broker().rack().id(), r.broker().id(), p.topicPartition(), index++, r.isLeader());
            }
        }
    }
    for (Broker b : clusterModel.brokers()) {
        for (Replica replica : b.replicas()) {
            AggregatedMetricValues aggregatedMetricValues = clusterModel.broker(b.id()).replica(replica.topicPartition()).load().loadByWindows();
            clusterWithNewBroker.setReplicaLoad(b.rack().id(), b.id(), replica.topicPartition(), aggregatedMetricValues, clusterModel.load().windows());
        }
    }
    for (int i = 1; i < 3; i++) {
        clusterWithNewBroker.createBroker(Integer.toString(i), Integer.toString(i + clusterModel.brokers().size() - 1), i + clusterModel.brokers().size() - 1, TestConstants.BROKER_CAPACITY);
        clusterWithNewBroker.setBrokerState(i + clusterModel.brokers().size() - 1, Broker.State.NEW);
    }
    assertTrue("Random Cluster Test failed to improve the existing state with new brokers.", OptimizationVerifier.executeGoalsFor(_balancingConstraint, clusterWithNewBroker, _goalNameByPriority, _verifications));
}

Example 7 with AggregatedMetricValues

use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.AggregatedMetricValues in project cruise-control by linkedin.

the class DeterministicCluster method createLoad.

private static AggregatedMetricValues createLoad(double cpu, double networkIn, double networkOut, double disk) {
    AggregatedMetricValues aggregatedMetricValues = new AggregatedMetricValues();
    MetricValues metricValues = new MetricValues(1);
    metricValues.set(0, cpu);
    aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.CPU), metricValues);
    metricValues = new MetricValues(1);
    metricValues.set(0, networkIn);
    aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_IN), metricValues);
    metricValues = new MetricValues(1);
    metricValues.set(0, networkOut);
    aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_OUT), metricValues);
    metricValues = new MetricValues(1);
    metricValues.set(0, disk);
    aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.DISK), metricValues);
    return aggregatedMetricValues;
}

Example 8 with AggregatedMetricValues

use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.AggregatedMetricValues in project cruise-control by linkedin.

the class RandomCluster method populate.

/**
 * Populate the given cluster with replicas having a certain load distribution using the given properties and
 * replica distribution. Balancing constraint sets the resources existing in the cluster at each broker.
 *
 * @param cluster             The state of the cluster.
 * @param properties          Representing the cluster properties as specified in {@link ClusterProperty}.
 * @param replicaDistribution The replica distribution showing the broker of each replica in the cluster.
 * @param rackAware           Whether the replicas should be rack aware or not.
 * @param leaderInFirstPosition Leader of each partition is in the first position or not.
 * @param excludedTopics      Excluded topics.
 */
public static void populate(ClusterModel cluster, Map<ClusterProperty, Number> properties, TestConstants.Distribution replicaDistribution, boolean rackAware, boolean leaderInFirstPosition, Set<String> excludedTopics) {
    // Sanity checks.
    int numBrokers = cluster.brokers().size();
    if (properties.get(ClusterProperty.MEAN_NW_IN).doubleValue() < 0 || properties.get(ClusterProperty.MEAN_NW_OUT).doubleValue() < 0 || properties.get(ClusterProperty.MEAN_DISK).doubleValue() < 0 || properties.get(ClusterProperty.MEAN_CPU).doubleValue() < 0 || properties.get(ClusterProperty.NUM_DEAD_BROKERS).intValue() < 0 || properties.get(ClusterProperty.NUM_TOPICS).intValue() <= 0 || properties.get(ClusterProperty.MIN_REPLICATION).intValue() > properties.get(ClusterProperty.MAX_REPLICATION).intValue() || (leaderInFirstPosition && properties.get(ClusterProperty.MIN_REPLICATION).intValue() < 2) || properties.get(ClusterProperty.MAX_REPLICATION).intValue() > numBrokers || properties.get(ClusterProperty.NUM_TOPICS).intValue() > properties.get(ClusterProperty.NUM_REPLICAS).intValue() || (properties.get(ClusterProperty.MIN_REPLICATION).intValue() == properties.get(ClusterProperty.MAX_REPLICATION).intValue() && properties.get(ClusterProperty.NUM_REPLICAS).intValue() % properties.get(ClusterProperty.MIN_REPLICATION).intValue() != 0)) {
        throw new IllegalArgumentException("Random cluster population failed due to bad input.");
    }
    // Generate topic to number of brokers and replicas distribution.
    List<TopicMetadata> metadata = new ArrayList<>();
    for (int i = 0; i < properties.get(ClusterProperty.NUM_TOPICS).intValue(); i++) {
        metadata.add(new TopicMetadata(i));
    }
    // Increase the replication factor.
    for (int i = 0; i < properties.get(ClusterProperty.NUM_TOPICS).intValue(); i++) {
        int randomReplicationFactor = uniformlyRandom(properties.get(ClusterProperty.MIN_REPLICATION).intValue(), properties.get(ClusterProperty.MAX_REPLICATION).intValue(), TestConstants.REPLICATION_SEED + i);
        metadata.get(i).setReplicationFactor(randomReplicationFactor);
        if (totalTopicReplicas(metadata) > properties.get(ClusterProperty.NUM_REPLICAS).intValue()) {
            // Rollback to minimum replicationFactor.
            metadata.get(i).setReplicationFactor(properties.get(ClusterProperty.MIN_REPLICATION).intValue());
        }
    }
    // Increase the number of leaders.
    int maxRandomLeaders = properties.get(ClusterProperty.NUM_REPLICAS).intValue() / properties.get(ClusterProperty.NUM_TOPICS).intValue();
    for (int i = 0; i < properties.get(ClusterProperty.NUM_TOPICS).intValue(); i++) {
        int oldNumTopicLeaders = metadata.get(i).numTopicLeaders();
        int randomTopicLeaderCount = uniformlyRandom(2, maxRandomLeaders, TestConstants.LEADER_SEED + i);
        metadata.get(i).setNumTopicLeaders(randomTopicLeaderCount);
        if (totalTopicReplicas(metadata) > properties.get(ClusterProperty.NUM_REPLICAS).intValue()) {
            // Rollback to previous number of topic leaders.
            metadata.get(i).setNumTopicLeaders(oldNumTopicLeaders);
        }
    }
    int totalTopicReplicas = totalTopicReplicas(metadata);
    // Fill in the remaining empty spots.
    while (totalTopicReplicas < properties.get(ClusterProperty.NUM_REPLICAS).intValue()) {
        for (int i = 0; i < properties.get(ClusterProperty.NUM_TOPICS).intValue(); i++) {
            metadata.get(i).incrementNumTopicLeaders();
            totalTopicReplicas = totalTopicReplicas(metadata);
            if (totalTopicReplicas > properties.get(ClusterProperty.NUM_REPLICAS).intValue()) {
                // Rollback to previous number of topic leaders.
                metadata.get(i).decrementNumTopicLeaders();
                totalTopicReplicas = totalTopicReplicas(metadata);
            }
            if (totalTopicReplicas == properties.get(ClusterProperty.NUM_REPLICAS).intValue()) {
                break;
            }
        }
    }
    // Create replicas and set their distribution
    int replicaIndex = 0;
    Map<Resource, Random> randomByResource = new HashMap<>();
    for (Resource resource : Resource.cachedValues()) {
        long seed = TestConstants.UTILIZATION_SEED_BY_RESOURCE.get(resource);
        randomByResource.put(resource, new Random(seed));
    }
    Random randomForTopicPopularity = new Random(TestConstants.TOPIC_POPULARITY_SEED);
    for (TopicMetadata datum : metadata) {
        double topicPopularity = exponentialRandom(1.0, randomForTopicPopularity);
        String topic = datum.topic();
        for (int i = 1; i <= datum.numTopicLeaders(); i++) {
            Set<Integer> replicaBrokerIds = new HashSet<>();
            Set<String> replicaRacks = new HashSet<>();
            int brokerConflictResolver = 0;
            TopicPartition pInfo = new TopicPartition(topic, i - 1);
            for (int j = 1; j <= datum.replicationFactor(); j++) {
                int randomBrokerId;
                if (replicaDistribution.equals(TestConstants.Distribution.UNIFORM)) {
                    randomBrokerId = uniformlyRandom(0, numBrokers - 1, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex);
                    while (replicaBrokerIds.contains(randomBrokerId) || (rackAware && replicaRacks.contains(cluster.broker(randomBrokerId).rack().id()))) {
                        brokerConflictResolver++;
                        randomBrokerId = uniformlyRandom(0, numBrokers - 1, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex + brokerConflictResolver);
                    }
                } else if (replicaDistribution.equals(TestConstants.Distribution.LINEAR)) {
                    int binRange = (numBrokers * (numBrokers + 1)) / 2;
                    int randomBinValue = uniformlyRandom(1, binRange, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex);
                    randomBrokerId = 0;
                    for (int bin = 1; bin <= numBrokers; bin++) {
                        int binValue = (2 * randomBinValue);
                        if (binValue <= bin * (bin + 1) && binValue > (bin - 1) * bin) {
                            randomBrokerId = bin - 1;
                            break;
                        }
                    }
                    while (replicaBrokerIds.contains(randomBrokerId) || (rackAware && replicaRacks.contains(cluster.broker(randomBrokerId).rack().id()))) {
                        brokerConflictResolver++;
                        randomBinValue = uniformlyRandom(1, binRange, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex + brokerConflictResolver);
                        for (int bin = 1; bin <= numBrokers; bin++) {
                            int binValue = (2 * randomBinValue);
                            if (binValue <= bin * (bin + 1) && binValue > (bin - 1) * bin) {
                                randomBrokerId = bin - 1;
                                break;
                            }
                        }
                    }
                } else {
                    // Exponential.
                    int binRange = numBrokers * numBrokers;
                    int randomBinValue = uniformlyRandom(1, binRange, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex);
                    randomBrokerId = 0;
                    for (int bin = 1; bin <= numBrokers; bin++) {
                        if (randomBinValue <= bin * bin) {
                            randomBrokerId = bin - 1;
                            break;
                        }
                    }
                    while (replicaBrokerIds.contains(randomBrokerId) || (rackAware && replicaRacks.contains(cluster.broker(randomBrokerId).rack().id()))) {
                        brokerConflictResolver++;
                        randomBinValue = uniformlyRandom(1, binRange, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex + brokerConflictResolver);
                        for (int bin = 1; bin <= numBrokers; bin++) {
                            if (randomBinValue <= bin * bin) {
                                randomBrokerId = bin - 1;
                                break;
                            }
                        }
                    }
                }
                // Set leadership properties and replica load.
                AggregatedMetricValues aggregatedMetricValues = new AggregatedMetricValues();
                MetricValues metricValues = new MetricValues(1);
                metricValues.set(0, exponentialRandom(properties.get(ClusterProperty.MEAN_CPU).doubleValue() * topicPopularity, randomByResource.get(Resource.CPU)));
                aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.CPU), metricValues);
                metricValues = new MetricValues(1);
                metricValues.set(0, exponentialRandom(properties.get(ClusterProperty.MEAN_NW_IN).doubleValue() * topicPopularity, randomByResource.get(Resource.NW_IN)));
                aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_IN), metricValues);
                metricValues = new MetricValues(1);
                metricValues.set(0, exponentialRandom(properties.get(ClusterProperty.MEAN_DISK).doubleValue() * topicPopularity, randomByResource.get(Resource.DISK)));
                aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.DISK), metricValues);
                if (j == 1) {
                    metricValues = new MetricValues(1);
                    metricValues.set(0, exponentialRandom(properties.get(ClusterProperty.MEAN_NW_OUT).doubleValue() * topicPopularity, randomByResource.get(Resource.NW_OUT)));
                    aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_OUT), metricValues);
                    cluster.createReplica(cluster.broker(randomBrokerId).rack().id(), randomBrokerId, pInfo, j - 1, true);
                } else {
                    metricValues = new MetricValues(1);
                    metricValues.set(0, 0.0);
                    aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_OUT), metricValues);
                    cluster.createReplica(cluster.broker(randomBrokerId).rack().id(), randomBrokerId, pInfo, j - 1, false);
                }
                cluster.setReplicaLoad(cluster.broker(randomBrokerId).rack().id(), randomBrokerId, pInfo, aggregatedMetricValues, Collections.singletonList(1L));
                // Update the set of replica locations.
                replicaBrokerIds.add(randomBrokerId);
                replicaRacks.add(cluster.broker(randomBrokerId).rack().id());
                // Update next replica index
                replicaIndex++;
            }
            // Move leader away from the first position if requested.
            if (!leaderInFirstPosition) {
                Partition partition = cluster.partition(pInfo);
                partition.swapReplicaPositions(1, partition.replicas().indexOf(partition.leader()));
            }
        }
    }
    // Mark dead brokers
    int numDeadBrokers = properties.get(ClusterProperty.NUM_DEAD_BROKERS).intValue();
    markDeadBrokers(cluster, numDeadBrokers, excludedTopics, leaderInFirstPosition);
}

Example 9 with AggregatedMetricValues

use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.AggregatedMetricValues in project cruise-control by linkedin.

the class MonitorUtils method toFollowerMetricValues.

/**
 * Derive follower load from the leader load.
 * <p>
 * If linear regression model is used, the The way we derive the follower metrics is the following:
 * <ul>
 * <li>FOLLOWER_CPU = LEADER_NETWORK_IN * coefficient + residue </li>
 * <li>FOLLOWER_DISK = LEADER_DISK </li>
 * <li>FOLLOWER_BYTES_IN = LEADER_BYTES_IN </li>
 * <li>FOLLOWER_BYTES_OUT = 0 </li>
 * </ul>
 *
 * If linear regression model is not used, CPU utilization of the follower will be fixed to be 0.2;
 *
 * @param aggregatedMetricValues the leader aggregated metric values to convert.
 */
public static AggregatedMetricValues toFollowerMetricValues(AggregatedMetricValues aggregatedMetricValues) {
    int cpuId = KafkaCruiseControlMetricDef.resourceToMetricId(Resource.CPU);
    int networkInId = KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_IN);
    int networkOutId = KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_OUT);
    AggregatedMetricValues followerLoad = new AggregatedMetricValues();
    for (int metricId : aggregatedMetricValues.metricIds()) {
        if (metricId != cpuId && metricId != networkOutId) {
            followerLoad.add(metricId, aggregatedMetricValues.valuesFor(metricId));
        }
    }
    MetricValues followerCpu = new MetricValues(aggregatedMetricValues.length());
    for (int i = 0; i < aggregatedMetricValues.length(); i++) {
        double followerCpuUtil = ModelUtils.getFollowerCpuUtilFromLeaderLoad(aggregatedMetricValues.valuesFor(networkInId).get(i), aggregatedMetricValues.valuesFor(networkOutId).get(i), aggregatedMetricValues.valuesFor(cpuId).get(i));
        followerCpu.set(i, followerCpuUtil);
    }
    followerLoad.add(networkOutId, new MetricValues(aggregatedMetricValues.length()));
    followerLoad.add(cpuId, followerCpu);
    return followerLoad;
}