use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.MetricValues in project cruise-control by linkedin.
the class RandomCluster method populate.
/**
* Populate the given cluster with replicas having a certain load distribution using the given properties and
* replica distribution. Balancing constraint sets the resources existing in the cluster at each broker.
*
* @param cluster The state of the cluster.
* @param properties Representing the cluster properties as specified in {@link ClusterProperty}.
* @param replicaDistribution The replica distribution showing the broker of each replica in the cluster.
* @param rackAware Whether the replicas should be rack aware or not.
* @param leaderInFirstPosition Leader of each partition is in the first position or not.
* @param excludedTopics Excluded topics.
*/
public static void populate(ClusterModel cluster, Map<ClusterProperty, Number> properties, TestConstants.Distribution replicaDistribution, boolean rackAware, boolean leaderInFirstPosition, Set<String> excludedTopics) {
// Sanity checks.
int numBrokers = cluster.brokers().size();
if (properties.get(ClusterProperty.MEAN_NW_IN).doubleValue() < 0 || properties.get(ClusterProperty.MEAN_NW_OUT).doubleValue() < 0 || properties.get(ClusterProperty.MEAN_DISK).doubleValue() < 0 || properties.get(ClusterProperty.MEAN_CPU).doubleValue() < 0 || properties.get(ClusterProperty.NUM_DEAD_BROKERS).intValue() < 0 || properties.get(ClusterProperty.NUM_TOPICS).intValue() <= 0 || properties.get(ClusterProperty.MIN_REPLICATION).intValue() > properties.get(ClusterProperty.MAX_REPLICATION).intValue() || (leaderInFirstPosition && properties.get(ClusterProperty.MIN_REPLICATION).intValue() < 2) || properties.get(ClusterProperty.MAX_REPLICATION).intValue() > numBrokers || properties.get(ClusterProperty.NUM_TOPICS).intValue() > properties.get(ClusterProperty.NUM_REPLICAS).intValue() || (properties.get(ClusterProperty.MIN_REPLICATION).intValue() == properties.get(ClusterProperty.MAX_REPLICATION).intValue() && properties.get(ClusterProperty.NUM_REPLICAS).intValue() % properties.get(ClusterProperty.MIN_REPLICATION).intValue() != 0)) {
throw new IllegalArgumentException("Random cluster population failed due to bad input.");
}
// Generate topic to number of brokers and replicas distribution.
List<TopicMetadata> metadata = new ArrayList<>();
for (int i = 0; i < properties.get(ClusterProperty.NUM_TOPICS).intValue(); i++) {
metadata.add(new TopicMetadata(i));
}
// Increase the replication factor.
for (int i = 0; i < properties.get(ClusterProperty.NUM_TOPICS).intValue(); i++) {
int randomReplicationFactor = uniformlyRandom(properties.get(ClusterProperty.MIN_REPLICATION).intValue(), properties.get(ClusterProperty.MAX_REPLICATION).intValue(), TestConstants.REPLICATION_SEED + i);
metadata.get(i).setReplicationFactor(randomReplicationFactor);
if (totalTopicReplicas(metadata) > properties.get(ClusterProperty.NUM_REPLICAS).intValue()) {
// Rollback to minimum replicationFactor.
metadata.get(i).setReplicationFactor(properties.get(ClusterProperty.MIN_REPLICATION).intValue());
}
}
// Increase the number of leaders.
int maxRandomLeaders = properties.get(ClusterProperty.NUM_REPLICAS).intValue() / properties.get(ClusterProperty.NUM_TOPICS).intValue();
for (int i = 0; i < properties.get(ClusterProperty.NUM_TOPICS).intValue(); i++) {
int oldNumTopicLeaders = metadata.get(i).numTopicLeaders();
int randomTopicLeaderCount = uniformlyRandom(2, maxRandomLeaders, TestConstants.LEADER_SEED + i);
metadata.get(i).setNumTopicLeaders(randomTopicLeaderCount);
if (totalTopicReplicas(metadata) > properties.get(ClusterProperty.NUM_REPLICAS).intValue()) {
// Rollback to previous number of topic leaders.
metadata.get(i).setNumTopicLeaders(oldNumTopicLeaders);
}
}
int totalTopicReplicas = totalTopicReplicas(metadata);
// Fill in the remaining empty spots.
while (totalTopicReplicas < properties.get(ClusterProperty.NUM_REPLICAS).intValue()) {
for (int i = 0; i < properties.get(ClusterProperty.NUM_TOPICS).intValue(); i++) {
metadata.get(i).incrementNumTopicLeaders();
totalTopicReplicas = totalTopicReplicas(metadata);
if (totalTopicReplicas > properties.get(ClusterProperty.NUM_REPLICAS).intValue()) {
// Rollback to previous number of topic leaders.
metadata.get(i).decrementNumTopicLeaders();
totalTopicReplicas = totalTopicReplicas(metadata);
}
if (totalTopicReplicas == properties.get(ClusterProperty.NUM_REPLICAS).intValue()) {
break;
}
}
}
// Create replicas and set their distribution
int replicaIndex = 0;
Map<Resource, Random> randomByResource = new HashMap<>();
for (Resource resource : Resource.cachedValues()) {
long seed = TestConstants.UTILIZATION_SEED_BY_RESOURCE.get(resource);
randomByResource.put(resource, new Random(seed));
}
Random randomForTopicPopularity = new Random(TestConstants.TOPIC_POPULARITY_SEED);
for (TopicMetadata datum : metadata) {
double topicPopularity = exponentialRandom(1.0, randomForTopicPopularity);
String topic = datum.topic();
for (int i = 1; i <= datum.numTopicLeaders(); i++) {
Set<Integer> replicaBrokerIds = new HashSet<>();
Set<String> replicaRacks = new HashSet<>();
int brokerConflictResolver = 0;
TopicPartition pInfo = new TopicPartition(topic, i - 1);
for (int j = 1; j <= datum.replicationFactor(); j++) {
int randomBrokerId;
if (replicaDistribution.equals(TestConstants.Distribution.UNIFORM)) {
randomBrokerId = uniformlyRandom(0, numBrokers - 1, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex);
while (replicaBrokerIds.contains(randomBrokerId) || (rackAware && replicaRacks.contains(cluster.broker(randomBrokerId).rack().id()))) {
brokerConflictResolver++;
randomBrokerId = uniformlyRandom(0, numBrokers - 1, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex + brokerConflictResolver);
}
} else if (replicaDistribution.equals(TestConstants.Distribution.LINEAR)) {
int binRange = (numBrokers * (numBrokers + 1)) / 2;
int randomBinValue = uniformlyRandom(1, binRange, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex);
randomBrokerId = 0;
for (int bin = 1; bin <= numBrokers; bin++) {
int binValue = (2 * randomBinValue);
if (binValue <= bin * (bin + 1) && binValue > (bin - 1) * bin) {
randomBrokerId = bin - 1;
break;
}
}
while (replicaBrokerIds.contains(randomBrokerId) || (rackAware && replicaRacks.contains(cluster.broker(randomBrokerId).rack().id()))) {
brokerConflictResolver++;
randomBinValue = uniformlyRandom(1, binRange, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex + brokerConflictResolver);
for (int bin = 1; bin <= numBrokers; bin++) {
int binValue = (2 * randomBinValue);
if (binValue <= bin * (bin + 1) && binValue > (bin - 1) * bin) {
randomBrokerId = bin - 1;
break;
}
}
}
} else {
// Exponential.
int binRange = numBrokers * numBrokers;
int randomBinValue = uniformlyRandom(1, binRange, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex);
randomBrokerId = 0;
for (int bin = 1; bin <= numBrokers; bin++) {
if (randomBinValue <= bin * bin) {
randomBrokerId = bin - 1;
break;
}
}
while (replicaBrokerIds.contains(randomBrokerId) || (rackAware && replicaRacks.contains(cluster.broker(randomBrokerId).rack().id()))) {
brokerConflictResolver++;
randomBinValue = uniformlyRandom(1, binRange, TestConstants.REPLICA_ASSIGNMENT_SEED + replicaIndex + brokerConflictResolver);
for (int bin = 1; bin <= numBrokers; bin++) {
if (randomBinValue <= bin * bin) {
randomBrokerId = bin - 1;
break;
}
}
}
}
// Set leadership properties and replica load.
AggregatedMetricValues aggregatedMetricValues = new AggregatedMetricValues();
MetricValues metricValues = new MetricValues(1);
metricValues.set(0, exponentialRandom(properties.get(ClusterProperty.MEAN_CPU).doubleValue() * topicPopularity, randomByResource.get(Resource.CPU)));
aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.CPU), metricValues);
metricValues = new MetricValues(1);
metricValues.set(0, exponentialRandom(properties.get(ClusterProperty.MEAN_NW_IN).doubleValue() * topicPopularity, randomByResource.get(Resource.NW_IN)));
aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_IN), metricValues);
metricValues = new MetricValues(1);
metricValues.set(0, exponentialRandom(properties.get(ClusterProperty.MEAN_DISK).doubleValue() * topicPopularity, randomByResource.get(Resource.DISK)));
aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.DISK), metricValues);
if (j == 1) {
metricValues = new MetricValues(1);
metricValues.set(0, exponentialRandom(properties.get(ClusterProperty.MEAN_NW_OUT).doubleValue() * topicPopularity, randomByResource.get(Resource.NW_OUT)));
aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_OUT), metricValues);
cluster.createReplica(cluster.broker(randomBrokerId).rack().id(), randomBrokerId, pInfo, j - 1, true);
} else {
metricValues = new MetricValues(1);
metricValues.set(0, 0.0);
aggregatedMetricValues.add(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_OUT), metricValues);
cluster.createReplica(cluster.broker(randomBrokerId).rack().id(), randomBrokerId, pInfo, j - 1, false);
}
cluster.setReplicaLoad(cluster.broker(randomBrokerId).rack().id(), randomBrokerId, pInfo, aggregatedMetricValues, Collections.singletonList(1L));
// Update the set of replica locations.
replicaBrokerIds.add(randomBrokerId);
replicaRacks.add(cluster.broker(randomBrokerId).rack().id());
// Update next replica index
replicaIndex++;
}
// Move leader away from the first position if requested.
if (!leaderInFirstPosition) {
Partition partition = cluster.partition(pInfo);
partition.swapReplicaPositions(1, partition.replicas().indexOf(partition.leader()));
}
}
}
// Mark dead brokers
int numDeadBrokers = properties.get(ClusterProperty.NUM_DEAD_BROKERS).intValue();
markDeadBrokers(cluster, numDeadBrokers, excludedTopics, leaderInFirstPosition);
}
use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.MetricValues in project cruise-control by linkedin.
the class MonitorUtils method toFollowerMetricValues.
/**
* Derive follower load from the leader load.
* <p>
* If linear regression model is used, the The way we derive the follower metrics is the following:
* <ul>
* <li>FOLLOWER_CPU = LEADER_NETWORK_IN * coefficient + residue </li>
* <li>FOLLOWER_DISK = LEADER_DISK </li>
* <li>FOLLOWER_BYTES_IN = LEADER_BYTES_IN </li>
* <li>FOLLOWER_BYTES_OUT = 0 </li>
* </ul>
*
* If linear regression model is not used, CPU utilization of the follower will be fixed to be 0.2;
*
* @param aggregatedMetricValues the leader aggregated metric values to convert.
*/
public static AggregatedMetricValues toFollowerMetricValues(AggregatedMetricValues aggregatedMetricValues) {
int cpuId = KafkaCruiseControlMetricDef.resourceToMetricId(Resource.CPU);
int networkInId = KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_IN);
int networkOutId = KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_OUT);
AggregatedMetricValues followerLoad = new AggregatedMetricValues();
for (int metricId : aggregatedMetricValues.metricIds()) {
if (metricId != cpuId && metricId != networkOutId) {
followerLoad.add(metricId, aggregatedMetricValues.valuesFor(metricId));
}
}
MetricValues followerCpu = new MetricValues(aggregatedMetricValues.length());
for (int i = 0; i < aggregatedMetricValues.length(); i++) {
double followerCpuUtil = ModelUtils.getFollowerCpuUtilFromLeaderLoad(aggregatedMetricValues.valuesFor(networkInId).get(i), aggregatedMetricValues.valuesFor(networkOutId).get(i), aggregatedMetricValues.valuesFor(cpuId).get(i));
followerCpu.set(i, followerCpuUtil);
}
followerLoad.add(networkOutId, new MetricValues(aggregatedMetricValues.length()));
followerLoad.add(cpuId, followerCpu);
return followerLoad;
}
use of com.linkedin.cruisecontrol.monitor.sampling.aggregator.MetricValues in project cruise-control by linkedin.
the class Load method getJsonStructure.
/**
* Return an object that can be further used
* to encode into JSON
*/
public Map<String, Object> getJsonStructure() {
MetricDef metricDef = KafkaCruiseControlMetricDef.metricDef();
Map<String, Object> loadMap = new HashMap<>();
List<Object> metricValueList = new ArrayList<>();
for (MetricInfo metricInfo : metricDef.all()) {
MetricValues metricValues = _metricValues.valuesFor(metricInfo.id());
if (metricValues != null) {
Map<Long, Double> metricValuesMap = new HashMap<>();
for (int i = 0; i < _windows.size(); i++) {
metricValuesMap.put(_windows.get(i), metricValues.get(i));
}
metricValueList.add(metricValuesMap);
}
}
loadMap.put("MetricValues", metricValueList);
return loadMap;
}
Aggregations