use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.
the class KafkaCruiseControlUnitTestUtils method getAggregatedMetricValues.
/**
* Get the aggregated metric values with the given resource usage.
*/
public static AggregatedMetricValues getAggregatedMetricValues(double cpuUsage, double networkInBoundUsage, double networkOutBoundUsage, double diskUsage) {
double[] values = new double[Resource.cachedValues().size()];
values[KafkaCruiseControlMetricDef.resourceToMetricId(CPU)] = cpuUsage;
values[KafkaCruiseControlMetricDef.resourceToMetricId(NW_IN)] = networkInBoundUsage;
values[KafkaCruiseControlMetricDef.resourceToMetricId(NW_OUT)] = networkOutBoundUsage;
values[KafkaCruiseControlMetricDef.resourceToMetricId(DISK)] = diskUsage;
AggregatedMetricValues aggregateMetricValues = new AggregatedMetricValues();
for (Resource r : Resource.cachedValues()) {
int metricId = KafkaCruiseControlMetricDef.resourceToMetricId(r);
MetricValues metricValues = new MetricValues(1);
metricValues.set(0, values[metricId]);
aggregateMetricValues.add(metricId, metricValues);
}
return aggregateMetricValues;
}
use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.
the class DeterministicClusterTest method data.
/**
* Populate parameters for the {@link OptimizationVerifier}. All brokers are alive.
*
* @return Parameters for the {@link OptimizationVerifier}.
*/
@Parameterized.Parameters
public static Collection<Object[]> data() {
Collection<Object[]> p = new ArrayList<>();
Map<Integer, String> goalNameByPriority = new HashMap<>();
goalNameByPriority.put(1, RackAwareGoal.class.getName());
goalNameByPriority.put(2, ReplicaCapacityGoal.class.getName());
goalNameByPriority.put(3, DiskCapacityGoal.class.getName());
goalNameByPriority.put(4, NetworkInboundCapacityGoal.class.getName());
goalNameByPriority.put(5, NetworkOutboundCapacityGoal.class.getName());
goalNameByPriority.put(6, CpuCapacityGoal.class.getName());
goalNameByPriority.put(7, ReplicaDistributionGoal.class.getName());
goalNameByPriority.put(8, PotentialNwOutGoal.class.getName());
goalNameByPriority.put(9, DiskUsageDistributionGoal.class.getName());
goalNameByPriority.put(10, NetworkInboundUsageDistributionGoal.class.getName());
goalNameByPriority.put(11, NetworkOutboundUsageDistributionGoal.class.getName());
goalNameByPriority.put(12, CpuUsageDistributionGoal.class.getName());
goalNameByPriority.put(13, TopicReplicaDistributionGoal.class.getName());
goalNameByPriority.put(14, PreferredLeaderElectionGoal.class.getName());
goalNameByPriority.put(15, LeaderBytesInDistributionGoal.class.getName());
Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
props.setProperty(KafkaCruiseControlConfig.MAX_REPLICAS_PER_BROKER_CONFIG, Long.toString(6L));
BalancingConstraint balancingConstraint = new BalancingConstraint(new KafkaCruiseControlConfig(props));
List<OptimizationVerifier.Verification> verifications = Arrays.asList(NEW_BROKERS, DEAD_BROKERS, REGRESSION);
// ----------##TEST: BALANCE PERCENTAGES.
balancingConstraint.setCapacityThreshold(TestConstants.MEDIUM_CAPACITY_THRESHOLD);
List<Double> balancePercentages = new ArrayList<>();
balancePercentages.add(TestConstants.HIGH_BALANCE_PERCENTAGE);
balancePercentages.add(TestConstants.MEDIUM_BALANCE_PERCENTAGE);
balancePercentages.add(TestConstants.LOW_BALANCE_PERCENTAGE);
// -- TEST DECK #1: SMALL CLUSTER.
for (Double balancePercentage : balancePercentages) {
balancingConstraint.setResourceBalancePercentage(balancePercentage);
p.add(params(balancingConstraint, DeterministicCluster.smallClusterModel(TestConstants.BROKER_CAPACITY), goalNameByPriority, verifications, null));
}
// -- TEST DECK #2: MEDIUM CLUSTER.
for (Double balancePercentage : balancePercentages) {
balancingConstraint.setResourceBalancePercentage(balancePercentage);
p.add(params(balancingConstraint, DeterministicCluster.mediumClusterModel(TestConstants.BROKER_CAPACITY), goalNameByPriority, verifications, null));
}
// ----------##TEST: CAPACITY THRESHOLD.
balancingConstraint.setResourceBalancePercentage(TestConstants.MEDIUM_BALANCE_PERCENTAGE);
List<Double> capacityThresholds = new ArrayList<>();
capacityThresholds.add(TestConstants.HIGH_CAPACITY_THRESHOLD);
capacityThresholds.add(TestConstants.MEDIUM_CAPACITY_THRESHOLD);
capacityThresholds.add(TestConstants.LOW_CAPACITY_THRESHOLD);
// -- TEST DECK #3: SMALL CLUSTER.
for (Double capacityThreshold : capacityThresholds) {
balancingConstraint.setCapacityThreshold(capacityThreshold);
p.add(params(balancingConstraint, DeterministicCluster.smallClusterModel(TestConstants.BROKER_CAPACITY), goalNameByPriority, verifications, null));
}
// -- TEST DECK #4: MEDIUM CLUSTER.
for (Double capacityThreshold : capacityThresholds) {
balancingConstraint.setCapacityThreshold(capacityThreshold);
p.add(params(balancingConstraint, DeterministicCluster.mediumClusterModel(TestConstants.BROKER_CAPACITY), goalNameByPriority, verifications, null));
}
// ----------##TEST: BROKER CAPACITY.
List<Double> brokerCapacities = new ArrayList<>();
brokerCapacities.add(TestConstants.LARGE_BROKER_CAPACITY);
brokerCapacities.add(TestConstants.MEDIUM_BROKER_CAPACITY);
brokerCapacities.add(TestConstants.SMALL_BROKER_CAPACITY);
// -- TEST DECK #5: SMALL AND MEDIUM CLUSTERS.
for (Double capacity : brokerCapacities) {
Map<Resource, Double> testBrokerCapacity = new HashMap<>();
testBrokerCapacity.put(Resource.CPU, capacity);
testBrokerCapacity.put(Resource.DISK, capacity);
testBrokerCapacity.put(Resource.NW_IN, capacity);
testBrokerCapacity.put(Resource.NW_OUT, capacity);
p.add(params(balancingConstraint, DeterministicCluster.smallClusterModel(testBrokerCapacity), goalNameByPriority, verifications, null));
p.add(params(balancingConstraint, DeterministicCluster.mediumClusterModel(testBrokerCapacity), goalNameByPriority, verifications, null));
}
Map<Integer, String> kafkaAssignerGoals = new HashMap<>();
kafkaAssignerGoals.put(0, KafkaAssignerEvenRackAwareGoal.class.getName());
kafkaAssignerGoals.put(1, KafkaAssignerDiskUsageDistributionGoal.class.getName());
List<OptimizationVerifier.Verification> kafkaAssignerVerifications = Arrays.asList(DEAD_BROKERS, REGRESSION);
// Small cluster.
p.add(params(balancingConstraint, DeterministicCluster.smallClusterModel(TestConstants.BROKER_CAPACITY), kafkaAssignerGoals, kafkaAssignerVerifications, null));
// Medium cluster.
p.add(params(balancingConstraint, DeterministicCluster.mediumClusterModel(TestConstants.BROKER_CAPACITY), kafkaAssignerGoals, kafkaAssignerVerifications, null));
// Rack-aware satisfiable.
p.add(params(balancingConstraint, DeterministicCluster.rackAwareSatisfiable(), kafkaAssignerGoals, kafkaAssignerVerifications, null));
// Rack-aware unsatisfiable.
p.add(params(balancingConstraint, DeterministicCluster.rackAwareUnsatisfiable(), kafkaAssignerGoals, kafkaAssignerVerifications, OptimizationFailureException.class));
return p;
}
use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.
the class RandomClusterTest method testNewBrokers.
/**
* This test first creates a random cluster, balance it. Then add two new brokers, balance the cluster again.
*/
public void testNewBrokers() throws Exception {
ClusterModel clusterModel = rebalance();
ClusterModel clusterWithNewBroker = new ClusterModel(new ModelGeneration(0, 0L), 1.0);
for (Broker b : clusterModel.brokers()) {
clusterWithNewBroker.createRack(b.rack().id());
Map<Resource, Double> brokerCapacity = new HashMap<>();
for (Resource r : Resource.cachedValues()) {
brokerCapacity.put(r, b.capacityFor(r));
}
clusterWithNewBroker.createBroker(b.rack().id(), Integer.toString(b.id()), b.id(), brokerCapacity);
}
for (Map.Entry<String, List<Partition>> entry : clusterModel.getPartitionsByTopic().entrySet()) {
for (Partition p : entry.getValue()) {
int index = 0;
for (Replica r : p.replicas()) {
clusterWithNewBroker.createReplica(r.broker().rack().id(), r.broker().id(), p.topicPartition(), index++, r.isLeader());
}
}
}
for (Broker b : clusterModel.brokers()) {
for (Replica replica : b.replicas()) {
AggregatedMetricValues aggregatedMetricValues = clusterModel.broker(b.id()).replica(replica.topicPartition()).load().loadByWindows();
clusterWithNewBroker.setReplicaLoad(b.rack().id(), b.id(), replica.topicPartition(), aggregatedMetricValues, clusterModel.load().windows());
}
}
for (int i = 1; i < 3; i++) {
clusterWithNewBroker.createBroker(Integer.toString(i), Integer.toString(i + clusterModel.brokers().size() - 1), i + clusterModel.brokers().size() - 1, TestConstants.BROKER_CAPACITY);
clusterWithNewBroker.setBrokerState(i + clusterModel.brokers().size() - 1, Broker.State.NEW);
}
assertTrue("Random Cluster Test failed to improve the existing state with new brokers.", OptimizationVerifier.executeGoalsFor(_balancingConstraint, clusterWithNewBroker, _goalNameByPriority, _verifications));
}
use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.
the class KafkaMetricSampleAggregatorTest method testExcludeInvalidMetricSample.
@Test
public void testExcludeInvalidMetricSample() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
MetricDef metricDef = KafkaCruiseControlMetricDef.metricDef();
populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
// Set the leader to be node 1, which is different from the leader in the metadata.
PartitionMetricSample sampleWithDifferentLeader = new PartitionMetricSample(1, TP);
sampleWithDifferentLeader.record(metricDef.metricInfo(DISK_USAGE.name()), 10000);
sampleWithDifferentLeader.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_IN.name()), 10000);
sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_OUT.name()), 10000);
sampleWithDifferentLeader.close(0);
// Only populate the CPU metric
PartitionMetricSample incompletePartitionMetricSample = new PartitionMetricSample(0, TP);
incompletePartitionMetricSample.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
incompletePartitionMetricSample.close(0);
metricSampleAggregator.addSample(sampleWithDifferentLeader);
metricSampleAggregator.addSample(incompletePartitionMetricSample);
// Check the snapshot value and make sure the metric samples above are excluded.
Map<PartitionEntity, ValuesAndExtrapolations> snapshotsForPartition = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress()).valuesAndExtrapolations();
ValuesAndExtrapolations snapshots = snapshotsForPartition.get(PE);
for (Resource resource : Resource.values()) {
int metricId = KafkaCruiseControlMetricDef.resourceToMetricId(resource);
double expectedValue = resource == Resource.DISK ? MIN_SAMPLES_PER_WINDOW - 1 : (MIN_SAMPLES_PER_WINDOW - 1) / 2.0;
assertEquals("The utilization for " + resource + " should be " + expectedValue, expectedValue, snapshots.metricValues().valuesFor(metricId).get(NUM_WINDOWS - 1), 0);
}
}
use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.
the class Rack method setBrokerState.
/**
* Set the broker state and update the capacity
*/
void setBrokerState(int brokerId, Broker.State newState) {
// Broker is dead
Broker broker = broker(brokerId);
broker.host().setBrokerState(brokerId, newState);
for (Resource r : Resource.cachedValues()) {
double capacity = 0;
for (Host h : _hosts.values()) {
capacity += h.capacityFor(r);
}
_rackCapacity[r.id()] = capacity;
}
}
Aggregations