use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class LoadMonitorTest method testStateWithOnlyActiveSnapshotWindow.
@Test
public void testStateWithOnlyActiveSnapshotWindow() {
TestContext context = prepareContext();
LoadMonitor loadMonitor = context.loadmonitor();
KafkaMetricSampleAggregator aggregator = context.aggregator();
// populate the metrics aggregator.
// four samples for each partition
CruiseControlUnitTestUtils.populateSampleAggregator(1, 4, aggregator, PE_T0P0, 0, WINDOW_MS, METRIC_DEF);
CruiseControlUnitTestUtils.populateSampleAggregator(1, 4, aggregator, PE_T0P1, 0, WINDOW_MS, METRIC_DEF);
CruiseControlUnitTestUtils.populateSampleAggregator(1, 4, aggregator, PE_T1P0, 0, WINDOW_MS, METRIC_DEF);
CruiseControlUnitTestUtils.populateSampleAggregator(1, 4, aggregator, PE_T1P1, 0, WINDOW_MS, METRIC_DEF);
LoadMonitorState state = loadMonitor.state(new OperationProgress());
// The load monitor only has an active window. There is no stable window.
assertEquals(0, state.numValidPartitions());
assertEquals(0, state.numValidWindows());
assertTrue(state.monitoredWindows().isEmpty());
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class OfflineProposalGenerator method main.
public static void main(String[] argv) throws Exception {
// TODO: probably need to save this in the original model file
Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(props);
ModelUtils.init(config);
ModelParameters.init(config);
BalancingConstraint balancingConstraint = new BalancingConstraint(config);
long start = System.currentTimeMillis();
ClusterModel clusterModel = clusterModelFromFile(argv[0]);
long end = System.currentTimeMillis();
double duration = (end - start) / 1000.0;
System.out.println("Model loaded in " + duration + "s.");
ClusterModelStats origStats = clusterModel.getClusterStats(balancingConstraint);
String loadBeforeOptimization = clusterModel.brokerStats().toString();
// Instantiate the components.
GoalOptimizer goalOptimizer = new GoalOptimizer(config, null, new SystemTime(), new MetricRegistry());
start = System.currentTimeMillis();
GoalOptimizer.OptimizerResult optimizerResult = goalOptimizer.optimizations(clusterModel, new OperationProgress());
end = System.currentTimeMillis();
duration = (end - start) / 1000.0;
String loadAfterOptimization = clusterModel.brokerStats().toString();
System.out.println("Optimize goals in " + duration + "s.");
System.out.println(optimizerResult.goalProposals().size());
System.out.println(loadBeforeOptimization);
System.out.println(loadAfterOptimization);
ClusterModelStats optimizedStats = clusterModel.getClusterStats(balancingConstraint);
double[] testStatistics = AnalyzerUtils.testDifference(origStats.utilizationMatrix(), optimizedStats.utilizationMatrix());
System.out.println(Arrays.stream(RawAndDerivedResource.values()).map(x -> x.toString()).collect(Collectors.joining(", ")));
System.out.println(Arrays.stream(testStatistics).boxed().map(pValue -> Double.toString(pValue)).collect(Collectors.joining(", ")));
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class OptimizationVerifier method executeGoalsFor.
/**
* Execute given goals in the given cluster enforcing the given constraint. Return pass / fail status of a test.
* A test fails if:
* 1) Rebalance: During the optimization process, optimization of a goal leads to a worse cluster state (in terms of
* the requirements of the same goal) than the cluster state just before starting the optimization.
* 2) Self Healing: There are replicas on dead brokers after self healing.
* 3) Adding a new broker causes the replicas to move among old brokers.
*
* @param constraint Balancing constraint for the given cluster.
* @param clusterModel The state of the cluster.
* @param goalNameByPriority Name of goals by the order of execution priority.
* @param excludedTopics The excluded topics.
* @param verifications The verifications to make after the optimization.
* @return Pass / fail status of a test.
*/
static boolean executeGoalsFor(BalancingConstraint constraint, ClusterModel clusterModel, Map<Integer, String> goalNameByPriority, Collection<String> excludedTopics, List<Verification> verifications) throws Exception {
// Get the initial stats from the cluster.
ClusterModelStats preOptimizedStats = clusterModel.getClusterStats(constraint);
// Set goals by their priority.
SortedMap<Integer, Goal> goalByPriority = new TreeMap<>();
for (Map.Entry<Integer, String> goalEntry : goalNameByPriority.entrySet()) {
Integer priority = goalEntry.getKey();
String goalClassName = goalEntry.getValue();
Class<? extends Goal> goalClass = (Class<? extends Goal>) Class.forName(goalClassName);
try {
Constructor<? extends Goal> constructor = goalClass.getDeclaredConstructor(BalancingConstraint.class);
constructor.setAccessible(true);
goalByPriority.put(priority, constructor.newInstance(constraint));
} catch (NoSuchMethodException badConstructor) {
// Try default constructor
goalByPriority.put(priority, goalClass.newInstance());
}
}
// Generate the goalOptimizer and optimize given goals.
long startTime = System.currentTimeMillis();
Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
StringJoiner stringJoiner = new StringJoiner(",");
excludedTopics.forEach(stringJoiner::add);
props.setProperty(KafkaCruiseControlConfig.TOPICS_EXCLUDED_FROM_PARTITION_MOVEMENT_CONFIG, stringJoiner.toString());
GoalOptimizer goalOptimizer = new GoalOptimizer(new KafkaCruiseControlConfig(constraint.setProps(props)), null, new SystemTime(), new MetricRegistry());
GoalOptimizer.OptimizerResult optimizerResult = goalOptimizer.optimizations(clusterModel, goalByPriority, new OperationProgress());
LOG.trace("Took {} ms to execute {} to generate {} proposals.", System.currentTimeMillis() - startTime, goalByPriority, optimizerResult.goalProposals().size());
for (Verification verification : verifications) {
switch(verification) {
case GOAL_VIOLATION:
if (!verifyGoalViolations(optimizerResult)) {
return false;
}
break;
case NEW_BROKERS:
if (!clusterModel.newBrokers().isEmpty() && !verifyNewBrokers(clusterModel, constraint)) {
return false;
}
break;
case DEAD_BROKERS:
if (!clusterModel.deadBrokers().isEmpty() && !verifyDeadBrokers(clusterModel)) {
return false;
}
break;
case REGRESSION:
if (clusterModel.selfHealingEligibleReplicas().isEmpty() && !verifyRegression(optimizerResult, preOptimizedStats)) {
return false;
}
break;
default:
throw new IllegalStateException("Invalid verification " + verification);
}
}
return true;
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class KafkaMetricSampleAggregatorTest method testNotEnoughSnapshots.
@Test
public void testNotEnoughSnapshots() {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
try {
// Only 4 snapshots has smaller timestamp than the timestamp we passed in.
ModelCompletenessRequirements requirements = new ModelCompletenessRequirements(NUM_WINDOWS, 0.0, false);
metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), -1L, (NUM_WINDOWS - 1) * WINDOW_MS - 1, requirements, new OperationProgress());
fail("Should throw NotEnoughValidWindowsException");
} catch (NotEnoughValidWindowsException nse) {
// let it go
}
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class KafkaMetricSampleAggregatorTest method testExcludeInvalidMetricSample.
@Test
public void testExcludeInvalidMetricSample() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
MetricDef metricDef = KafkaCruiseControlMetricDef.metricDef();
populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
// Set the leader to be node 1, which is different from the leader in the metadata.
PartitionMetricSample sampleWithDifferentLeader = new PartitionMetricSample(1, TP);
sampleWithDifferentLeader.record(metricDef.metricInfo(DISK_USAGE.name()), 10000);
sampleWithDifferentLeader.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_IN.name()), 10000);
sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_OUT.name()), 10000);
sampleWithDifferentLeader.close(0);
// Only populate the CPU metric
PartitionMetricSample incompletePartitionMetricSample = new PartitionMetricSample(0, TP);
incompletePartitionMetricSample.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
incompletePartitionMetricSample.close(0);
metricSampleAggregator.addSample(sampleWithDifferentLeader);
metricSampleAggregator.addSample(incompletePartitionMetricSample);
// Check the snapshot value and make sure the metric samples above are excluded.
Map<PartitionEntity, ValuesAndExtrapolations> snapshotsForPartition = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress()).valuesAndExtrapolations();
ValuesAndExtrapolations snapshots = snapshotsForPartition.get(PE);
for (Resource resource : Resource.values()) {
int metricId = KafkaCruiseControlMetricDef.resourceToMetricId(resource);
double expectedValue = resource == Resource.DISK ? MIN_SAMPLES_PER_WINDOW - 1 : (MIN_SAMPLES_PER_WINDOW - 1) / 2.0;
assertEquals("The utilization for " + resource + " should be " + expectedValue, expectedValue, snapshots.metricValues().valuesFor(metricId).get(NUM_WINDOWS - 1), 0);
}
}
Aggregations