use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class KafkaMetricSampleAggregatorTest method testTooManyFlaws.
@Test
public void testTooManyFlaws() throws NotEnoughValidWindowsException {
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
Metadata metadata = getMetadata(Collections.singleton(TP));
KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
// Only give two samples to the aggregator.
CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 2, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator, PE, 3, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress());
assertTrue(result.valuesAndExtrapolations().isEmpty());
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class LoadMonitorTest method testClusterModelWithInvalidPartitionAndInsufficientSnapshotWindows.
// Not enough snapshot windows and some partitions are missing from all snapshot windows.
@Test
public void testClusterModelWithInvalidPartitionAndInsufficientSnapshotWindows() throws NotEnoughValidWindowsException {
TestContext context = prepareContext();
LoadMonitor loadMonitor = context.loadmonitor();
KafkaMetricSampleAggregator aggregator = context.aggregator();
ModelCompletenessRequirements requirements1 = new ModelCompletenessRequirements(1, 1.0, false);
ModelCompletenessRequirements requirements2 = new ModelCompletenessRequirements(1, 0.5, false);
ModelCompletenessRequirements requirements3 = new ModelCompletenessRequirements(2, 1.0, false);
ModelCompletenessRequirements requirements4 = new ModelCompletenessRequirements(2, 0.5, false);
// populate the metrics aggregator.
// two samples for each partition except T1P1
CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T0P0, 0, WINDOW_MS, METRIC_DEF);
CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T0P1, 0, WINDOW_MS, METRIC_DEF);
CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T1P0, 0, WINDOW_MS, METRIC_DEF);
try {
loadMonitor.clusterModel(-1, Long.MAX_VALUE, requirements1, new OperationProgress());
fail("Should have thrown NotEnoughValidWindowsException.");
} catch (NotEnoughValidWindowsException nevwe) {
// let it go
}
ClusterModel clusterModel = loadMonitor.clusterModel(-1L, Long.MAX_VALUE, requirements2, new OperationProgress());
assertNull(clusterModel.partition(T1P0));
assertNull(clusterModel.partition(T1P1));
assertEquals(1, clusterModel.partition(T0P0).leader().load().numWindows());
assertEquals(3, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.DISK), 0.0);
assertEquals(1.5, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.CPU), 0.0);
assertEquals(1.5, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.NW_IN), 0.0);
assertEquals(1.5, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.NW_OUT), 0.0);
try {
loadMonitor.clusterModel(-1L, Long.MAX_VALUE, requirements3, new OperationProgress());
fail("Should have thrown NotEnoughValidWindowsException.");
} catch (NotEnoughValidWindowsException nevwe) {
// let it go
}
try {
loadMonitor.clusterModel(-1L, Long.MAX_VALUE, requirements4, new OperationProgress());
fail("Should have thrown NotEnoughValidWindowsException.");
} catch (NotEnoughValidWindowsException nevwe) {
// let it go
}
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class LoadMonitorTest method prepareContext.
private TestContext prepareContext() {
// Create mock metadata client.
Metadata metadata = getMetadata(Arrays.asList(T0P0, T0P1, T1P0, T1P1));
MetadataClient mockMetadataClient = EasyMock.mock(MetadataClient.class);
EasyMock.expect(mockMetadataClient.cluster()).andReturn(metadata.fetch()).anyTimes();
EasyMock.expect(mockMetadataClient.clusterAndGeneration()).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
EasyMock.expect(mockMetadataClient.metadata()).andReturn(metadata).anyTimes();
EasyMock.expect(mockMetadataClient.refreshMetadata()).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
EasyMock.expect(mockMetadataClient.refreshMetadata(anyLong())).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
EasyMock.replay(mockMetadataClient);
// create load monitor.
Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
props.put(KafkaCruiseControlConfig.NUM_METRICS_WINDOWS_CONFIG, Integer.toString(NUM_WINDOWS));
props.put(KafkaCruiseControlConfig.MIN_SAMPLES_PER_METRICS_WINDOW_CONFIG, Integer.toString(MIN_SAMPLES_PER_WINDOW));
props.put(KafkaCruiseControlConfig.METRICS_WINDOW_MS_CONFIG, Long.toString(WINDOW_MS));
props.put("cleanup.policy", DEFAULT_CLEANUP_POLICY);
props.put(KafkaCruiseControlConfig.SAMPLE_STORE_CLASS_CONFIG, NoopSampleStore.class.getName());
KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(props);
LoadMonitor loadMonitor = new LoadMonitor(config, mockMetadataClient, _time, new MetricRegistry(), METRIC_DEF);
KafkaMetricSampleAggregator aggregator = loadMonitor.aggregator();
ModelParameters.init(config);
loadMonitor.startUp();
while (loadMonitor.state(new OperationProgress()).state() != LoadMonitorTaskRunner.LoadMonitorTaskRunnerState.RUNNING) {
try {
Thread.sleep(1);
} catch (InterruptedException e) {
// let it go.
}
}
return new TestContext(loadMonitor, aggregator, config, metadata);
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class LoadMonitor method getMonitoredPartitionsPercentage.
private double getMonitoredPartitionsPercentage() {
MetadataClient.ClusterAndGeneration clusterAndGeneration = _metadataClient.refreshMetadata();
Cluster kafkaCluster = clusterAndGeneration.cluster();
MetricSampleAggregationResult<String, PartitionEntity> metricSampleAggregationResult;
try {
metricSampleAggregationResult = _metricSampleAggregator.aggregate(clusterAndGeneration, System.currentTimeMillis(), new OperationProgress());
} catch (NotEnoughValidWindowsException e) {
return 0.0;
}
Map<PartitionEntity, ValuesAndExtrapolations> partitionLoads = metricSampleAggregationResult.valuesAndExtrapolations();
AtomicInteger numPartitionsWithExtrapolations = new AtomicInteger(0);
partitionLoads.values().forEach(valuesAndExtrapolations -> {
if (!valuesAndExtrapolations.extrapolations().isEmpty()) {
numPartitionsWithExtrapolations.incrementAndGet();
}
});
_numPartitionsWithExtrapolations = numPartitionsWithExtrapolations.get();
int totalNumPartitions = MonitorUtils.totalNumPartitions(kafkaCluster);
return totalNumPartitions > 0 ? metricSampleAggregationResult.completeness().validEntityRatio() : 0.0;
}
use of com.linkedin.kafka.cruisecontrol.async.progress.OperationProgress in project cruise-control by linkedin.
the class GoalViolationDetector method run.
@Override
public void run() {
long now = _time.milliseconds();
if (_loadMonitor.clusterModelGeneration().equals(_lastCheckedModelGeneration)) {
LOG.debug("Skipping goal violation detection because the model generation hasn't changed. Current model generation {}", _loadMonitor.clusterModelGeneration());
return;
}
AutoCloseable clusterModelSemaphore = null;
try {
LoadMonitorTaskRunner.LoadMonitorTaskRunnerState loadMonitorTaskRunnerState = _loadMonitor.taskRunnerState();
if (!ViolationUtils.isLoadMonitorReady(loadMonitorTaskRunnerState)) {
LOG.info("Skipping goal violation detection because load monitor is in {} state.", loadMonitorTaskRunnerState);
return;
}
GoalViolations goalViolations = new GoalViolations();
boolean newModelNeeded = true;
ClusterModel clusterModel = null;
for (Map.Entry<Integer, Goal> entry : _goals.entrySet()) {
Goal goal = entry.getValue();
if (_loadMonitor.meetCompletenessRequirements(goal.clusterModelCompletenessRequirements())) {
LOG.debug("Detecting if {} is violated.", entry.getValue().name());
// Because the model generation could be slow, We only get new cluster model if needed.
if (newModelNeeded) {
if (clusterModelSemaphore != null) {
clusterModelSemaphore.close();
}
clusterModelSemaphore = _loadMonitor.acquireForModelGeneration(new OperationProgress());
// Make cluster model null before generating a new cluster model so the current one can be GCed.
clusterModel = null;
clusterModel = _loadMonitor.clusterModel(now, goal.clusterModelCompletenessRequirements(), new OperationProgress());
}
int priority = entry.getKey();
newModelNeeded = optimizeForGoal(clusterModel, priority, goal, goalViolations);
} else {
LOG.debug("Skipping goal violation detection for {} because load completeness requirement is not met.", goal);
}
}
if (clusterModel != null) {
_lastCheckedModelGeneration = clusterModel.generation();
}
if (!goalViolations.violations().isEmpty()) {
_anomalies.add(goalViolations);
}
} catch (NotEnoughValidWindowsException nevwe) {
LOG.debug("Skipping goal violation detection because there are not enough valid windows.");
} catch (KafkaCruiseControlException kcce) {
LOG.warn("Goal violation detector received exception", kcce);
} catch (Exception e) {
LOG.error("Unexpected exception", e);
} finally {
if (clusterModelSemaphore != null) {
try {
clusterModelSemaphore.close();
} catch (Exception e) {
LOG.error("Received exception when closing auto closable semaphore", e);
}
}
LOG.debug("Goal violation detection finished.");
}
}
Aggregations