Examples with Anomaly com.linkedin.cruisecontrol.detector.Anomaly used on opensource projects

Example 1 with Anomaly

use of com.linkedin.cruisecontrol.detector.Anomaly in project cruise-control by linkedin.

the class AnomalyDetectorManagerTest method testShutdown.

@Test
public void testShutdown() throws InterruptedException {
    PriorityBlockingQueue<Anomaly> anomalies = new PriorityBlockingQueue<>(ANOMALY_DETECTOR_INITIAL_QUEUE_SIZE, anomalyComparator());
    AnomalyNotifier mockAnomalyNotifier = EasyMock.createNiceMock(AnomalyNotifier.class);
    BrokerFailureDetector mockBrokerFailureDetector = EasyMock.createNiceMock(BrokerFailureDetector.class);
    GoalViolationDetector mockGoalViolationDetector = EasyMock.createNiceMock(GoalViolationDetector.class);
    MetricAnomalyDetector mockMetricAnomalyDetector = EasyMock.createNiceMock(MetricAnomalyDetector.class);
    TopicAnomalyDetector mockTopicAnomalyDetector = EasyMock.createNiceMock(TopicAnomalyDetector.class);
    MaintenanceEventDetector mockMaintenanceEventDetector = EasyMock.createNiceMock(MaintenanceEventDetector.class);
    DiskFailureDetector mockDiskFailureDetector = EasyMock.createNiceMock(DiskFailureDetector.class);
    KafkaCruiseControl mockKafkaCruiseControl = EasyMock.createNiceMock(KafkaCruiseControl.class);
    ScheduledExecutorService detectorScheduler = Executors.newScheduledThreadPool(2, new KafkaCruiseControlThreadFactory("AnomalyDetector", false, null));
    AnomalyDetectorManager anomalyDetectorManager = new AnomalyDetectorManager(anomalies, MOCK_ANOMALY_DETECTION_INTERVAL_MS, mockKafkaCruiseControl, mockAnomalyNotifier, mockGoalViolationDetector, mockBrokerFailureDetector, mockMetricAnomalyDetector, mockDiskFailureDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, detectorScheduler);
    anomalyDetectorManager.shutdown();
    Thread t = new Thread(anomalyDetectorManager::shutdown);
    t.start();
    t.join(TimeUnit.SECONDS.toMillis(30));
    assertEquals(0, anomalyDetectorManager.numSelfHealingStarted());
    assertEquals(0, anomalyDetectorManager.numCheckedWithDelay());
    assertTrue(detectorScheduler.isTerminated());
}

Example 2 with Anomaly

use of com.linkedin.cruisecontrol.detector.Anomaly in project cruise-control by linkedin.

the class AnomalyDetectorManagerTest method testExecutionInProgress.

@Test
public void testExecutionInProgress() throws InterruptedException {
    PriorityBlockingQueue<Anomaly> anomalies = new PriorityBlockingQueue<>(ANOMALY_DETECTOR_INITIAL_QUEUE_SIZE, anomalyComparator());
    AnomalyNotifier mockAnomalyNotifier = EasyMock.mock(AnomalyNotifier.class);
    BrokerFailureDetector mockBrokerFailureDetector = EasyMock.createNiceMock(BrokerFailureDetector.class);
    GoalViolationDetector mockGoalViolationDetector = EasyMock.createNiceMock(GoalViolationDetector.class);
    MetricAnomalyDetector mockMetricAnomalyDetector = EasyMock.createNiceMock(MetricAnomalyDetector.class);
    TopicAnomalyDetector mockTopicAnomalyDetector = EasyMock.createNiceMock(TopicAnomalyDetector.class);
    MaintenanceEventDetector mockMaintenanceEventDetector = EasyMock.createNiceMock(MaintenanceEventDetector.class);
    DiskFailureDetector mockDiskFailureDetector = EasyMock.createNiceMock(DiskFailureDetector.class);
    ScheduledExecutorService mockDetectorScheduler = EasyMock.mock(ScheduledExecutorService.class);
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    KafkaCruiseControl mockKafkaCruiseControl = EasyMock.mock(KafkaCruiseControl.class);
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    KafkaCruiseControlConfig kafkaCruiseControlConfig = new KafkaCruiseControlConfig(props);
    EasyMock.expect(mockKafkaCruiseControl.config()).andReturn(kafkaCruiseControlConfig).times(2);
    startRunnableDetectors(mockDetectorScheduler, mockGoalViolationDetector, mockMetricAnomalyDetector, mockDiskFailureDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, executorService);
    shutdownDetector(mockDetectorScheduler, executorService);
    EasyMock.expect(mockAnomalyNotifier.selfHealingEnabledRatio()).andReturn(MOCK_SELF_HEALING_ENABLED_RATIO);
    // The following state are used to test the delayed check when executor is idle.
    EasyMock.expect(mockKafkaCruiseControl.executionState()).andReturn(ExecutorState.State.INTER_BROKER_REPLICA_MOVEMENT_TASK_IN_PROGRESS);
    replayCommonMocks(mockAnomalyNotifier, mockBrokerFailureDetector, mockGoalViolationDetector, mockMetricAnomalyDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, mockDiskFailureDetector, mockDetectorScheduler, mockKafkaCruiseControl);
    AnomalyDetectorManager anomalyDetectorManager = new AnomalyDetectorManager(anomalies, MOCK_ANOMALY_DETECTION_INTERVAL_MS, mockKafkaCruiseControl, mockAnomalyNotifier, mockGoalViolationDetector, mockBrokerFailureDetector, mockMetricAnomalyDetector, mockDiskFailureDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, mockDetectorScheduler);
    try {
        anomalyDetectorManager.startDetection();
        Map<String, Object> parameterConfigOverrides = Map.of(KAFKA_CRUISE_CONTROL_OBJECT_CONFIG, mockKafkaCruiseControl, ANOMALY_DETECTION_TIME_MS_OBJECT_CONFIG, 100L);
        anomalies.add(kafkaCruiseControlConfig.getConfiguredInstance(AnomalyDetectorConfig.GOAL_VIOLATIONS_CLASS_CONFIG, GoalViolations.class, parameterConfigOverrides));
        while (!anomalies.isEmpty()) {
        // Just wait for the anomalies to be drained.
        }
        anomalyDetectorManager.shutdown();
        assertEquals(0, anomalyDetectorManager.numSelfHealingStarted());
        assertEquals(0, anomalyDetectorManager.numCheckedWithDelay());
        assertTrue(executorService.awaitTermination(MOCK_ANOMALY_DETECTOR_SHUTDOWN_MS, TimeUnit.MILLISECONDS));
        AnomalyDetectorState anomalyDetectorState = anomalyDetectorManager.anomalyDetectorState();
        assertEquals((long) anomalyDetectorState.metrics().get(NUM_SELF_HEALING_STARTED), 0L);
        assertEquals(anomalyDetectorState.recentAnomaliesByType().get(KafkaAnomalyType.BROKER_FAILURE).size(), 0);
        assertEquals(anomalyDetectorState.recentAnomaliesByType().get(KafkaAnomalyType.GOAL_VIOLATION).size(), 1);
        assertEquals(anomalyDetectorState.recentAnomaliesByType().get(KafkaAnomalyType.METRIC_ANOMALY).size(), 0);
    } finally {
        executorService.shutdown();
    }
    EasyMock.verify(mockAnomalyNotifier, mockDetectorScheduler, mockKafkaCruiseControl, mockBrokerFailureDetector, mockGoalViolationDetector, mockMetricAnomalyDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, mockDiskFailureDetector);
}

Example 3 with Anomaly

use of com.linkedin.cruisecontrol.detector.Anomaly in project cruise-control by linkedin.

the class AnomalyDetectorManagerTest method testFixAnomaly.

private void testFixAnomaly(AnomalyType anomalyType) throws InterruptedException, KafkaCruiseControlException, NotEnoughValidWindowsException, TimeoutException {
    PriorityBlockingQueue<Anomaly> anomalies = new PriorityBlockingQueue<>(ANOMALY_DETECTOR_INITIAL_QUEUE_SIZE, anomalyComparator());
    AnomalyNotifier mockAnomalyNotifier = EasyMock.mock(AnomalyNotifier.class);
    BrokerFailureDetector mockBrokerFailureDetector = EasyMock.createNiceMock(BrokerFailureDetector.class);
    GoalViolationDetector mockGoalViolationDetector = EasyMock.createNiceMock(GoalViolationDetector.class);
    MetricAnomalyDetector mockMetricAnomalyDetector = EasyMock.createNiceMock(MetricAnomalyDetector.class);
    TopicAnomalyDetector mockTopicAnomalyDetector = EasyMock.createNiceMock(TopicAnomalyDetector.class);
    MaintenanceEventDetector mockMaintenanceEventDetector = EasyMock.createNiceMock(MaintenanceEventDetector.class);
    DiskFailureDetector mockDiskFailureDetector = EasyMock.createNiceMock(DiskFailureDetector.class);
    ScheduledExecutorService mockDetectorScheduler = EasyMock.mock(ScheduledExecutorService.class);
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    KafkaCruiseControl mockKafkaCruiseControl = EasyMock.mock(KafkaCruiseControl.class);
    ModelCompletenessRequirements mockModelCompletenessRequirements = EasyMock.mock(ModelCompletenessRequirements.class);
    EasyMock.expect(mockModelCompletenessRequirements.weaker(EasyMock.anyObject())).andReturn(mockModelCompletenessRequirements);
    EasyMock.expect(mockModelCompletenessRequirements.minRequiredNumWindows()).andReturn(0);
    EasyMock.replay(mockModelCompletenessRequirements);
    OptimizerResult mockOptimizerResult = EasyMock.mock(OptimizerResult.class);
    BrokerStats mockBrokerStats = EasyMock.mock(BrokerStats.class);
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    props.setProperty(AnomalyDetectorConfig.METRIC_ANOMALY_CLASS_CONFIG, SlowBrokers.class.getName());
    KafkaCruiseControlConfig kafkaCruiseControlConfig = new KafkaCruiseControlConfig(props);
    EasyMock.expect(mockKafkaCruiseControl.config()).andReturn(kafkaCruiseControlConfig).times(1, 17);
    mockKafkaCruiseControl.sanityCheckDryRun(EasyMock.eq(SELF_HEALING_DRYRUN), EasyMock.eq(false));
    EasyMock.expect(mockKafkaCruiseControl.modelCompletenessRequirements(EasyMock.anyObject())).andReturn(mockModelCompletenessRequirements).times(0, 2);
    EasyMock.expect(mockKafkaCruiseControl.getLoadMonitorTaskRunnerState()).andReturn(LoadMonitorTaskRunner.LoadMonitorTaskRunnerState.RUNNING).times(1, 2);
    startRunnableDetectors(mockDetectorScheduler, mockGoalViolationDetector, mockMetricAnomalyDetector, mockDiskFailureDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, executorService);
    shutdownDetector(mockDetectorScheduler, executorService);
    // The following state are used to test the delayed check when executor is idle.
    EasyMock.expect(mockKafkaCruiseControl.executionState()).andReturn(ExecutorState.State.NO_TASK_IN_PROGRESS);
    EasyMock.expect(mockAnomalyNotifier.selfHealingEnabledRatio()).andReturn(MOCK_SELF_HEALING_ENABLED_RATIO);
    if (anomalyType == KafkaAnomalyType.GOAL_VIOLATION) {
        mockKafkaCruiseControl.sanityCheckDryRun(EasyMock.eq(true), EasyMock.eq(false));
        EasyMock.expect(mockKafkaCruiseControl.ignoreProposalCache(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(SELF_HEALING_EXCLUDED_TOPICS), EasyMock.eq(AnomalyDetectorConfig.DEFAULT_SELF_HEALING_EXCLUDE_RECENT_BROKERS_CONFIG), EasyMock.eq(SELF_HEALING_IGNORE_PROPOSAL_CACHE), EasyMock.eq(true), EasyMock.eq(SELF_HEALING_DESTINATION_BROKER_IDS), EasyMock.eq(SELF_HEALING_IS_REBALANCE_DISK_MODE))).andReturn(false);
        EasyMock.expect(mockKafkaCruiseControl.getProposals(EasyMock.anyObject(), EasyMock.eq(AnomalyDetectorConfig.DEFAULT_ANOMALY_DETECTION_ALLOW_CAPACITY_ESTIMATION_CONFIG))).andReturn(mockOptimizerResult);
        mockKafkaCruiseControl.executeProposals(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(false), EasyMock.eq(SELF_HEALING_CONCURRENT_MOVEMENTS), EasyMock.anyObject(), EasyMock.eq(SELF_HEALING_CONCURRENT_MOVEMENTS), EasyMock.eq(SELF_HEALING_CONCURRENT_MOVEMENTS), EasyMock.eq(SELF_HEALING_EXECUTION_PROGRESS_CHECK_INTERVAL_MS), EasyMock.eq(SELF_HEALING_REPLICA_MOVEMENT_STRATEGY), EasyMock.eq(null), EasyMock.eq(false), EasyMock.anyString(), EasyMock.eq(false));
        EasyMock.expect(mockAnomalyNotifier.onGoalViolation(EasyMock.isA(GoalViolations.class))).andReturn(AnomalyNotificationResult.fix());
    } else if (anomalyType == KafkaAnomalyType.DISK_FAILURE) {
        ClusterModel singleBrokerWithBadDisk = singleBrokerWithBadDisk();
        EasyMock.expect(mockKafkaCruiseControl.clusterModel(EasyMock.anyObject(), EasyMock.eq(true), EasyMock.anyObject())).andReturn(singleBrokerWithBadDisk);
        EasyMock.expect(mockKafkaCruiseControl.dropRecentBrokers(EasyMock.eq(Collections.emptySet()), EasyMock.eq(true))).andReturn(false);
        EasyMock.expect(mockKafkaCruiseControl.dropRecentBrokers(EasyMock.eq(Collections.emptySet()), EasyMock.eq(false))).andReturn(false);
        ExecutorState executorState = ExecutorState.noTaskInProgress(Collections.emptySet(), Collections.emptySet());
        EasyMock.expect(mockKafkaCruiseControl.executorState()).andReturn(executorState).once();
        EasyMock.expect(mockKafkaCruiseControl.excludedTopics(singleBrokerWithBadDisk, SELF_HEALING_EXCLUDED_TOPICS)).andReturn(Collections.emptySet());
        EasyMock.expect(mockKafkaCruiseControl.optimizations(EasyMock.eq(singleBrokerWithBadDisk), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(null), EasyMock.anyObject())).andReturn(mockOptimizerResult);
        mockKafkaCruiseControl.executeProposals(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(false), EasyMock.eq(SELF_HEALING_CONCURRENT_MOVEMENTS), EasyMock.anyObject(), EasyMock.eq(SELF_HEALING_CONCURRENT_MOVEMENTS), EasyMock.eq(SELF_HEALING_CONCURRENT_MOVEMENTS), EasyMock.eq(SELF_HEALING_EXECUTION_PROGRESS_CHECK_INTERVAL_MS), EasyMock.eq(SELF_HEALING_REPLICA_MOVEMENT_STRATEGY), EasyMock.eq(null), EasyMock.eq(false), EasyMock.anyString(), EasyMock.eq(false));
        EasyMock.expect(mockKafkaCruiseControl.acquireForModelGeneration(EasyMock.anyObject())).andReturn(null);
        EasyMock.expect(mockAnomalyNotifier.onDiskFailure(EasyMock.isA(DiskFailures.class))).andReturn(AnomalyNotificationResult.fix());
    } else if (anomalyType == KafkaAnomalyType.METRIC_ANOMALY) {
        ClusterModel smallCluster = smallClusterModel(TestConstants.BROKER_CAPACITY);
        EasyMock.expect(mockKafkaCruiseControl.clusterModel(EasyMock.anyObject(), EasyMock.eq(true), EasyMock.anyObject())).andReturn(smallCluster);
        EasyMock.expect(mockKafkaCruiseControl.kafkaCluster()).andReturn(Cluster.empty());
        EasyMock.expect(mockKafkaCruiseControl.acquireForModelGeneration(EasyMock.anyObject())).andReturn(null);
        mockKafkaCruiseControl.sanityCheckBrokerPresence(EasyMock.anyObject());
        EasyMock.expect(mockKafkaCruiseControl.dropRecentBrokers(EasyMock.eq(Collections.emptySet()), EasyMock.eq(true))).andReturn(false);
        EasyMock.expect(mockKafkaCruiseControl.dropRecentBrokers(EasyMock.eq(Collections.emptySet()), EasyMock.eq(false))).andReturn(false);
        ExecutorState executorState = ExecutorState.noTaskInProgress(Collections.emptySet(), Collections.emptySet());
        EasyMock.expect(mockKafkaCruiseControl.executorState()).andReturn(executorState).once();
        EasyMock.expect(mockKafkaCruiseControl.excludedTopics(smallCluster, SELF_HEALING_EXCLUDED_TOPICS)).andReturn(Collections.emptySet());
        EasyMock.expect(mockKafkaCruiseControl.optimizations(EasyMock.eq(smallCluster), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(null), EasyMock.anyObject())).andReturn(mockOptimizerResult);
        mockKafkaCruiseControl.executeDemotion(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(SELF_HEALING_CONCURRENT_MOVEMENTS), EasyMock.eq(smallCluster.brokers().size()), EasyMock.eq(SELF_HEALING_EXECUTION_PROGRESS_CHECK_INTERVAL_MS), EasyMock.eq(SELF_HEALING_REPLICA_MOVEMENT_STRATEGY), EasyMock.eq(null), EasyMock.eq(false), EasyMock.anyString());
        EasyMock.expect(mockAnomalyNotifier.onMetricAnomaly(EasyMock.isA(SlowBrokers.class))).andReturn(AnomalyNotificationResult.fix());
    } else if (anomalyType == KafkaAnomalyType.TOPIC_ANOMALY) {
        ClusterModel clusterModel = unbalanced();
        EasyMock.expect(mockKafkaCruiseControl.clusterModel(EasyMock.anyObject(), EasyMock.eq(true), EasyMock.anyObject())).andReturn(clusterModel);
        EasyMock.expect(mockKafkaCruiseControl.kafkaCluster()).andReturn(generateClusterFromClusterModel(clusterModel));
        EasyMock.expect(mockKafkaCruiseControl.acquireForModelGeneration(EasyMock.anyObject())).andReturn(null);
        EasyMock.expect(mockKafkaCruiseControl.dropRecentBrokers(EasyMock.eq(Collections.emptySet()), EasyMock.eq(true))).andReturn(false);
        EasyMock.expect(mockKafkaCruiseControl.dropRecentBrokers(EasyMock.eq(Collections.emptySet()), EasyMock.eq(false))).andReturn(false);
        ExecutorState executorState = ExecutorState.noTaskInProgress(Collections.emptySet(), Collections.emptySet());
        EasyMock.expect(mockKafkaCruiseControl.executorState()).andReturn(executorState).once();
        EasyMock.expect(mockKafkaCruiseControl.excludedTopics(clusterModel, SELF_HEALING_EXCLUDED_TOPICS)).andReturn(Collections.emptySet());
        EasyMock.expect(mockKafkaCruiseControl.optimizations(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.anyObject())).andReturn(mockOptimizerResult);
        mockKafkaCruiseControl.executeProposals(EasyMock.anyObject(), EasyMock.eq(Collections.emptySet()), EasyMock.eq(false), EasyMock.eq(SELF_HEALING_CONCURRENT_MOVEMENTS), EasyMock.anyObject(), EasyMock.eq(0), EasyMock.eq(SELF_HEALING_CONCURRENT_MOVEMENTS), EasyMock.eq(SELF_HEALING_EXECUTION_PROGRESS_CHECK_INTERVAL_MS), EasyMock.eq(SELF_HEALING_REPLICA_MOVEMENT_STRATEGY), EasyMock.eq(null), EasyMock.eq(false), EasyMock.anyString(), EasyMock.eq(true));
        EasyMock.expect(mockAnomalyNotifier.onTopicAnomaly(EasyMock.isA(TopicAnomaly.class))).andReturn(AnomalyNotificationResult.fix());
    }
    EasyMock.expect(mockKafkaCruiseControl.meetCompletenessRequirements(Collections.emptyList())).andReturn(true);
    EasyMock.expect(mockDetectorScheduler.schedule(EasyMock.isA(Runnable.class), EasyMock.eq(0L), EasyMock.eq(TimeUnit.MILLISECONDS))).andReturn(null);
    // Set generating proposals for execution.
    mockKafkaCruiseControl.setGeneratingProposalsForExecution(EasyMock.anyObject(), EasyMock.anyObject(), EasyMock.eq(false));
    replayCommonMocks(mockAnomalyNotifier, mockBrokerFailureDetector, mockGoalViolationDetector, mockMetricAnomalyDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, mockDiskFailureDetector, mockDetectorScheduler, mockKafkaCruiseControl);
    expectAndReplayFixMocks(mockOptimizerResult, mockBrokerStats);
    AnomalyDetectorManager anomalyDetectorManager = new AnomalyDetectorManager(anomalies, MOCK_ANOMALY_DETECTION_INTERVAL_MS, mockKafkaCruiseControl, mockAnomalyNotifier, mockGoalViolationDetector, mockBrokerFailureDetector, mockMetricAnomalyDetector, mockDiskFailureDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, mockDetectorScheduler);
    try {
        Map<String, Object> parameterConfigOverrides = new HashMap<>();
        parameterConfigOverrides.put(KAFKA_CRUISE_CONTROL_OBJECT_CONFIG, mockKafkaCruiseControl);
        parameterConfigOverrides.put(ANOMALY_DETECTION_TIME_MS_OBJECT_CONFIG, 100L);
        if (anomalyType == KafkaAnomalyType.GOAL_VIOLATION || anomalyType == KafkaAnomalyType.METRIC_ANOMALY || anomalyType == KafkaAnomalyType.DISK_FAILURE) {
            GoalViolations violations = kafkaCruiseControlConfig.getConfiguredInstance(AnomalyDetectorConfig.GOAL_VIOLATIONS_CLASS_CONFIG, GoalViolations.class, parameterConfigOverrides);
            assertTrue(violations.reasonSupplier().get().contains(String.format("%s: {}", GoalViolations.FIXABLE_GOAL_VIOLATIONS)));
            violations.addViolation("RackAwareGoal", true);
            assertTrue(violations.reasonSupplier().get().contains(String.format("%s: {RackAwareGoal}", GoalViolations.FIXABLE_GOAL_VIOLATIONS)));
            anomalies.add(violations);
        }
        if (anomalyType == KafkaAnomalyType.METRIC_ANOMALY || anomalyType == KafkaAnomalyType.DISK_FAILURE) {
            Map<BrokerEntity, Long> detectedSlowBrokers = Collections.singletonMap(new BrokerEntity("", 0), 100L);
            parameterConfigOverrides.put(METRIC_ANOMALY_BROKER_ENTITIES_OBJECT_CONFIG, detectedSlowBrokers);
            parameterConfigOverrides.put(REMOVE_SLOW_BROKER_CONFIG, false);
            parameterConfigOverrides.put(METRIC_ANOMALY_FIXABLE_OBJECT_CONFIG, true);
            SlowBrokers slowBrokers = kafkaCruiseControlConfig.getConfiguredInstance(AnomalyDetectorConfig.METRIC_ANOMALY_CLASS_CONFIG, SlowBrokers.class, parameterConfigOverrides);
            anomalies.add(slowBrokers);
        }
        if (anomalyType == KafkaAnomalyType.DISK_FAILURE) {
            Map<Integer, Map<String, Long>> failedDisksByBroker = Collections.singletonMap(0, Collections.singletonMap("tmp", 100L));
            parameterConfigOverrides.put(FAILED_DISKS_OBJECT_CONFIG, failedDisksByBroker);
            DiskFailures diskFailures = kafkaCruiseControlConfig.getConfiguredInstance(AnomalyDetectorConfig.DISK_FAILURES_CLASS_CONFIG, DiskFailures.class, parameterConfigOverrides);
            anomalies.add(diskFailures);
        }
        if (anomalyType == KafkaAnomalyType.GOAL_VIOLATION || anomalyType == KafkaAnomalyType.METRIC_ANOMALY || anomalyType == KafkaAnomalyType.DISK_FAILURE || anomalyType == KafkaAnomalyType.TOPIC_ANOMALY) {
            parameterConfigOverrides.put(BAD_TOPICS_BY_DESIRED_RF_CONFIG, Collections.singletonMap((short) 2, Collections.singleton(TOPIC_REPLICATION_FACTOR_ANOMALY_ENTRY)));
            parameterConfigOverrides.put(SELF_HEALING_TARGET_TOPIC_REPLICATION_FACTOR_CONFIG, (short) 2);
            TopicAnomaly topicAnomaly = new TopicReplicationFactorAnomaly();
            topicAnomaly.configure(parameterConfigOverrides);
            anomalies.add(topicAnomaly);
        }
        anomalyDetectorManager.startDetection();
        while (anomalyDetectorManager.numSelfHealingStarted() < 1) {
        // Wait for the anomaly to be fixed before attempting to shutdown the anomaly detector.
        }
        anomalyDetectorManager.shutdown();
        assertEquals(1, anomalyDetectorManager.numSelfHealingStarted());
        assertEquals(0, anomalyDetectorManager.numCheckedWithDelay());
        assertTrue(executorService.awaitTermination(MOCK_ANOMALY_DETECTOR_SHUTDOWN_MS, TimeUnit.MILLISECONDS));
        AnomalyDetectorState anomalyDetectorState = anomalyDetectorManager.anomalyDetectorState();
        assertEquals((long) anomalyDetectorState.metrics().get(NUM_SELF_HEALING_STARTED), 1L);
        assertEquals(anomalyDetectorState.recentAnomaliesByType().get(KafkaAnomalyType.BROKER_FAILURE).size(), 0);
        assertEquals(anomalyDetectorState.recentAnomaliesByType().get(KafkaAnomalyType.GOAL_VIOLATION).size(), anomalyType == KafkaAnomalyType.GOAL_VIOLATION ? 1 : 0);
        assertEquals(anomalyDetectorState.recentAnomaliesByType().get(KafkaAnomalyType.DISK_FAILURE).size(), anomalyType == KafkaAnomalyType.DISK_FAILURE ? 1 : 0);
        assertEquals(anomalyDetectorState.recentAnomaliesByType().get(KafkaAnomalyType.METRIC_ANOMALY).size(), anomalyType == KafkaAnomalyType.METRIC_ANOMALY ? 1 : 0);
        assertEquals(anomalyDetectorState.recentAnomaliesByType().get(KafkaAnomalyType.TOPIC_ANOMALY).size(), anomalyType == KafkaAnomalyType.TOPIC_ANOMALY ? 1 : 0);
        EasyMock.verify(mockAnomalyNotifier, mockDetectorScheduler, mockKafkaCruiseControl, mockBrokerFailureDetector, mockGoalViolationDetector, mockMetricAnomalyDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, mockDiskFailureDetector);
    } finally {
        executorService.shutdown();
    }
    EasyMock.verify(mockAnomalyNotifier, mockDetectorScheduler, mockKafkaCruiseControl, mockBrokerFailureDetector, mockGoalViolationDetector, mockMetricAnomalyDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, mockDiskFailureDetector);
}

Example 4 with Anomaly

use of com.linkedin.cruisecontrol.detector.Anomaly in project cruise-control by linkedin.

the class AnomalyDetectorManagerTest method testDelayedCheck.

@Test
public void testDelayedCheck() throws InterruptedException {
    PriorityBlockingQueue<Anomaly> anomalies = new PriorityBlockingQueue<>(ANOMALY_DETECTOR_INITIAL_QUEUE_SIZE, anomalyComparator());
    AnomalyNotifier mockAnomalyNotifier = EasyMock.mock(AnomalyNotifier.class);
    BrokerFailureDetector mockBrokerFailureDetector = EasyMock.createNiceMock(BrokerFailureDetector.class);
    GoalViolationDetector mockGoalViolationDetector = EasyMock.createNiceMock(GoalViolationDetector.class);
    MetricAnomalyDetector mockMetricAnomalyDetector = EasyMock.createNiceMock(MetricAnomalyDetector.class);
    TopicAnomalyDetector mockTopicAnomalyDetector = EasyMock.createNiceMock(TopicAnomalyDetector.class);
    MaintenanceEventDetector mockMaintenanceEventDetector = EasyMock.createNiceMock(MaintenanceEventDetector.class);
    DiskFailureDetector mockDiskFailureDetector = EasyMock.createNiceMock(DiskFailureDetector.class);
    ScheduledExecutorService mockDetectorScheduler = EasyMock.mock(ScheduledExecutorService.class);
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    KafkaCruiseControl mockKafkaCruiseControl = EasyMock.mock(KafkaCruiseControl.class);
    EasyMock.expect(mockAnomalyNotifier.onBrokerFailure(EasyMock.isA(BrokerFailures.class))).andReturn(AnomalyNotificationResult.check(MOCK_DELAY_CHECK_MS));
    EasyMock.expect(mockAnomalyNotifier.selfHealingEnabledRatio()).andReturn(MOCK_SELF_HEALING_ENABLED_RATIO);
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    KafkaCruiseControlConfig kafkaCruiseControlConfig = new KafkaCruiseControlConfig(props);
    EasyMock.expect(mockKafkaCruiseControl.config()).andReturn(kafkaCruiseControlConfig).times(1, 4);
    startRunnableDetectors(mockDetectorScheduler, mockGoalViolationDetector, mockMetricAnomalyDetector, mockDiskFailureDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, executorService);
    // Schedule a delayed check
    EasyMock.expect(mockDetectorScheduler.schedule(EasyMock.isA(Runnable.class), EasyMock.eq(MOCK_DELAY_CHECK_MS), EasyMock.eq(TimeUnit.MILLISECONDS))).andReturn(null);
    shutdownDetector(mockDetectorScheduler, executorService);
    // The following state are used to test the delayed check when executor is idle.
    EasyMock.expect(mockKafkaCruiseControl.executionState()).andReturn(ExecutorState.State.NO_TASK_IN_PROGRESS);
    replayCommonMocks(mockAnomalyNotifier, mockBrokerFailureDetector, mockGoalViolationDetector, mockMetricAnomalyDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, mockDiskFailureDetector, mockDetectorScheduler, mockKafkaCruiseControl);
    AnomalyDetectorManager anomalyDetectorManager = new AnomalyDetectorManager(anomalies, MOCK_ANOMALY_DETECTION_INTERVAL_MS, mockKafkaCruiseControl, mockAnomalyNotifier, mockGoalViolationDetector, mockBrokerFailureDetector, mockMetricAnomalyDetector, mockDiskFailureDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, mockDetectorScheduler);
    try {
        anomalyDetectorManager.startDetection();
        Map<String, Object> parameterConfigOverrides = Map.of(KAFKA_CRUISE_CONTROL_OBJECT_CONFIG, mockKafkaCruiseControl, FAILED_BROKERS_OBJECT_CONFIG, Collections.singletonMap(0, 100L), ANOMALY_DETECTION_TIME_MS_OBJECT_CONFIG, 100L, BROKER_FAILURES_FIXABLE_CONFIG, true);
        anomalies.add(kafkaCruiseControlConfig.getConfiguredInstance(AnomalyDetectorConfig.BROKER_FAILURES_CLASS_CONFIG, BrokerFailures.class, parameterConfigOverrides));
        while (anomalyDetectorManager.numCheckedWithDelay() < 1) {
        // Wait for the anomaly to be checked with delay before attempting to shutdown the anomaly detector.
        }
        anomalyDetectorManager.shutdown();
        assertEquals(0, anomalyDetectorManager.numSelfHealingStarted());
        assertEquals(1, anomalyDetectorManager.numCheckedWithDelay());
        assertTrue(executorService.awaitTermination(MOCK_ANOMALY_DETECTOR_SHUTDOWN_MS, TimeUnit.MILLISECONDS));
        AnomalyDetectorState anomalyDetectorState = anomalyDetectorManager.anomalyDetectorState();
        assertEquals((long) anomalyDetectorState.metrics().get(NUM_SELF_HEALING_STARTED), 0L);
        assertEquals(anomalyDetectorState.recentAnomaliesByType().get(KafkaAnomalyType.BROKER_FAILURE).size(), 1);
        assertEquals(anomalyDetectorState.recentAnomaliesByType().get(KafkaAnomalyType.GOAL_VIOLATION).size(), 0);
        assertEquals(anomalyDetectorState.recentAnomaliesByType().get(KafkaAnomalyType.METRIC_ANOMALY).size(), 0);
    } finally {
        executorService.shutdown();
    }
    EasyMock.verify(mockAnomalyNotifier, mockDetectorScheduler, mockKafkaCruiseControl, mockBrokerFailureDetector, mockGoalViolationDetector, mockMetricAnomalyDetector, mockTopicAnomalyDetector, mockMaintenanceEventDetector, mockDiskFailureDetector);
}

Example 5 with Anomaly

use of com.linkedin.cruisecontrol.detector.Anomaly in project cruise-control by linkedin.

the class BrokerFailureDetectorTest method testFailureDetection.

@Test
public void testFailureDetection() throws Exception {
    Time mockTime = getMockTime();
    Queue<Anomaly> anomalies = new PriorityBlockingQueue<>(ANOMALY_DETECTOR_INITIAL_QUEUE_SIZE, anomalyComparator());
    File failedBrokersFile = File.createTempFile("testFailureDetection", ".txt");
    BrokerFailureDetector detector = createBrokerFailureDetector(anomalies, mockTime, failedBrokersFile.getPath());
    try {
        String failedBrokerListString = detector.loadPersistedFailedBrokerList();
        assertTrue(failedBrokerListString.isEmpty());
        // Start detection.
        detector.startDetection();
        assertTrue(anomalies.isEmpty());
        int brokerId = 0;
        long anomalyTime = mockTime.milliseconds();
        killBroker(brokerId);
        long start = System.currentTimeMillis();
        while (anomalies.isEmpty() && System.currentTimeMillis() < start + 30000) {
        // wait for the anomalies to be drained.
        }
        assertEquals("One broker failure should have been detected before timeout.", 1, anomalies.size());
        failedBrokerListString = detector.loadPersistedFailedBrokerList();
        assertEquals(String.format("%d=%d", brokerId, anomalyTime), failedBrokerListString);
        Anomaly anomaly = anomalies.remove();
        assertThat(anomaly, instanceOf(BrokerFailures.class));
        BrokerFailures brokerFailures = (BrokerFailures) anomaly;
        assertEquals("The failed broker should be 0 and time should be 100L", Collections.singletonMap(brokerId, 100L), brokerFailures.failedBrokers());
        assertFalse(brokerFailures.fixable());
        // Ensure that broker failure is detected as long as the broker is down.
        detector.detectBrokerFailures(false);
        assertEquals("One broker failure should have been detected before timeout.", 1, anomalies.size());
        // Bring the broker back
        restartDeadBroker(brokerId);
        detector.detectBrokerFailures(true);
        assertTrue(detector.failedBrokers().isEmpty());
        failedBrokerListString = detector.loadPersistedFailedBrokerList();
        assertTrue(failedBrokerListString.isEmpty());
    } finally {
        detector.shutdown();
        Files.delete(failedBrokersFile.toPath());
    }
}