Examples with KafkaCruiseControlState com.linkedin.kafka.cruisecontrol.KafkaCruiseControlState used on opensource projects

Example 1 with KafkaCruiseControlState

use of com.linkedin.kafka.cruisecontrol.KafkaCruiseControlState in project cruise-control by linkedin.

the class KafkaCruiseControlServlet method getState.

private boolean getState(HttpServletRequest request, HttpServletResponse response) throws Exception {
    boolean verbose;
    boolean superVerbose;
    boolean json = wantJSON(request);
    try {
        String verboseString = request.getParameter(VERBOSE_PARAM);
        verbose = verboseString != null && Boolean.parseBoolean(verboseString);
        String superVerboseString = request.getParameter(SUPER_VERBOSE_PARM);
        superVerbose = superVerboseString != null && Boolean.parseBoolean(superVerboseString);
    } catch (Exception e) {
        StringWriter sw = new StringWriter();
        e.printStackTrace(new PrintWriter(sw));
        setErrorResponse(response, sw.toString(), e.getMessage(), SC_BAD_REQUEST, json);
        // Close session
        return true;
    }
    KafkaCruiseControlState state = getAndMaybeReturnProgress(request, response, _asyncKafkaCruiseControl::state);
    if (state == null) {
        return false;
    }
    OutputStream out = response.getOutputStream();
    if (json) {
        String stateString = state.getJSONString(JSON_VERSION);
        setJSONResponseCode(response, SC_OK);
        response.setContentLength(stateString.length());
        out.write(stateString.getBytes(StandardCharsets.UTF_8));
    } else {
        String stateString = state.toString();
        setResponseCode(response, SC_OK);
        out.write(stateString.getBytes(StandardCharsets.UTF_8));
        if (verbose || superVerbose) {
            out.write(String.format("%n%nMonitored Windows [Window End_Time=Data_Completeness]:%n").getBytes(StandardCharsets.UTF_8));
            StringJoiner joiner = new StringJoiner(", ", "{", "}");
            for (Map.Entry<Long, Float> entry : state.monitorState().monitoredWindows().entrySet()) {
                joiner.add(String.format("%d=%.3f%%", entry.getKey(), entry.getValue() * 100));
            }
            out.write(joiner.toString().getBytes(StandardCharsets.UTF_8));
            out.write(String.format("%n%nGoal Readiness:%n").getBytes(StandardCharsets.UTF_8));
            for (Map.Entry<Goal, Boolean> entry : state.analyzerState().readyGoals().entrySet()) {
                Goal goal = entry.getKey();
                out.write(String.format("%50s, %s, %s%n", goal.getClass().getSimpleName(), goal.clusterModelCompletenessRequirements(), entry.getValue() ? "Ready" : "NotReady").getBytes(StandardCharsets.UTF_8));
            }
            ExecutorState executorState = state.executorState();
            if (executorState.state() == ExecutorState.State.REPLICA_MOVEMENT_TASK_IN_PROGRESS || executorState.state() == ExecutorState.State.STOPPING_EXECUTION) {
                out.write(String.format("%n%nIn progress %s:%n", PARTITION_MOVEMENTS).getBytes(StandardCharsets.UTF_8));
                for (ExecutionTask task : executorState.inProgressPartitionMovements()) {
                    out.write(String.format("%s%n", task).getBytes(StandardCharsets.UTF_8));
                }
                out.write(String.format("%n%nAborting %s:%n", PARTITION_MOVEMENTS).getBytes(StandardCharsets.UTF_8));
                for (ExecutionTask task : executorState.abortingPartitionMovements()) {
                    out.write(String.format("%s%n", task).getBytes(StandardCharsets.UTF_8));
                }
                out.write(String.format("%n%nAborted %s:%n", PARTITION_MOVEMENTS).getBytes(StandardCharsets.UTF_8));
                for (ExecutionTask task : executorState.abortedPartitionMovements()) {
                    out.write(String.format("%s%n", task).getBytes(StandardCharsets.UTF_8));
                }
                out.write(String.format("%n%nDead %s:%n", PARTITION_MOVEMENTS).getBytes(StandardCharsets.UTF_8));
                for (ExecutionTask task : executorState.deadPartitionMovements()) {
                    out.write(String.format("%s%n", task).getBytes(StandardCharsets.UTF_8));
                }
                out.write(String.format("%n%n%s %s:%n", executorState.state() == ExecutorState.State.STOPPING_EXECUTION ? "Cancelled" : "Pending", PARTITION_MOVEMENTS).getBytes(StandardCharsets.UTF_8));
                for (ExecutionTask task : executorState.pendingPartitionMovements()) {
                    out.write(String.format("%s%n", task).getBytes(StandardCharsets.UTF_8));
                }
            }
            if (superVerbose) {
                out.write(String.format("%n%nExtrapolated metric samples:%n").getBytes(StandardCharsets.UTF_8));
                Map<TopicPartition, List<SampleExtrapolation>> sampleFlaws = state.monitorState().sampleExtrapolations();
                if (sampleFlaws != null && !sampleFlaws.isEmpty()) {
                    for (Map.Entry<TopicPartition, List<SampleExtrapolation>> entry : sampleFlaws.entrySet()) {
                        out.write(String.format("%n%s: %s", entry.getKey(), entry.getValue()).getBytes(StandardCharsets.UTF_8));
                    }
                } else {
                    out.write("None".getBytes(StandardCharsets.UTF_8));
                }
                out.write(String.format("%n%nLinear Regression Model State:%n%s", state.monitorState().detailTrainingProgress()).getBytes(StandardCharsets.UTF_8));
            }
        }
    }
    response.getOutputStream().flush();
    return true;
}

Example 2 with KafkaCruiseControlState

use of com.linkedin.kafka.cruisecontrol.KafkaCruiseControlState in project cruise-control by linkedin.

the class KafkaCruiseControlServletDataFromTest method test.

@Test
public void test() throws Exception {
    AsyncKafkaCruiseControl mockKCC = EasyMock.createMock(AsyncKafkaCruiseControl.class);
    HttpServletRequest request = EasyMock.createMock(HttpServletRequest.class);
    HttpServletResponse response = EasyMock.createMock(HttpServletResponse.class);
    HttpSession session = EasyMock.createMock(HttpSession.class);
    EasyMock.expect(request.getSession()).andReturn(session).anyTimes();
    EasyMock.expect(request.getMethod()).andReturn("GET").anyTimes();
    EasyMock.expect(request.getRequestURI()).andReturn("/test").anyTimes();
    EasyMock.expect(request.getParameterMap()).andReturn(Collections.emptyMap()).anyTimes();
    EasyMock.expect(session.getLastAccessedTime()).andReturn(Long.MAX_VALUE);
    KafkaCruiseControlState kccState = getState(_numReadyGoals, _totalGoals, _numValidWindows);
    OperationFuture<KafkaCruiseControlState> kccStateFuture = new OperationFuture<>("test");
    kccStateFuture.complete(kccState);
    EasyMock.expect(mockKCC.state()).andReturn(kccStateFuture).anyTimes();
    EasyMock.replay(mockKCC, request, response, session);
    KafkaCruiseControlServlet servlet = new KafkaCruiseControlServlet(mockKCC, 10, 100, new MetricRegistry());
    KafkaCruiseControlServlet.GoalsAndRequirements goalsAndRequirements = servlet.getGoalsAndRequirements(request, response, Collections.emptyList(), _dataFrom, false);
    assertEquals(new HashSet<>(goalsAndRequirements.goals()), new HashSet<>(_expectedGoalsToUse));
    if (_expectedRequirements != null) {
        assertEquals(_expectedRequirements.minRequiredNumWindows(), goalsAndRequirements.requirements().minRequiredNumWindows());
        assertEquals(_expectedRequirements.minMonitoredPartitionsPercentage(), goalsAndRequirements.requirements().minMonitoredPartitionsPercentage(), 0.0);
        assertEquals(_expectedRequirements.includeAllTopics(), goalsAndRequirements.requirements().includeAllTopics());
    } else {
        assertNull("The requirement should be null", goalsAndRequirements.requirements());
    }
}

Example 3 with KafkaCruiseControlState

use of com.linkedin.kafka.cruisecontrol.KafkaCruiseControlState in project cruise-control by linkedin.

the class KafkaCruiseControlServletDataFromTest method getState.

/**
 * Generate the KCC state.
 */
private KafkaCruiseControlState getState(int numReadyGoals, int totalGoals, int numValidWindows) {
    ExecutorState executorState = ExecutorState.noTaskInProgress();
    LoadMonitorState loadMonitorState = LoadMonitorState.running(numValidWindows, new TreeMap<>(), 1, 10, Collections.emptyMap());
    Map<Goal, Boolean> goalReadiness = new HashMap<>();
    int i = 0;
    for (; i < numReadyGoals; i++) {
        goalReadiness.put(new MockGoal(i), true);
    }
    for (; i < totalGoals; i++) {
        goalReadiness.put(new MockGoal(i), false);
    }
    AnalyzerState analyzerState = new AnalyzerState(true, goalReadiness);
    return new KafkaCruiseControlState(executorState, loadMonitorState, analyzerState);
}

Example 4 with KafkaCruiseControlState

use of com.linkedin.kafka.cruisecontrol.KafkaCruiseControlState in project cruise-control by linkedin.

the class AnomalyDetectorTest method testExecutionInProgress.

@Test
public void testExecutionInProgress() throws InterruptedException {
    LinkedBlockingDeque<Anomaly> anomalies = new LinkedBlockingDeque<>();
    AnomalyNotifier mockAnomalyNotifier = EasyMock.mock(AnomalyNotifier.class);
    BrokerFailureDetector mockBrokerFailureDetector = EasyMock.createNiceMock(BrokerFailureDetector.class);
    GoalViolationDetector mockGoalViolationDetector = EasyMock.createNiceMock(GoalViolationDetector.class);
    ScheduledExecutorService mockDetectorScheduler = EasyMock.mock(ScheduledExecutorService.class);
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    KafkaCruiseControl mockKafkaCruiseControl = EasyMock.mock(KafkaCruiseControl.class);
    // Starting periodic goal violation detection.
    EasyMock.expect(mockDetectorScheduler.scheduleAtFixedRate(EasyMock.eq(mockGoalViolationDetector), EasyMock.anyLong(), EasyMock.eq(3000L), EasyMock.eq(TimeUnit.MILLISECONDS))).andReturn(null);
    // Starting anomaly handler
    EasyMock.expect(mockDetectorScheduler.submit(EasyMock.isA(AnomalyDetector.AnomalyHandlerTask.class))).andDelegateTo(executorService);
    // For detector shutdown.
    mockDetectorScheduler.shutdown();
    EasyMock.expectLastCall().andDelegateTo(executorService);
    EasyMock.expect(mockDetectorScheduler.awaitTermination(3000L, TimeUnit.MILLISECONDS)).andDelegateTo(executorService);
    EasyMock.expect(mockDetectorScheduler.isTerminated()).andDelegateTo(executorService);
    // The following state are used to test the delayed check when executor is busy.
    EasyMock.expect(mockKafkaCruiseControl.state(EasyMock.anyObject())).andReturn(new KafkaCruiseControlState(ExecutorState.replicaMovementInProgress(1, Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), Collections.emptySet(), 1L, 1), null, null));
    EasyMock.replay(mockAnomalyNotifier);
    EasyMock.replay(mockBrokerFailureDetector);
    EasyMock.replay(mockGoalViolationDetector);
    EasyMock.replay(mockDetectorScheduler);
    EasyMock.replay(mockKafkaCruiseControl);
    AnomalyDetector anomalyDetector = new AnomalyDetector(anomalies, 3000L, mockKafkaCruiseControl, mockAnomalyNotifier, mockGoalViolationDetector, mockBrokerFailureDetector, mockDetectorScheduler, EasyMock.mock(LoadMonitor.class));
    try {
        anomalyDetector.startDetection();
        anomalies.add(new GoalViolations());
        while (!anomalies.isEmpty()) {
        // Just wait for the anomalies to be drained.
        }
        anomalyDetector.shutdown();
        assertTrue(executorService.awaitTermination(5000, TimeUnit.MILLISECONDS));
        EasyMock.verify(mockAnomalyNotifier, mockDetectorScheduler, mockKafkaCruiseControl);
    } finally {
        executorService.shutdown();
    }
}

Example 5 with KafkaCruiseControlState

use of com.linkedin.kafka.cruisecontrol.KafkaCruiseControlState in project cruise-control by linkedin.

the class AnomalyDetectorTest method testDelayedCheck.

@Test
public void testDelayedCheck() throws InterruptedException {
    LinkedBlockingDeque<Anomaly> anomalies = new LinkedBlockingDeque<>();
    AnomalyNotifier mockAnomalyNotifier = EasyMock.mock(AnomalyNotifier.class);
    BrokerFailureDetector mockBrokerFailureDetector = EasyMock.createNiceMock(BrokerFailureDetector.class);
    GoalViolationDetector mockGoalViolationDetector = EasyMock.createNiceMock(GoalViolationDetector.class);
    ScheduledExecutorService mockDetectorScheduler = EasyMock.mock(ScheduledExecutorService.class);
    ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
    KafkaCruiseControl mockKafkaCruiseControl = EasyMock.mock(KafkaCruiseControl.class);
    EasyMock.expect(mockAnomalyNotifier.onBrokerFailure(EasyMock.isA(BrokerFailures.class))).andReturn(AnomalyNotificationResult.check(1000L));
    // Starting periodic goal violation detection.
    EasyMock.expect(mockDetectorScheduler.scheduleAtFixedRate(EasyMock.eq(mockGoalViolationDetector), EasyMock.anyLong(), EasyMock.eq(3000L), EasyMock.eq(TimeUnit.MILLISECONDS))).andReturn(null);
    // Starting anomaly handler
    EasyMock.expect(mockDetectorScheduler.submit(EasyMock.isA(AnomalyDetector.AnomalyHandlerTask.class))).andDelegateTo(executorService);
    // Schedule a delayed check
    EasyMock.expect(mockDetectorScheduler.schedule(EasyMock.isA(Runnable.class), EasyMock.eq(1000L), EasyMock.eq(TimeUnit.MILLISECONDS))).andReturn(null);
    mockDetectorScheduler.shutdown();
    EasyMock.expectLastCall().andDelegateTo(executorService);
    EasyMock.expect(mockDetectorScheduler.awaitTermination(3000L, TimeUnit.MILLISECONDS)).andDelegateTo(executorService);
    EasyMock.expect(mockDetectorScheduler.isTerminated()).andDelegateTo(executorService);
    // The following state are used to test the delayed check when executor is busy.
    EasyMock.expect(mockKafkaCruiseControl.state(EasyMock.anyObject())).andReturn(new KafkaCruiseControlState(ExecutorState.noTaskInProgress(), null, null));
    EasyMock.replay(mockAnomalyNotifier);
    EasyMock.replay(mockBrokerFailureDetector);
    EasyMock.replay(mockGoalViolationDetector);
    EasyMock.replay(mockDetectorScheduler);
    EasyMock.replay(mockKafkaCruiseControl);
    AnomalyDetector anomalyDetector = new AnomalyDetector(anomalies, 3000L, mockKafkaCruiseControl, mockAnomalyNotifier, mockGoalViolationDetector, mockBrokerFailureDetector, mockDetectorScheduler, EasyMock.mock(LoadMonitor.class));
    try {
        anomalyDetector.startDetection();
        anomalies.add(new BrokerFailures(Collections.singletonMap(0, 100L)));
        while (!anomalies.isEmpty()) {
        // just wait for the anomalies to be drained.
        }
        anomalyDetector.shutdown();
        assertTrue(executorService.awaitTermination(5000, TimeUnit.MILLISECONDS));
        EasyMock.verify(mockAnomalyNotifier, mockDetectorScheduler, mockKafkaCruiseControl);
    } finally {
        executorService.shutdown();
    }
}