Examples with ClusterModelStats com.linkedin.kafka.cruisecontrol.model.ClusterModelStats used on opensource projects

Example 1 with ClusterModelStats

use of com.linkedin.kafka.cruisecontrol.model.ClusterModelStats in project cruise-control by linkedin.

the class KafkaCruiseControlServlet method rebalance.

private boolean rebalance(HttpServletRequest request, HttpServletResponse response) throws Exception {
    boolean dryrun;
    DataFrom dataFrom;
    List<String> goals;
    boolean json = wantJSON(request);
    try {
        dryrun = getDryRun(request);
        goals = getGoals(request);
        dataFrom = getDataFrom(request);
    } catch (Exception e) {
        StringWriter sw = new StringWriter();
        e.printStackTrace(new PrintWriter(sw));
        setErrorResponse(response, sw.toString(), e.getMessage(), SC_BAD_REQUEST, json);
        // Close session
        return true;
    }
    GoalsAndRequirements goalsAndRequirements = getGoalsAndRequirements(request, response, goals, dataFrom, false);
    if (goalsAndRequirements == null) {
        return false;
    }
    GoalOptimizer.OptimizerResult optimizerResult = getAndMaybeReturnProgress(request, response, () -> _asyncKafkaCruiseControl.rebalance(goalsAndRequirements.goals(), dryrun, goalsAndRequirements.requirements()));
    if (optimizerResult == null) {
        return false;
    }
    setResponseCode(response, SC_OK);
    OutputStream out = response.getOutputStream();
    out.write(KafkaCruiseControlServletUtils.getProposalSummary(optimizerResult).getBytes(StandardCharsets.UTF_8));
    for (Map.Entry<Goal, ClusterModelStats> entry : optimizerResult.statsByGoalPriority().entrySet()) {
        Goal goal = entry.getKey();
        out.write(String.format("%n%nStats for goal %s%s:%n", goal.name(), goalResultDescription(goal, optimizerResult)).getBytes(StandardCharsets.UTF_8));
        out.write(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
    }
    out.write(String.format("%nCluster load after rebalance:%n").getBytes(StandardCharsets.UTF_8));
    out.write(optimizerResult.brokerStatsAfterOptimization().toString().getBytes(StandardCharsets.UTF_8));
    out.flush();
    return true;
}

Example 2 with ClusterModelStats

use of com.linkedin.kafka.cruisecontrol.model.ClusterModelStats in project cruise-control by linkedin.

the class KafkaCruiseControlServlet method addOrRemoveBroker.

private boolean addOrRemoveBroker(HttpServletRequest request, HttpServletResponse response, EndPoint endPoint) throws Exception {
    List<Integer> brokerIds = new ArrayList<>();
    boolean dryrun;
    DataFrom dataFrom;
    boolean throttleAddedOrRemovedBrokers;
    List<String> goals;
    boolean json = wantJSON(request);
    try {
        String[] brokerIdsString = request.getParameter(BROKER_ID_PARAM).split(",");
        for (String brokerIdString : brokerIdsString) {
            brokerIds.add(Integer.parseInt(brokerIdString));
        }
        dryrun = getDryRun(request);
        goals = getGoals(request);
        dataFrom = getDataFrom(request);
        String throttleBrokerString = endPoint == ADD_BROKER ? request.getParameter(THROTTLE_ADDED_BROKER_PARAM) : request.getParameter(THROTTLE_REMOVED_BROKER_PARAM);
        throttleAddedOrRemovedBrokers = throttleBrokerString == null || Boolean.parseBoolean(throttleBrokerString);
    } catch (Exception e) {
        StringWriter sw = new StringWriter();
        e.printStackTrace(new PrintWriter(sw));
        setErrorResponse(response, sw.toString(), e.getMessage(), SC_BAD_REQUEST, json);
        // Close session
        return true;
    }
    GoalsAndRequirements goalsAndRequirements = getGoalsAndRequirements(request, response, goals, dataFrom, false);
    if (goalsAndRequirements == null) {
        return false;
    }
    // Get proposals asynchronously.
    GoalOptimizer.OptimizerResult optimizerResult;
    if (endPoint == ADD_BROKER) {
        optimizerResult = getAndMaybeReturnProgress(request, response, () -> _asyncKafkaCruiseControl.addBrokers(brokerIds, dryrun, throttleAddedOrRemovedBrokers, goalsAndRequirements.goals(), goalsAndRequirements.requirements()));
    } else {
        optimizerResult = getAndMaybeReturnProgress(request, response, () -> _asyncKafkaCruiseControl.decommissionBrokers(brokerIds, dryrun, throttleAddedOrRemovedBrokers, goalsAndRequirements.goals(), goalsAndRequirements.requirements()));
    }
    if (optimizerResult == null) {
        return false;
    }
    setResponseCode(response, SC_OK);
    OutputStream out = response.getOutputStream();
    out.write(KafkaCruiseControlServletUtils.getProposalSummary(optimizerResult).getBytes(StandardCharsets.UTF_8));
    for (Map.Entry<Goal, ClusterModelStats> entry : optimizerResult.statsByGoalPriority().entrySet()) {
        Goal goal = entry.getKey();
        out.write(String.format("%n%nStats for goal %s%s:%n", goal.name(), goalResultDescription(goal, optimizerResult)).getBytes(StandardCharsets.UTF_8));
        out.write(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
    }
    out.write(String.format("%nCluster load after %s broker %s:%n", endPoint == ADD_BROKER ? "adding" : "removing", brokerIds).getBytes(StandardCharsets.UTF_8));
    out.write(optimizerResult.brokerStatsAfterOptimization().toString().getBytes(StandardCharsets.UTF_8));
    out.flush();
    return true;
}

Example 3 with ClusterModelStats

use of com.linkedin.kafka.cruisecontrol.model.ClusterModelStats in project cruise-control by linkedin.

the class OptimizationVerifier method verifyRegression.

private static boolean verifyRegression(GoalOptimizer.OptimizerResult optimizerResult, ClusterModelStats preOptimizationStats) {
    // Check whether test has failed for rebalance: fails if rebalance caused a worse goal state after rebalance.
    Map<Goal, ClusterModelStats> clusterStatsByPriority = optimizerResult.statsByGoalPriority();
    ClusterModelStats preStats = preOptimizationStats;
    for (Map.Entry<Goal, ClusterModelStats> entry : clusterStatsByPriority.entrySet()) {
        Goal.ClusterModelStatsComparator comparator = entry.getKey().clusterModelStatsComparator();
        boolean success = comparator.compare(entry.getValue(), preStats) >= 0;
        if (!success) {
            LOG.error("Failed goal comparison " + entry.getKey().name() + ". " + comparator.explainLastComparison());
            return false;
        }
        preStats = entry.getValue();
    }
    return true;
}

Example 4 with ClusterModelStats

use of com.linkedin.kafka.cruisecontrol.model.ClusterModelStats in project cruise-control by linkedin.

the class KafkaCruiseControlServlet method getProposals.

private boolean getProposals(HttpServletRequest request, HttpServletResponse response) throws Exception {
    boolean verbose;
    boolean ignoreProposalCache;
    DataFrom dataFrom;
    List<String> goals;
    boolean json = wantJSON(request);
    try {
        String verboseString = request.getParameter(VERBOSE_PARAM);
        verbose = verboseString != null && Boolean.parseBoolean(verboseString);
        goals = getGoals(request);
        dataFrom = getDataFrom(request);
        String ignoreProposalCacheString = request.getParameter(IGNORE_PROPOSAL_CACHE_PARAM);
        ignoreProposalCache = (ignoreProposalCacheString != null && Boolean.parseBoolean(ignoreProposalCacheString)) || !goals.isEmpty();
    } catch (Exception e) {
        StringWriter sw = new StringWriter();
        e.printStackTrace(new PrintWriter(sw));
        setErrorResponse(response, sw.toString(), e.getMessage(), SC_BAD_REQUEST, json);
        // Close session
        return true;
    }
    GoalsAndRequirements goalsAndRequirements = getGoalsAndRequirements(request, response, goals, dataFrom, ignoreProposalCache);
    if (goalsAndRequirements == null) {
        return false;
    }
    // Get the optimization result asynchronously.
    GoalOptimizer.OptimizerResult optimizerResult = getAndMaybeReturnProgress(request, response, () -> _asyncKafkaCruiseControl.getOptimizationProposals(goalsAndRequirements.goals(), goalsAndRequirements.requirements()));
    if (optimizerResult == null) {
        return false;
    }
    setResponseCode(response, SC_OK);
    OutputStream out = response.getOutputStream();
    if (!json) {
        String loadBeforeOptimization = optimizerResult.brokerStatsBeforeOptimization().toString();
        String loadAfterOptimization = optimizerResult.brokerStatsAfterOptimization().toString();
        if (!verbose) {
            out.write(KafkaCruiseControlServletUtils.getProposalSummary(optimizerResult).getBytes(StandardCharsets.UTF_8));
        } else {
            out.write(optimizerResult.goalProposals().toString().getBytes(StandardCharsets.UTF_8));
        }
        for (Map.Entry<Goal, ClusterModelStats> entry : optimizerResult.statsByGoalPriority().entrySet()) {
            Goal goal = entry.getKey();
            out.write(String.format("%n%nStats for goal %s%s:%n", goal.name(), goalResultDescription(goal, optimizerResult)).getBytes(StandardCharsets.UTF_8));
            out.write(entry.getValue().toString().getBytes(StandardCharsets.UTF_8));
        }
        setResponseCode(response, SC_OK);
        // Print summary before & after optimization
        out.write(String.format("%n%nCurrent load:").getBytes(StandardCharsets.UTF_8));
        out.write(loadBeforeOptimization.getBytes(StandardCharsets.UTF_8));
        out.write(String.format("%n%nOptimized load:").getBytes(StandardCharsets.UTF_8));
        out.write(loadAfterOptimization.getBytes(StandardCharsets.UTF_8));
    } else {
        if (!verbose) {
            out.write(KafkaCruiseControlServletUtils.getProposalSummary(optimizerResult).getBytes(StandardCharsets.UTF_8));
        }
        // Build all the goal summary
        List<Map<String, Object>> allGoals = new ArrayList<>();
        for (Map.Entry<Goal, ClusterModelStats> entry : optimizerResult.statsByGoalPriority().entrySet()) {
            Goal goal = entry.getKey();
            String goalViolation = goalResultDescription(goal, optimizerResult);
            Map<String, Object> goalMap = new HashMap<>();
            goalMap.put("goal", goal.name());
            goalMap.put("goalViolated", goalViolation);
            goalMap.put("clusterModelStats", entry.getValue().getJsonStructure());
            allGoals.add(goalMap);
        }
        Map<String, Object> proposalMap = new HashMap<>();
        proposalMap.put("version", JSON_VERSION);
        proposalMap.put("goals", allGoals);
        proposalMap.put("loadBeforeOptimization", optimizerResult.brokerStatsBeforeOptimization().getJsonStructure());
        proposalMap.put("loadAfterOptimization", optimizerResult.brokerStatsAfterOptimization().getJsonStructure());
        Gson gson = new GsonBuilder().serializeNulls().serializeSpecialFloatingPointValues().create();
        String proposalsString = gson.toJson(proposalMap);
        setJSONResponseCode(response, SC_OK);
        out.write(proposalsString.getBytes(StandardCharsets.UTF_8));
    }
    out.flush();
    return true;
}

Example 5 with ClusterModelStats

use of com.linkedin.kafka.cruisecontrol.model.ClusterModelStats in project cruise-control by linkedin.

the class OfflineProposalGenerator method main.

public static void main(String[] argv) throws Exception {
    // TODO: probably need to save this in the original model file
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(props);
    ModelUtils.init(config);
    ModelParameters.init(config);
    BalancingConstraint balancingConstraint = new BalancingConstraint(config);
    long start = System.currentTimeMillis();
    ClusterModel clusterModel = clusterModelFromFile(argv[0]);
    long end = System.currentTimeMillis();
    double duration = (end - start) / 1000.0;
    System.out.println("Model loaded in " + duration + "s.");
    ClusterModelStats origStats = clusterModel.getClusterStats(balancingConstraint);
    String loadBeforeOptimization = clusterModel.brokerStats().toString();
    // Instantiate the components.
    GoalOptimizer goalOptimizer = new GoalOptimizer(config, null, new SystemTime(), new MetricRegistry());
    start = System.currentTimeMillis();
    GoalOptimizer.OptimizerResult optimizerResult = goalOptimizer.optimizations(clusterModel, new OperationProgress());
    end = System.currentTimeMillis();
    duration = (end - start) / 1000.0;
    String loadAfterOptimization = clusterModel.brokerStats().toString();
    System.out.println("Optimize goals in " + duration + "s.");
    System.out.println(optimizerResult.goalProposals().size());
    System.out.println(loadBeforeOptimization);
    System.out.println(loadAfterOptimization);
    ClusterModelStats optimizedStats = clusterModel.getClusterStats(balancingConstraint);
    double[] testStatistics = AnalyzerUtils.testDifference(origStats.utilizationMatrix(), optimizedStats.utilizationMatrix());
    System.out.println(Arrays.stream(RawAndDerivedResource.values()).map(x -> x.toString()).collect(Collectors.joining(", ")));
    System.out.println(Arrays.stream(testStatistics).boxed().map(pValue -> Double.toString(pValue)).collect(Collectors.joining(", ")));
}