Examples with ClusterModelStats com.linkedin.kafka.cruisecontrol.model.ClusterModelStats used on opensource projects

Example 6 with ClusterModelStats

use of com.linkedin.kafka.cruisecontrol.model.ClusterModelStats in project cruise-control by linkedin.

the class OptimizationVerifier method executeGoalsFor.

/**
 * Execute given goals in the given cluster enforcing the given constraint. Return pass / fail status of a test.
 * A test fails if:
 * 1) Rebalance: During the optimization process, optimization of a goal leads to a worse cluster state (in terms of
 * the requirements of the same goal) than the cluster state just before starting the optimization.
 * 2) Self Healing: There are replicas on dead brokers after self healing.
 * 3) Adding a new broker causes the replicas to move among old brokers.
 *
 * @param constraint         Balancing constraint for the given cluster.
 * @param clusterModel       The state of the cluster.
 * @param goalNameByPriority Name of goals by the order of execution priority.
 * @param excludedTopics     The excluded topics.
 * @param verifications      The verifications to make after the optimization.
 * @return Pass / fail status of a test.
 */
static boolean executeGoalsFor(BalancingConstraint constraint, ClusterModel clusterModel, Map<Integer, String> goalNameByPriority, Collection<String> excludedTopics, List<Verification> verifications) throws Exception {
    // Get the initial stats from the cluster.
    ClusterModelStats preOptimizedStats = clusterModel.getClusterStats(constraint);
    // Set goals by their priority.
    SortedMap<Integer, Goal> goalByPriority = new TreeMap<>();
    for (Map.Entry<Integer, String> goalEntry : goalNameByPriority.entrySet()) {
        Integer priority = goalEntry.getKey();
        String goalClassName = goalEntry.getValue();
        Class<? extends Goal> goalClass = (Class<? extends Goal>) Class.forName(goalClassName);
        try {
            Constructor<? extends Goal> constructor = goalClass.getDeclaredConstructor(BalancingConstraint.class);
            constructor.setAccessible(true);
            goalByPriority.put(priority, constructor.newInstance(constraint));
        } catch (NoSuchMethodException badConstructor) {
            // Try default constructor
            goalByPriority.put(priority, goalClass.newInstance());
        }
    }
    // Generate the goalOptimizer and optimize given goals.
    long startTime = System.currentTimeMillis();
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    StringJoiner stringJoiner = new StringJoiner(",");
    excludedTopics.forEach(stringJoiner::add);
    props.setProperty(KafkaCruiseControlConfig.TOPICS_EXCLUDED_FROM_PARTITION_MOVEMENT_CONFIG, stringJoiner.toString());
    GoalOptimizer goalOptimizer = new GoalOptimizer(new KafkaCruiseControlConfig(constraint.setProps(props)), null, new SystemTime(), new MetricRegistry());
    GoalOptimizer.OptimizerResult optimizerResult = goalOptimizer.optimizations(clusterModel, goalByPriority, new OperationProgress());
    LOG.trace("Took {} ms to execute {} to generate {} proposals.", System.currentTimeMillis() - startTime, goalByPriority, optimizerResult.goalProposals().size());
    for (Verification verification : verifications) {
        switch(verification) {
            case GOAL_VIOLATION:
                if (!verifyGoalViolations(optimizerResult)) {
                    return false;
                }
                break;
            case NEW_BROKERS:
                if (!clusterModel.newBrokers().isEmpty() && !verifyNewBrokers(clusterModel, constraint)) {
                    return false;
                }
                break;
            case DEAD_BROKERS:
                if (!clusterModel.deadBrokers().isEmpty() && !verifyDeadBrokers(clusterModel)) {
                    return false;
                }
                break;
            case REGRESSION:
                if (clusterModel.selfHealingEligibleReplicas().isEmpty() && !verifyRegression(optimizerResult, preOptimizedStats)) {
                    return false;
                }
                break;
            default:
                throw new IllegalStateException("Invalid verification " + verification);
        }
    }
    return true;
}

Example 7 with ClusterModelStats

use of com.linkedin.kafka.cruisecontrol.model.ClusterModelStats in project cruise-control by linkedin.

the class GoalOptimizer method optimizations.

/**
 * Depending the existence of dead/decommissioned brokers in the given cluster:
 * (1) Re-balance: Generates proposals to update the state of the cluster to achieve a final balanced state.
 * (2) Self-healing: Generates proposals to move replicas away from decommissioned brokers.
 * Returns a map from goal names to stats. Initial stats are returned under goal name "init".
 *
 * @param clusterModel The state of the cluster over which the balancing proposal will be applied. Function execution
 *                     updates the cluster state with balancing proposals. If the cluster model is specified, the
 *                     cached proposal will be ignored.
 * @param goalsByPriority the goals ordered by priority.
 * @param operationProgress to report the job progress.
 * @return Results of optimization containing the proposals and stats.
 */
public OptimizerResult optimizations(ClusterModel clusterModel, Map<Integer, Goal> goalsByPriority, OperationProgress operationProgress) throws KafkaCruiseControlException {
    if (clusterModel == null) {
        throw new IllegalArgumentException("The cluster model cannot be null");
    }
    // Sanity check for optimizing goals.
    if (!clusterModel.isClusterAlive()) {
        throw new IllegalArgumentException("All brokers are dead in the cluster.");
    }
    LOG.trace("Cluster before optimization is {}", clusterModel);
    ClusterModel.BrokerStats brokerStatsBeforeOptimization = clusterModel.brokerStats();
    Map<TopicPartition, List<Integer>> initReplicaDistribution = clusterModel.getReplicaDistribution();
    Map<TopicPartition, Integer> initLeaderDistribution = clusterModel.getLeaderDistribution();
    boolean isSelfHealing = !clusterModel.selfHealingEligibleReplicas().isEmpty();
    // Set of balancing proposals that will be applied to the given cluster state to satisfy goals (leadership
    // transfer AFTER partition transfer.)
    Set<Goal> optimizedGoals = new HashSet<>();
    Set<Goal> violatedGoalsBeforeOptimization = new HashSet<>();
    Set<Goal> violatedGoalsAfterOptimization = new HashSet<>();
    Map<Goal, ClusterModelStats> statsByGoalPriority = new LinkedHashMap<>();
    Map<TopicPartition, List<Integer>> preOptimizedReplicaDistribution = null;
    Map<TopicPartition, Integer> preOptimizedLeaderDistribution = null;
    Set<String> excludedTopics = excludedTopics(clusterModel);
    LOG.debug("Topics excluded from partition movement: {}", excludedTopics);
    for (Map.Entry<Integer, Goal> entry : goalsByPriority.entrySet()) {
        preOptimizedReplicaDistribution = preOptimizedReplicaDistribution == null ? initReplicaDistribution : clusterModel.getReplicaDistribution();
        preOptimizedLeaderDistribution = preOptimizedLeaderDistribution == null ? initLeaderDistribution : clusterModel.getLeaderDistribution();
        Goal goal = entry.getValue();
        OptimizationForGoal step = new OptimizationForGoal(goal.name());
        operationProgress.addStep(step);
        LOG.debug("Optimizing goal {}", goal.name());
        boolean succeeded = goal.optimize(clusterModel, optimizedGoals, excludedTopics);
        optimizedGoals.add(goal);
        statsByGoalPriority.put(goal, clusterModel.getClusterStats(_balancingConstraint));
        Set<ExecutionProposal> goalProposals = AnalyzerUtils.getDiff(preOptimizedReplicaDistribution, preOptimizedLeaderDistribution, clusterModel);
        if (!goalProposals.isEmpty() || !succeeded) {
            violatedGoalsBeforeOptimization.add(goal);
        }
        if (!succeeded) {
            violatedGoalsAfterOptimization.add(goal);
        }
        logProgress(isSelfHealing, goal.name(), optimizedGoals.size(), goalProposals);
        step.done();
        LOG.debug("Broker level stats after optimization: {}", clusterModel.brokerStats());
    }
    clusterModel.sanityCheck();
    // Broker level stats in the final cluster state.
    if (LOG.isTraceEnabled()) {
        LOG.trace("Broker level stats after optimization: {}%n", clusterModel.brokerStats());
    }
    Set<ExecutionProposal> proposals = AnalyzerUtils.getDiff(initReplicaDistribution, initLeaderDistribution, clusterModel);
    return new OptimizerResult(statsByGoalPriority, violatedGoalsBeforeOptimization, violatedGoalsAfterOptimization, proposals, brokerStatsBeforeOptimization, clusterModel.brokerStats(), clusterModel.generation(), clusterModel.getClusterStats(_balancingConstraint));
}

Example 8 with ClusterModelStats

use of com.linkedin.kafka.cruisecontrol.model.ClusterModelStats in project cruise-control by linkedin.

the class AbstractGoal method optimize.

@Override
public boolean optimize(ClusterModel clusterModel, Set<Goal> optimizedGoals, Set<String> excludedTopics) throws OptimizationFailureException {
    _succeeded = true;
    LOG.debug("Starting optimization for {}.", name());
    // Initialize pre-optimized stats.
    ClusterModelStats statsBeforeOptimization = clusterModel.getClusterStats(_balancingConstraint);
    LOG.trace("[PRE - {}] {}", name(), statsBeforeOptimization);
    _finished = false;
    long goalStartTime = System.currentTimeMillis();
    initGoalState(clusterModel, excludedTopics);
    Collection<Broker> deadBrokers = clusterModel.deadBrokers();
    while (!_finished) {
        for (Broker broker : brokersToBalance(clusterModel)) {
            rebalanceForBroker(broker, clusterModel, optimizedGoals, excludedTopics);
        }
        updateGoalState(clusterModel, excludedTopics);
    }
    ClusterModelStats statsAfterOptimization = clusterModel.getClusterStats(_balancingConstraint);
    LOG.trace("[POST - {}] {}", name(), statsAfterOptimization);
    LOG.debug("Finished optimization for {} in {}ms.", name(), System.currentTimeMillis() - goalStartTime);
    LOG.trace("Cluster after optimization is {}", clusterModel);
    // We only ensure the optimization did not make stats worse when it is not self-healing.
    if (deadBrokers.isEmpty()) {
        ClusterModelStatsComparator comparator = clusterModelStatsComparator();
        // Throw exception when the stats before optimization is preferred.
        if (comparator.compare(statsAfterOptimization, statsBeforeOptimization) < 0) {
            throw new OptimizationFailureException("Optimization for Goal " + name() + " failed because the optimized" + "result is worse than before. Detail reason: " + comparator.explainLastComparison());
        }
    }
    return _succeeded;
}