Examples with SortedReplicasHelper com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper used on opensource projects

Example 1 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class RackAwareGoal method initGoalState.

/**
 * This is a hard goal; hence, the proposals are not limited to dead broker replicas in case of self-healing.
 * Sanity Check: There exists sufficient number of racks for achieving rack-awareness.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    // Sanity Check: not enough racks to satisfy rack awareness.
    int numAliveRacks = clusterModel.numAliveRacks();
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    if (!excludedTopics.isEmpty()) {
        int maxReplicationFactorOfIncludedTopics = 1;
        Map<String, Integer> replicationFactorByTopic = clusterModel.replicationFactorByTopic();
        for (Map.Entry<String, Integer> replicationFactorByTopicEntry : replicationFactorByTopic.entrySet()) {
            if (!excludedTopics.contains(replicationFactorByTopicEntry.getKey())) {
                maxReplicationFactorOfIncludedTopics = Math.max(maxReplicationFactorOfIncludedTopics, replicationFactorByTopicEntry.getValue());
                if (maxReplicationFactorOfIncludedTopics > numAliveRacks) {
                    int missingRacks = maxReplicationFactorOfIncludedTopics - numAliveRacks;
                    ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numRacks(missingRacks).build();
                    throw new OptimizationFailureException(String.format("[%s] Insufficient number of racks to distribute included replicas (Current: %d, Needed: %d).", name(), numAliveRacks, maxReplicationFactorOfIncludedTopics), recommendation);
                }
            }
        }
    } else if (clusterModel.maxReplicationFactor() > numAliveRacks) {
        int missingRacks = clusterModel.maxReplicationFactor() - numAliveRacks;
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numRacks(missingRacks).build();
        throw new OptimizationFailureException(String.format("[%s] Insufficient number of racks to distribute each replica (Current: %d, Needed: %d).", name(), numAliveRacks, clusterModel.maxReplicationFactor()), recommendation);
    }
    int numExtraRacks = numAliveRacks - clusterModel.maxReplicationFactor();
    if (numExtraRacks >= _balancingConstraint.overprovisionedMinExtraRacks()) {
        int numRacksToDrop = numExtraRacks - _balancingConstraint.overprovisionedMinExtraRacks() + 1;
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.OVER_PROVISIONED).numRacks(numRacksToDrop).build();
        _provisionResponse = new ProvisionResponse(ProvisionStatus.OVER_PROVISIONED, recommendation, name());
    }
    // Filter out some replicas based on optimization options.
    new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).trackSortedReplicasFor(replicaSortName(this, false, false), clusterModel);
}

Example 2 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class ResourceDistributionGoal method sortedCandidateReplicas.

/**
 * Get the sorted replicas in the given broker whose (1) topic is not an excluded topic AND (2) do not violate the given load
 * limit in ascending or descending order. Load limit requires the replica load to be (1) above the given limit in
 * descending order, (2) below the given limit in ascending order. Offline replicas have priority over online replicas.
 *
 * @param broker Broker whose replicas will be considered.
 * @param clusterModel The state of the cluster.
 * @param excludedTopics Excluded topics for which the replicas will be remove from the returned candidate replicas.
 * @param loadLimit Load limit determining the lower cutoff in descending order, upper cutoff in ascending order.
 * @param isAscending {@code true} if sort requested in ascending order, {@code false} otherwise.
 * @param followersOnly Candidate replicas contain only the followers.
 * @param leadersOnly Candidate replicas contain only the leaders.
 * @param immigrantsOnly Candidate replicas contain only the immigrant replicas.
 * @return The name of tracked sorted replicas.
 */
private String sortedCandidateReplicas(Broker broker, ClusterModel clusterModel, Set<String> excludedTopics, double loadLimit, boolean isAscending, boolean followersOnly, boolean leadersOnly, boolean immigrantsOnly) {
    SortedReplicasHelper helper = new SortedReplicasHelper();
    helper.maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectFollowers(), followersOnly).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectLeaders(), leadersOnly).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), immigrantsOnly).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).maybeAddPriorityFunc(ReplicaSortFunctionFactory.prioritizeOfflineReplicas(), !clusterModel.selfHealingEligibleReplicas().isEmpty());
    if (isAscending) {
        helper.maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBelowLimit(resource(), loadLimit), loadLimit < Double.MAX_VALUE).setScoreFunc(ReplicaSortFunctionFactory.sortByMetricGroupValue(resource().name()));
    } else {
        helper.addSelectionFunc(ReplicaSortFunctionFactory.selectReplicasAboveLimit(resource(), loadLimit)).setScoreFunc(ReplicaSortFunctionFactory.reverseSortByMetricGroupValue(resource().name()));
    }
    String replicaSortName = replicaSortName(this, !isAscending, leadersOnly);
    helper.trackSortedReplicasFor(replicaSortName, broker);
    return replicaSortName;
}

Example 3 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class TopicReplicaDistributionGoal method initGoalState.

/**
 * Initiates this goal.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    Set<String> topicsToRebalance = GoalUtils.topicsToRebalance(clusterModel, excludedTopics);
    if (topicsToRebalance.isEmpty()) {
        LOG.warn("All topics are excluded from {}.", name());
    }
    _brokersAllowedReplicaMove = GoalUtils.aliveBrokersNotExcludedForReplicaMove(clusterModel, optimizationOptions);
    if (_brokersAllowedReplicaMove.isEmpty()) {
        // Handle the case when all alive brokers are excluded from replica moves.
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(clusterModel.maxReplicationFactor()).build();
        throw new OptimizationFailureException(String.format("[%s] All alive brokers are excluded from replica moves.", name()), recommendation);
    }
    // Initialize the average replicas on an alive broker.
    for (String topic : clusterModel.topics()) {
        int numTopicReplicas = clusterModel.numTopicReplicas(topic);
        _avgTopicReplicasOnAliveBroker.put(topic, (numTopicReplicas / (double) _brokersAllowedReplicaMove.size()));
        _balanceUpperLimitByTopic.put(topic, balanceUpperLimit(topic, optimizationOptions));
        _balanceLowerLimitByTopic.put(topic, balanceLowerLimit(topic, optimizationOptions));
        // Retain only the topics to rebalance in _avgTopicReplicasOnAliveBroker
        if (!topicsToRebalance.contains(topic)) {
            _avgTopicReplicasOnAliveBroker.remove(topic);
        }
    }
    // Filter out replicas to be considered for replica movement.
    for (Broker broker : clusterModel.brokers()) {
        new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrantOrOfflineReplicas(), !clusterModel.selfHealingEligibleReplicas().isEmpty() && broker.isAlive()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).trackSortedReplicasFor(replicaSortName(this, false, false), broker);
    }
    _fixOfflineReplicasOnly = false;
}

Example 4 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class ReplicaCapacityGoal method initGoalState.

/**
 * This is a hard goal; hence, the proposals are not limited to dead broker replicas in case of self-healing.
 * Sanity Check: Each node has sufficient number of replicas that can be moved to satisfy the replica capacity goal.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    List<String> topicsToRebalance = new ArrayList<>(clusterModel.topics());
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    topicsToRebalance.removeAll(excludedTopics);
    if (topicsToRebalance.isEmpty()) {
        LOG.warn("All topics are excluded from {}.", name());
    }
    // Sanity check: excluded topic replicas in a broker cannot exceed the max number of allowed replicas per broker.
    int totalReplicasInCluster = 0;
    for (Broker broker : brokersToBalance(clusterModel)) {
        // Calculate total number of replicas for the next sanity check.
        totalReplicasInCluster += broker.replicas().size();
        if (!broker.isAlive()) {
            _isSelfHealingMode = true;
            continue;
        }
        Set<Replica> excludedReplicasInBroker = new HashSet<>();
        for (String topic : excludedTopics) {
            excludedReplicasInBroker.addAll(broker.replicasOfTopicInBroker(topic));
        }
        if (broker.state() == Broker.State.BAD_DISKS) {
            _isSelfHealingMode = true;
            excludedReplicasInBroker.removeAll(broker.currentOfflineReplicas());
        }
        if (excludedReplicasInBroker.size() > _balancingConstraint.maxReplicasPerBroker()) {
            throw new OptimizationFailureException(String.format("[%s] Replicas of excluded topics in broker: %d exceeds the maximum allowed number of replicas per broker: %d.", name(), excludedReplicasInBroker.size(), _balancingConstraint.maxReplicasPerBroker()));
        }
    }
    // Sanity check: total replicas in the cluster cannot be more than the allowed replicas in the cluster.
    Set<Integer> brokersAllowedReplicaMove = GoalUtils.aliveBrokersNotExcludedForReplicaMove(clusterModel, optimizationOptions);
    long maxReplicasInCluster = _balancingConstraint.maxReplicasPerBroker() * brokersAllowedReplicaMove.size();
    if (totalReplicasInCluster > maxReplicasInCluster) {
        int minRequiredBrokers = (int) Math.ceil(totalReplicasInCluster / (double) _balancingConstraint.maxReplicasPerBroker());
        int numBrokersToAdd = minRequiredBrokers - brokersAllowedReplicaMove.size();
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(numBrokersToAdd).build();
        throw new OptimizationFailureException(String.format("[%s] Total replicas in cluster: %d exceeds the maximum allowed replicas in cluster: %d (Alive " + "brokers: %d, Allowed number of replicas per broker: %d).", name(), totalReplicasInCluster, maxReplicasInCluster, clusterModel.aliveBrokers().size(), _balancingConstraint.maxReplicasPerBroker()), recommendation);
    }
    // Filter out some replicas based on optimization options.
    new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).trackSortedReplicasFor(replicaSortName(this, false, false), clusterModel);
}

Example 5 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class MinTopicLeadersPerBrokerGoal method initGoalState.

/**
 * This is a hard goal; hence, the proposals are not limited to dead broker replicas in case of self-healing.
 * Sanity Check: The total number of leader replicas of certain topics is sufficient so that these leader replicas
 * can be moved to satisfy this goal.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    Set<String> mustHaveTopicLeadersPerBroker = Collections.unmodifiableSet(Utils.getTopicNamesMatchedWithPattern(_balancingConstraint.topicsWithMinLeadersPerBrokerPattern(), clusterModel::topics));
    // populate min leaders per broker for each topic
    _mustHaveTopicMinLeadersPerBroker = new HashMap<>();
    if (mustHaveTopicLeadersPerBroker.isEmpty()) {
        return;
    }
    Map<String, Integer> numLeadersByTopicNames = clusterModel.numLeadersPerTopic(mustHaveTopicLeadersPerBroker);
    Set<Broker> eligibleBrokersForLeadership = eligibleBrokersForLeadership(clusterModel, optimizationOptions);
    for (String topicName : mustHaveTopicLeadersPerBroker) {
        int topicNumLeaders = numLeadersByTopicNames.get(topicName);
        _mustHaveTopicMinLeadersPerBroker.put(topicName, _balancingConstraint.minTopicLeadersPerBroker() == 0 ? eligibleBrokersForLeadership.size() == 0 ? 0 : topicNumLeaders / eligibleBrokersForLeadership.size() : _balancingConstraint.minTopicLeadersPerBroker());
    }
    // Sanity checks
    validateTopicsWithMinLeaderIsNotExcluded(optimizationOptions);
    validateEnoughLeaderToDistribute(numLeadersByTopicNames, eligibleBrokersForLeadership);
    validateBrokersAllowedReplicaMoveExist(clusterModel, optimizationOptions);
    boolean onlyMoveImmigrantReplicas = optimizationOptions.onlyMoveImmigrantReplicas();
    new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), onlyMoveImmigrantReplicas).addSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnIncludedTopics(mustHaveTopicLeadersPerBroker)).maybeAddPriorityFunc(ReplicaSortFunctionFactory.prioritizeImmigrants(), !onlyMoveImmigrantReplicas).trackSortedReplicasFor(_replicaSortName, clusterModel);
}