Examples with ProvisionRecommendation com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation used on opensource projects

Example 1 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class BasicBrokerProvisioner method executeFor.

/**
 * Determine the broker recommendation to execute using the given broker recommendations. Then execute this recommendation to add/remove brokers.
 * The determination of the broker recommendation to execute is based on picking the recommendation with the
 * <ul>
 *   <li>maximum number of brokers to add for {@link ProvisionStatus#UNDER_PROVISIONED} recommendations.</li>
 *   <li>minimum number of brokers to remove for {@link ProvisionStatus#OVER_PROVISIONED} recommendations.</li>
 * </ul>
 *
 * @param brokerRecommendations Broker recommendations by recommender based on which the broker recommendation to execute will be determined.
 * @return A {@link ProvisionerState}, indicating the result of executing the broker recommendation to add or remove brokers.
 */
protected ProvisionerState executeFor(Map<String, ProvisionRecommendation> brokerRecommendations) {
    if (brokerRecommendations.isEmpty()) {
        return null;
    }
    // 1. Identify the recommender for the recommendation to execute.
    String recommender = null;
    int numBrokersToAddOrRemove = (brokerRecommendations.values().iterator().next().status() == ProvisionStatus.UNDER_PROVISIONED) ? Integer.MIN_VALUE : Integer.MAX_VALUE;
    for (Map.Entry<String, ProvisionRecommendation> recommendationEntry : brokerRecommendations.entrySet()) {
        int recommendedNumBrokers = recommendationEntry.getValue().numBrokers();
        if (((recommendationEntry.getValue().status() == ProvisionStatus.UNDER_PROVISIONED) ? (recommendedNumBrokers > numBrokersToAddOrRemove) : (recommendedNumBrokers < numBrokersToAddOrRemove))) {
            numBrokersToAddOrRemove = recommendedNumBrokers;
            recommender = recommendationEntry.getKey();
        }
    }
    // 2. Execute the recommendation to add or remove brokers.
    ProvisionRecommendation recommendationToExecute = brokerRecommendations.get(recommender);
    LOG.info("Executing the provision recommendation: [{}] {}.", recommender, recommendationToExecute);
    return addOrRemoveBrokers(recommendationToExecute);
}

Example 2 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class RackAwareGoal method initGoalState.

/**
 * This is a hard goal; hence, the proposals are not limited to dead broker replicas in case of self-healing.
 * Sanity Check: There exists sufficient number of racks for achieving rack-awareness.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    // Sanity Check: not enough racks to satisfy rack awareness.
    int numAliveRacks = clusterModel.numAliveRacks();
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    if (!excludedTopics.isEmpty()) {
        int maxReplicationFactorOfIncludedTopics = 1;
        Map<String, Integer> replicationFactorByTopic = clusterModel.replicationFactorByTopic();
        for (Map.Entry<String, Integer> replicationFactorByTopicEntry : replicationFactorByTopic.entrySet()) {
            if (!excludedTopics.contains(replicationFactorByTopicEntry.getKey())) {
                maxReplicationFactorOfIncludedTopics = Math.max(maxReplicationFactorOfIncludedTopics, replicationFactorByTopicEntry.getValue());
                if (maxReplicationFactorOfIncludedTopics > numAliveRacks) {
                    int missingRacks = maxReplicationFactorOfIncludedTopics - numAliveRacks;
                    ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numRacks(missingRacks).build();
                    throw new OptimizationFailureException(String.format("[%s] Insufficient number of racks to distribute included replicas (Current: %d, Needed: %d).", name(), numAliveRacks, maxReplicationFactorOfIncludedTopics), recommendation);
                }
            }
        }
    } else if (clusterModel.maxReplicationFactor() > numAliveRacks) {
        int missingRacks = clusterModel.maxReplicationFactor() - numAliveRacks;
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numRacks(missingRacks).build();
        throw new OptimizationFailureException(String.format("[%s] Insufficient number of racks to distribute each replica (Current: %d, Needed: %d).", name(), numAliveRacks, clusterModel.maxReplicationFactor()), recommendation);
    }
    int numExtraRacks = numAliveRacks - clusterModel.maxReplicationFactor();
    if (numExtraRacks >= _balancingConstraint.overprovisionedMinExtraRacks()) {
        int numRacksToDrop = numExtraRacks - _balancingConstraint.overprovisionedMinExtraRacks() + 1;
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.OVER_PROVISIONED).numRacks(numRacksToDrop).build();
        _provisionResponse = new ProvisionResponse(ProvisionStatus.OVER_PROVISIONED, recommendation, name());
    }
    // Filter out some replicas based on optimization options.
    new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).trackSortedReplicasFor(replicaSortName(this, false, false), clusterModel);
}

Example 3 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class ReplicaDistributionAbstractGoal method initGoalState.

/**
 * Initiates replica distribution abstract goal.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    _brokersAllowedReplicaMove = GoalUtils.aliveBrokersNotExcludedForReplicaMove(clusterModel, optimizationOptions);
    if (_brokersAllowedReplicaMove.isEmpty()) {
        // Handle the case when all alive brokers are excluded from replica moves.
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(clusterModel.maxReplicationFactor()).build();
        throw new OptimizationFailureException(String.format("[%s] All alive brokers are excluded from replica moves.", name()), recommendation);
    }
    // Initialize the average replicas on an alive broker.
    _avgReplicasOnAliveBroker = numInterestedReplicas(clusterModel) / (double) _brokersAllowedReplicaMove.size();
    // Log a warning if all replicas are excluded.
    if (clusterModel.topics().equals(optimizationOptions.excludedTopics())) {
        LOG.warn("All replicas are excluded from {}.", name());
    }
    _fixOfflineReplicasOnly = false;
    _balanceUpperLimit = balanceUpperLimit(optimizationOptions, balancePercentage());
    _balanceLowerLimit = balanceLowerLimit(optimizationOptions, balancePercentage());
}

Example 4 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class TopicReplicaDistributionGoal method initGoalState.

/**
 * Initiates this goal.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    Set<String> topicsToRebalance = GoalUtils.topicsToRebalance(clusterModel, excludedTopics);
    if (topicsToRebalance.isEmpty()) {
        LOG.warn("All topics are excluded from {}.", name());
    }
    _brokersAllowedReplicaMove = GoalUtils.aliveBrokersNotExcludedForReplicaMove(clusterModel, optimizationOptions);
    if (_brokersAllowedReplicaMove.isEmpty()) {
        // Handle the case when all alive brokers are excluded from replica moves.
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(clusterModel.maxReplicationFactor()).build();
        throw new OptimizationFailureException(String.format("[%s] All alive brokers are excluded from replica moves.", name()), recommendation);
    }
    // Initialize the average replicas on an alive broker.
    for (String topic : clusterModel.topics()) {
        int numTopicReplicas = clusterModel.numTopicReplicas(topic);
        _avgTopicReplicasOnAliveBroker.put(topic, (numTopicReplicas / (double) _brokersAllowedReplicaMove.size()));
        _balanceUpperLimitByTopic.put(topic, balanceUpperLimit(topic, optimizationOptions));
        _balanceLowerLimitByTopic.put(topic, balanceLowerLimit(topic, optimizationOptions));
        // Retain only the topics to rebalance in _avgTopicReplicasOnAliveBroker
        if (!topicsToRebalance.contains(topic)) {
            _avgTopicReplicasOnAliveBroker.remove(topic);
        }
    }
    // Filter out replicas to be considered for replica movement.
    for (Broker broker : clusterModel.brokers()) {
        new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrantOrOfflineReplicas(), !clusterModel.selfHealingEligibleReplicas().isEmpty() && broker.isAlive()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).trackSortedReplicasFor(replicaSortName(this, false, false), broker);
    }
    _fixOfflineReplicasOnly = false;
}

Example 5 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class ReplicaCapacityGoal method initGoalState.

/**
 * This is a hard goal; hence, the proposals are not limited to dead broker replicas in case of self-healing.
 * Sanity Check: Each node has sufficient number of replicas that can be moved to satisfy the replica capacity goal.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    List<String> topicsToRebalance = new ArrayList<>(clusterModel.topics());
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    topicsToRebalance.removeAll(excludedTopics);
    if (topicsToRebalance.isEmpty()) {
        LOG.warn("All topics are excluded from {}.", name());
    }
    // Sanity check: excluded topic replicas in a broker cannot exceed the max number of allowed replicas per broker.
    int totalReplicasInCluster = 0;
    for (Broker broker : brokersToBalance(clusterModel)) {
        // Calculate total number of replicas for the next sanity check.
        totalReplicasInCluster += broker.replicas().size();
        if (!broker.isAlive()) {
            _isSelfHealingMode = true;
            continue;
        }
        Set<Replica> excludedReplicasInBroker = new HashSet<>();
        for (String topic : excludedTopics) {
            excludedReplicasInBroker.addAll(broker.replicasOfTopicInBroker(topic));
        }
        if (broker.state() == Broker.State.BAD_DISKS) {
            _isSelfHealingMode = true;
            excludedReplicasInBroker.removeAll(broker.currentOfflineReplicas());
        }
        if (excludedReplicasInBroker.size() > _balancingConstraint.maxReplicasPerBroker()) {
            throw new OptimizationFailureException(String.format("[%s] Replicas of excluded topics in broker: %d exceeds the maximum allowed number of replicas per broker: %d.", name(), excludedReplicasInBroker.size(), _balancingConstraint.maxReplicasPerBroker()));
        }
    }
    // Sanity check: total replicas in the cluster cannot be more than the allowed replicas in the cluster.
    Set<Integer> brokersAllowedReplicaMove = GoalUtils.aliveBrokersNotExcludedForReplicaMove(clusterModel, optimizationOptions);
    long maxReplicasInCluster = _balancingConstraint.maxReplicasPerBroker() * brokersAllowedReplicaMove.size();
    if (totalReplicasInCluster > maxReplicasInCluster) {
        int minRequiredBrokers = (int) Math.ceil(totalReplicasInCluster / (double) _balancingConstraint.maxReplicasPerBroker());
        int numBrokersToAdd = minRequiredBrokers - brokersAllowedReplicaMove.size();
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(numBrokersToAdd).build();
        throw new OptimizationFailureException(String.format("[%s] Total replicas in cluster: %d exceeds the maximum allowed replicas in cluster: %d (Alive " + "brokers: %d, Allowed number of replicas per broker: %d).", name(), totalReplicasInCluster, maxReplicasInCluster, clusterModel.aliveBrokers().size(), _balancingConstraint.maxReplicasPerBroker()), recommendation);
    }
    // Filter out some replicas based on optimization options.
    new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).trackSortedReplicasFor(replicaSortName(this, false, false), clusterModel);
}