Examples with ProvisionRecommendation com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation used on opensource projects

Example 11 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class RackAwareDistributionGoal method initGoalState.

/**
 * This is a hard goal; hence, the proposals are not limited to dead broker replicas in case of self-healing.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    _brokersAllowedReplicaMove = GoalUtils.aliveBrokersNotExcludedForReplicaMove(clusterModel, optimizationOptions);
    if (_brokersAllowedReplicaMove.isEmpty()) {
        // Handle the case when all alive brokers are excluded from replica moves.
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(clusterModel.maxReplicationFactor()).build();
        throw new OptimizationFailureException(String.format("[%s] All alive brokers are excluded from replica moves.", name()), recommendation);
    }
    _balanceLimit = new BalanceLimit(clusterModel, optimizationOptions);
    int numExtraRacks = _balanceLimit.numAliveRacksAllowedReplicaMoves() - clusterModel.maxReplicationFactor();
    if (numExtraRacks >= _balancingConstraint.overprovisionedMinExtraRacks()) {
        int numRacksToDrop = numExtraRacks - _balancingConstraint.overprovisionedMinExtraRacks() + 1;
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.OVER_PROVISIONED).numRacks(numRacksToDrop).build();
        _provisionResponse = new ProvisionResponse(ProvisionStatus.OVER_PROVISIONED, recommendation, name());
    }
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    // Filter out some replicas based on optimization options.
    new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).trackSortedReplicasFor(replicaSortName(this, false, false), clusterModel);
}

Example 12 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class RackAwareDistributionGoal method ensureRackAwareDistribution.

/**
 * Ensures that replicas of all partitions in the cluster satisfy rack-aware distribution.
 *
 * @param clusterModel A cluster model with no offline replicas.
 * @param optimizationOptions Options to take into account during optimization.
 */
private void ensureRackAwareDistribution(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    // Sanity check to confirm that replicas of each partition are evenly distributed across the racks.
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    for (Replica leader : clusterModel.leaderReplicas()) {
        if (excludedTopics.contains(leader.topicPartition().topic())) {
            continue;
        }
        Set<Broker> partitionBrokers = clusterModel.partition(leader.topicPartition()).partitionBrokers();
        Map<String, Integer> numReplicasByRack = numPartitionReplicasByRackId(partitionBrokers);
        int maxNumReplicasInARack = max(numReplicasByRack.values());
        // Check if rack(s) have multiple replicas from the same partition (i.e. otherwise the distribution is rack-aware).
        if (maxNumReplicasInARack > 1) {
            // Check whether there are alive racks allowed replica moves with (1) no replicas despite having RF > number of
            // alive racks allowed replica moves or (2) more replicas that they could have been placed into other racks.
            boolean someAliveRacksHaveNoReplicas = numReplicasByRack.size() < _balanceLimit.numAliveRacksAllowedReplicaMoves();
            if (someAliveRacksHaveNoReplicas || maxNumReplicasInARack - min(numReplicasByRack.values()) > 1) {
                Set<String> excludedRackIds = new HashSet<>();
                if (someAliveRacksHaveNoReplicas) {
                    // Exclude all racks containing replicas of this partition.
                    excludedRackIds.addAll(numReplicasByRack.keySet());
                } else {
                    for (Map.Entry<String, Integer> entry : numReplicasByRack.entrySet()) {
                        if (entry.getValue() == maxNumReplicasInARack) {
                            // Racks except for the one that hosts the maximum number of replicas are available for hosting more replicas.
                            excludedRackIds.add(entry.getKey());
                        }
                    }
                }
                ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(1).excludedRackIds(excludedRackIds).build();
                throw new OptimizationFailureException(String.format("[%s] Partition %s is not rack-aware. Brokers (%s) and replicas per rack (%s).", name(), leader.topicPartition(), partitionBrokers, numReplicasByRack), recommendation);
            }
        }
    }
}

Example 13 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class ReplicaDistributionGoal method updateGoalState.

@Override
protected void updateGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    super.updateGoalState(clusterModel, optimizationOptions);
    Integer numBrokersToDrop = numBrokersToDrop(clusterModel);
    if (numBrokersToDrop != null) {
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.OVER_PROVISIONED).numBrokers(numBrokersToDrop).build();
        _provisionResponse = new ProvisionResponse(ProvisionStatus.OVER_PROVISIONED, recommendation, name());
    } else if (_succeeded) {
        // The cluster is not overprovisioned and all brokers are within the upper and lower balance limits.
        _provisionResponse = new ProvisionResponse(ProvisionStatus.RIGHT_SIZED);
    }
}

Example 14 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class LeaderBytesInDistributionGoal method initGoalState.

@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    _brokersAllowedReplicaMove = GoalUtils.aliveBrokersNotExcludedForReplicaMove(clusterModel, optimizationOptions);
    if (_brokersAllowedReplicaMove.isEmpty()) {
        // Handle the case when all alive brokers are excluded from replica moves.
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(clusterModel.maxReplicationFactor()).build();
        throw new OptimizationFailureException(String.format("[%s] All alive brokers are excluded from replica moves.", name()), recommendation);
    }
    _meanLeaderBytesIn = 0.0;
    _overLimitBrokerIds = new HashSet<>();
    // Sort leader replicas for each broker.
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    new SortedReplicasHelper().addSelectionFunc(ReplicaSortFunctionFactory.selectLeaders()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).setScoreFunc(ReplicaSortFunctionFactory.reverseSortByMetricGroupValue(Resource.NW_IN.toString())).trackSortedReplicasFor(replicaSortName(this, true, true), clusterModel);
}

Example 15 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class MinTopicLeadersPerBrokerGoal method moveAwayOfflineReplicas.

private void moveAwayOfflineReplicas(Broker srcBroker, ClusterModel clusterModel, Set<Goal> optimizedGoals, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    if (srcBroker.currentOfflineReplicas().isEmpty()) {
        return;
    }
    SortedSet<Broker> eligibleBrokersToMoveOfflineReplicasTo = new TreeSet<>(Comparator.comparingInt((Broker broker) -> broker.replicas().size()).thenComparingInt(Broker::id));
    eligibleBrokersToMoveOfflineReplicasTo.addAll(clusterModel.aliveBrokers());
    Set<Replica> offlineReplicas = new HashSet<>(srcBroker.currentOfflineReplicas());
    for (Replica offlineReplica : offlineReplicas) {
        if (maybeApplyBalancingAction(clusterModel, offlineReplica, eligibleBrokersToMoveOfflineReplicasTo, INTER_BROKER_REPLICA_MOVEMENT, optimizedGoals, optimizationOptions) == null) {
            ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(1).build();
            throw new OptimizationFailureException(String.format("[%s] Cannot remove %s from %s broker %d (has %d replicas).", name(), offlineReplica, srcBroker.state(), srcBroker.id(), srcBroker.replicas().size()), recommendation);
        }
    }
}