Examples with SortedReplicasHelper com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper used on opensource projects

Example 6 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class CapacityGoal method initGoalState.

/**
 * Sanity checks: Existing total load on cluster is less than the limiting capacity
 * determined by the total capacity of alive cluster multiplied by the capacity threshold.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    // Sanity Check -- i.e. not enough resources.
    Load recentClusterLoad = clusterModel.load();
    // While proposals exclude the excludedTopics, the existingUtilization still considers replicas of the excludedTopics.
    double existingUtilization = recentClusterLoad.expectedUtilizationFor(resource());
    double capacity = clusterModel.capacityWithAllowedReplicaMovesFor(resource(), optimizationOptions);
    double allowedCapacity = capacity * _balancingConstraint.capacityThreshold(resource());
    if (allowedCapacity < existingUtilization) {
        Set<Integer> brokersAllowedReplicaMove = GoalUtils.aliveBrokersNotExcludedForReplicaMove(clusterModel, optimizationOptions);
        if (brokersAllowedReplicaMove.isEmpty()) {
            // Handle the case when all alive brokers are excluded from replica moves.
            ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(clusterModel.maxReplicationFactor()).build();
            throw new OptimizationFailureException(String.format("[%s] All alive brokers are excluded from replica moves.", name()), recommendation);
        }
        // Identify a typical broker capacity to be used in recommendations in case the cluster is under-provisioned.
        int typicalBrokerId = brokersAllowedReplicaMove.iterator().next();
        double typicalCapacity = clusterModel.broker(typicalBrokerId).capacityFor(resource());
        double missingCapacity = existingUtilization - allowedCapacity;
        int numBrokersToAdd = (int) Math.ceil(missingCapacity / (typicalCapacity * _balancingConstraint.capacityThreshold(resource())));
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(numBrokersToAdd).typicalBrokerCapacity(typicalCapacity).typicalBrokerId(typicalBrokerId).resource(resource()).build();
        throw new OptimizationFailureException(String.format("[%s] Insufficient capacity for %s (Utilization %.2f, Allowed Capacity %.2f, Threshold: %.2f).", name(), resource(), existingUtilization, allowedCapacity, _balancingConstraint.capacityThreshold(resource())), recommendation);
    }
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    boolean onlyMoveImmigrantReplicas = optimizationOptions.onlyMoveImmigrantReplicas();
    // Sort all replicas for each broker based on resource utilization.
    new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), onlyMoveImmigrantReplicas).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).maybeAddPriorityFunc(ReplicaSortFunctionFactory.prioritizeOfflineReplicas(), !clusterModel.selfHealingEligibleReplicas().isEmpty()).maybeAddPriorityFunc(ReplicaSortFunctionFactory.prioritizeImmigrants(), !onlyMoveImmigrantReplicas).setScoreFunc(ReplicaSortFunctionFactory.reverseSortByMetricGroupValue(resource().name())).trackSortedReplicasFor(replicaSortName(this, true, false), clusterModel);
    // Sort leader replicas for each broker based on resource utilization.
    new SortedReplicasHelper().addSelectionFunc(ReplicaSortFunctionFactory.selectLeaders()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), onlyMoveImmigrantReplicas).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).maybeAddPriorityFunc(ReplicaSortFunctionFactory.prioritizeImmigrants(), !onlyMoveImmigrantReplicas).setScoreFunc(ReplicaSortFunctionFactory.reverseSortByMetricGroupValue(resource().name())).trackSortedReplicasFor(replicaSortName(this, true, true), clusterModel);
}

Example 7 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class IntraBrokerDiskCapacityGoal method initGoalState.

/**
 * Sanity checks: For each alive broker in the cluster, the load for {@link Resource#DISK} less than the limiting capacity
 * determined by the total capacity of alive disks multiplied by the capacity threshold.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    // While proposals exclude the excludedTopics, the existingUtilization still considers replicas of the excludedTopics.
    for (Broker broker : clusterModel.aliveBrokers()) {
        double existingUtilization = broker.load().expectedUtilizationFor(RESOURCE);
        double allowedCapacity = broker.capacityFor(RESOURCE) * _balancingConstraint.capacityThreshold(RESOURCE);
        if (allowedCapacity < existingUtilization) {
            double requiredCapacity = existingUtilization / _balancingConstraint.capacityThreshold(RESOURCE);
            ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(1).totalCapacity(requiredCapacity).build();
            throw new OptimizationFailureException(String.format("[%s] Insufficient disk capacity at broker %d (Utilization %.2f, Allowed " + "Capacity %.2f).", name(), broker.id(), existingUtilization, allowedCapacity), recommendation);
        }
    }
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    // Sort all the replicas for each disk based on disk utilization.
    new SortedReplicasHelper().addSelectionFunc(ReplicaSortFunctionFactory.selectOnlineReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).addPriorityFunc(ReplicaSortFunctionFactory.prioritizeDiskImmigrants()).setScoreFunc(ReplicaSortFunctionFactory.reverseSortByMetricGroupValue(RESOURCE.name())).trackSortedReplicasFor(replicaSortName(this, true, false), clusterModel);
}

Example 8 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class LeaderReplicaDistributionGoal method rebalanceByMovingLeaderReplicasIn.

private boolean rebalanceByMovingLeaderReplicasIn(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, OptimizationOptions optimizationOptions) {
    long moveStartTimeMs = System.currentTimeMillis();
    if (optimizationOptions.excludedBrokersForLeadership().contains(broker.id())) {
        return true;
    }
    PriorityQueue<Broker> eligibleBrokers = new PriorityQueue<>((b1, b2) -> {
        int result = Integer.compare(b2.leaderReplicas().size(), b1.leaderReplicas().size());
        return result == 0 ? Integer.compare(b1.id(), b2.id()) : result;
    });
    for (Broker aliveBroker : clusterModel.aliveBrokers()) {
        if (aliveBroker.leaderReplicas().size() > _balanceLowerLimit) {
            eligibleBrokers.add(aliveBroker);
        }
    }
    List<Broker> candidateBrokers = Collections.singletonList(broker);
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    boolean onlyMoveImmigrantReplicas = optimizationOptions.onlyMoveImmigrantReplicas();
    String replicaSortName = replicaSortName(this, false, true);
    new SortedReplicasHelper().addSelectionFunc(ReplicaSortFunctionFactory.selectLeaders()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), !clusterModel.brokenBrokers().isEmpty() || onlyMoveImmigrantReplicas).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).trackSortedReplicasFor(replicaSortName, clusterModel);
    int numLeaderReplicas = broker.leaderReplicas().size();
    boolean fastMode = optimizationOptions.fastMode();
    while (!eligibleBrokers.isEmpty()) {
        if (fastMode && remainingTimeMs(_balancingConstraint.fastModePerBrokerMoveTimeoutMs(), moveStartTimeMs) <= 0) {
            LOG.debug("Move leaders in timeout in fast mode for broker {}.", broker.id());
            break;
        }
        Broker sourceBroker = eligibleBrokers.poll();
        for (Replica replica : sourceBroker.trackedSortedReplicas(replicaSortName).sortedReplicas(true)) {
            Broker b = maybeApplyBalancingAction(clusterModel, replica, candidateBrokers, ActionType.INTER_BROKER_REPLICA_MOVEMENT, optimizedGoals, optimizationOptions);
            // has nothing to move in. In that case we will never reenqueue that source broker.
            if (b != null) {
                if (++numLeaderReplicas >= _balanceLowerLimit) {
                    clusterModel.untrackSortedReplicas(replicaSortName);
                    return false;
                }
                // queue, we reenqueue the source broker and switch to the next broker.
                if (!eligibleBrokers.isEmpty() && sourceBroker.leaderReplicas().size() < eligibleBrokers.peek().leaderReplicas().size()) {
                    eligibleBrokers.add(sourceBroker);
                    break;
                }
            }
        }
    }
    clusterModel.untrackSortedReplicas(replicaSortName);
    return true;
}

Example 9 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class PotentialNwOutGoal method initGoalState.

/**
 * Set the flag which indicates whether the self healing failed to relocate all offline replicas away from dead brokers
 * or the corresponding dead disks in its initial attempt. Since self healing has not been executed yet, this flag is false.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) {
    // While proposals exclude the excludedTopics, the potential nw_out still considers replicas of the excludedTopics.
    _fixOfflineReplicasOnly = false;
    // Filter out some replicas based on optimization options.
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).trackSortedReplicasFor(replicaSortName(this, false, false), clusterModel);
}

Example 10 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class RackAwareDistributionGoal method initGoalState.

/**
 * This is a hard goal; hence, the proposals are not limited to dead broker replicas in case of self-healing.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    _brokersAllowedReplicaMove = GoalUtils.aliveBrokersNotExcludedForReplicaMove(clusterModel, optimizationOptions);
    if (_brokersAllowedReplicaMove.isEmpty()) {
        // Handle the case when all alive brokers are excluded from replica moves.
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(clusterModel.maxReplicationFactor()).build();
        throw new OptimizationFailureException(String.format("[%s] All alive brokers are excluded from replica moves.", name()), recommendation);
    }
    _balanceLimit = new BalanceLimit(clusterModel, optimizationOptions);
    int numExtraRacks = _balanceLimit.numAliveRacksAllowedReplicaMoves() - clusterModel.maxReplicationFactor();
    if (numExtraRacks >= _balancingConstraint.overprovisionedMinExtraRacks()) {
        int numRacksToDrop = numExtraRacks - _balancingConstraint.overprovisionedMinExtraRacks() + 1;
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.OVER_PROVISIONED).numRacks(numRacksToDrop).build();
        _provisionResponse = new ProvisionResponse(ProvisionStatus.OVER_PROVISIONED, recommendation, name());
    }
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    // Filter out some replicas based on optimization options.
    new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).trackSortedReplicasFor(replicaSortName(this, false, false), clusterModel);
}