Examples with SortedReplicasHelper com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper used on opensource projects

Example 11 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class ReplicaDistributionGoal method initGoalState.

/**
 * Initiates replica distribution goal.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    super.initGoalState(clusterModel, optimizationOptions);
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    // Sort replicas for each broker in the cluster.
    for (Broker broker : clusterModel.brokers()) {
        new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrantOrOfflineReplicas(), !clusterModel.selfHealingEligibleReplicas().isEmpty() && broker.isAlive()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).maybeAddPriorityFunc(ReplicaSortFunctionFactory.prioritizeOfflineReplicas(), !clusterModel.selfHealingEligibleReplicas().isEmpty()).maybeAddPriorityFunc(ReplicaSortFunctionFactory.prioritizeImmigrants(), !optimizationOptions.onlyMoveImmigrantReplicas()).setScoreFunc(ReplicaSortFunctionFactory.sortByMetricGroupValue(DISK.name())).trackSortedReplicasFor(replicaSortName(this, false, false), broker);
    }
}

Example 12 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class IntraBrokerDiskUsageDistributionGoal method initGoalState.

/**
 * Initialize the utilization thresholds.
 * To avoid churns, we add a balance margin to the user specified rebalance threshold. e.g. when user sets the
 * threshold to be resourceBalancePercentage, we use (resourceBalancePercentage-1)*balanceMargin instead.
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) {
    double balancePercentageWithMargin = (_balancingConstraint.resourceBalancePercentage(RESOURCE) - 1) * BALANCE_MARGIN;
    for (Broker broker : brokersToBalance(clusterModel)) {
        double averageDiskUtilization = averageDiskUtilizationPercentage(broker);
        _balanceUpperThresholdByBroker.put(broker, averageDiskUtilization * (1 + balancePercentageWithMargin));
        _balanceLowerThresholdByBroker.put(broker, averageDiskUtilization * Math.max(0, (1 - balancePercentageWithMargin)));
    }
    // Sort all the replicas for each disk based on disk utilization.
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    new SortedReplicasHelper().addSelectionFunc(ReplicaSortFunctionFactory.selectOnlineReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).addPriorityFunc(ReplicaSortFunctionFactory.prioritizeDiskImmigrants()).setScoreFunc(ReplicaSortFunctionFactory.reverseSortByMetricGroupValue(RESOURCE.name())).trackSortedReplicasFor(replicaSortName(this, true, false), clusterModel);
    new SortedReplicasHelper().addSelectionFunc(ReplicaSortFunctionFactory.selectOnlineReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).addPriorityFunc(ReplicaSortFunctionFactory.prioritizeDiskImmigrants()).setScoreFunc(ReplicaSortFunctionFactory.sortByMetricGroupValue(RESOURCE.name())).trackSortedReplicasFor(replicaSortName(this, false, false), clusterModel);
}

Example 13 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class LeaderReplicaDistributionGoal method rebalanceByMovingReplicasOut.

private boolean rebalanceByMovingReplicasOut(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, OptimizationOptions optimizationOptions) {
    long moveStartTimeMs = System.currentTimeMillis();
    // Get the eligible brokers.
    SortedSet<Broker> candidateBrokers;
    if (_fixOfflineReplicasOnly) {
        candidateBrokers = new TreeSet<>(Comparator.comparingInt((Broker b) -> b.replicas().size()).thenComparingInt(Broker::id));
        candidateBrokers.addAll(clusterModel.aliveBrokers());
    } else {
        candidateBrokers = new TreeSet<>(Comparator.comparingInt((Broker b) -> b.leaderReplicas().size()).thenComparingInt(Broker::id));
        candidateBrokers.addAll(clusterModel.aliveBrokers().stream().filter(b -> b.leaderReplicas().size() < _balanceUpperLimit).collect(Collectors.toSet()));
    }
    int balanceUpperLimit = _fixOfflineReplicasOnly ? 0 : _balanceUpperLimit;
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    String replicaSortName = replicaSortName(this, false, !_fixOfflineReplicasOnly);
    new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectLeaders(), !_fixOfflineReplicasOnly).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectOfflineReplicas(), _fixOfflineReplicasOnly).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), (!_fixOfflineReplicasOnly && !clusterModel.selfHealingEligibleReplicas().isEmpty()) || optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).trackSortedReplicasFor(replicaSortName, broker);
    SortedSet<Replica> candidateReplicas = broker.trackedSortedReplicas(replicaSortName).sortedReplicas(true);
    int numReplicas = candidateReplicas.size();
    boolean fastMode = optimizationOptions.fastMode();
    for (Replica replica : candidateReplicas) {
        if (fastMode && !replica.isCurrentOffline() && remainingTimeMs(_balancingConstraint.fastModePerBrokerMoveTimeoutMs(), moveStartTimeMs) <= 0) {
            LOG.debug("Move replicas out timeout in fast mode for broker {}.", broker.id());
            break;
        }
        Broker b = maybeApplyBalancingAction(clusterModel, replica, candidateBrokers, ActionType.INTER_BROKER_REPLICA_MOVEMENT, optimizedGoals, optimizationOptions);
        // Only check if we successfully moved something.
        if (b != null) {
            if (--numReplicas <= balanceUpperLimit) {
                broker.untrackSortedReplicas(replicaSortName);
                return false;
            }
            // Remove and reinsert the broker so the order is correct.
            candidateBrokers.remove(b);
            if (b.leaderReplicas().size() < _balanceUpperLimit || _fixOfflineReplicasOnly) {
                candidateBrokers.add(b);
            }
        }
    }
    broker.untrackSortedReplicas(replicaSortName);
    return true;
}

Example 14 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class LeaderBytesInDistributionGoal method initGoalState.

@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    _brokersAllowedReplicaMove = GoalUtils.aliveBrokersNotExcludedForReplicaMove(clusterModel, optimizationOptions);
    if (_brokersAllowedReplicaMove.isEmpty()) {
        // Handle the case when all alive brokers are excluded from replica moves.
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(clusterModel.maxReplicationFactor()).build();
        throw new OptimizationFailureException(String.format("[%s] All alive brokers are excluded from replica moves.", name()), recommendation);
    }
    _meanLeaderBytesIn = 0.0;
    _overLimitBrokerIds = new HashSet<>();
    // Sort leader replicas for each broker.
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    new SortedReplicasHelper().addSelectionFunc(ReplicaSortFunctionFactory.selectLeaders()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).setScoreFunc(ReplicaSortFunctionFactory.reverseSortByMetricGroupValue(Resource.NW_IN.toString())).trackSortedReplicasFor(replicaSortName(this, true, true), clusterModel);
}

Example 15 with SortedReplicasHelper

use of com.linkedin.kafka.cruisecontrol.model.SortedReplicasHelper in project cruise-control by linkedin.

the class ResourceDistributionGoal method rebalanceByMovingLoadOut.

private boolean rebalanceByMovingLoadOut(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, ActionType actionType, OptimizationOptions optimizationOptions) {
    long moveStartTimeMs = System.currentTimeMillis();
    Set<String> excludedTopics = optimizationOptions.excludedTopics();
    // Get the eligible brokers.
    SortedSet<Broker> candidateBrokers = new TreeSet<>(Comparator.comparingDouble((Broker b) -> utilization(b, resource())).thenComparingInt(Broker::id));
    if (_fixOfflineReplicasOnly) {
        candidateBrokers.addAll(clusterModel.aliveBrokers());
    } else {
        candidateBrokers.addAll(clusterModel.aliveBrokersUnderThreshold(resource(), _balanceUpperThreshold));
    }
    // Get the replicas to rebalance.
    new SortedReplicasHelper().maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectLeaders(), actionType == LEADERSHIP_MOVEMENT).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrants(), optimizationOptions.onlyMoveImmigrantReplicas()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectImmigrantOrOfflineReplicas(), !clusterModel.selfHealingEligibleReplicas().isEmpty() && broker.isAlive()).maybeAddSelectionFunc(ReplicaSortFunctionFactory.selectReplicasBasedOnExcludedTopics(excludedTopics), !excludedTopics.isEmpty()).maybeAddPriorityFunc(ReplicaSortFunctionFactory.prioritizeOfflineReplicas(), !clusterModel.selfHealingEligibleReplicas().isEmpty()).maybeAddPriorityFunc(ReplicaSortFunctionFactory.prioritizeImmigrants(), !optimizationOptions.onlyMoveImmigrantReplicas()).setScoreFunc(ReplicaSortFunctionFactory.reverseSortByMetricGroupValue(resource().name())).trackSortedReplicasFor(replicaSortName(this, true, actionType == LEADERSHIP_MOVEMENT), broker);
    SortedSet<Replica> replicasToMove = broker.trackedSortedReplicas(replicaSortName(this, true, actionType == LEADERSHIP_MOVEMENT)).sortedReplicas(true);
    // If the source broker is excluded for replica move, set its upper limit to 0.
    double balanceUpperThresholdForSourceBroker = isExcludedForReplicaMove(broker) ? 0 : _balanceUpperThreshold;
    boolean fastMode = optimizationOptions.fastMode();
    // Now let's move things around.
    for (Replica replica : replicasToMove) {
        if (!replica.isCurrentOffline()) {
            if (fastMode && remainingTimeMs(_balancingConstraint.fastModePerBrokerMoveTimeoutMs(), moveStartTimeMs) <= 0) {
                LOG.debug("Move load out timeout in fast mode for broker {}.", broker.id());
                break;
            }
            // It does not make sense to move an online replica without utilization from a live broker.
            if (replica.load().expectedUtilizationFor(resource()) == 0.0) {
                break;
            }
        }
        // An optimization for leader movements.
        SortedSet<Broker> eligibleBrokers;
        if (actionType == LEADERSHIP_MOVEMENT) {
            eligibleBrokers = new TreeSet<>(Comparator.comparingDouble((Broker b) -> utilization(b, resource())).thenComparingInt(Broker::id));
            clusterModel.partition(replica.topicPartition()).onlineFollowerBrokers().forEach(b -> {
                if (candidateBrokers.contains(b)) {
                    eligibleBrokers.add(b);
                }
            });
        } else {
            eligibleBrokers = candidateBrokers;
        }
        Broker b = maybeApplyBalancingAction(clusterModel, replica, eligibleBrokers, actionType, optimizedGoals, optimizationOptions);
        // Only check if we successfully moved something.
        if (b != null) {
            if (isLoadUnderBalanceUpperLimit(broker, balanceUpperThresholdForSourceBroker) && !(_fixOfflineReplicasOnly && !broker.currentOfflineReplicas().isEmpty())) {
                broker.clearSortedReplicas();
                return false;
            }
            // Remove and reinsert the broker so the order is correct.
            candidateBrokers.remove(b);
            if (utilization(b, resource()) < _balanceUpperThreshold) {
                candidateBrokers.add(b);
            }
        }
    }
    // If all the replicas has been moved away from the broker and we still reach here, that means the broker
    // capacity is negative, i.e. the broker is dead. So as long as there is no replicas on the broker anymore
    // we consider it as not over limit.
    broker.clearSortedReplicas();
    return !broker.replicas().isEmpty();
}