Examples with Broker com.linkedin.kafka.cruisecontrol.model.Broker used on opensource projects

Example 6 with Broker

use of com.linkedin.kafka.cruisecontrol.model.Broker in project cruise-control by linkedin.

the class ResourceDistributionGoal method rebalanceBySwappingLoadIn.

private boolean rebalanceBySwappingLoadIn(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, Set<String> excludedTopics) {
    if (!broker.isAlive() || broker.replicas().isEmpty()) {
        // Source broker is dead or has no replicas to swap.
        return true;
    }
    // Get the replicas to rebalance.
    SortedSet<Replica> sourceReplicas = new TreeSet<>(Comparator.comparingDouble((Replica r) -> r.load().expectedUtilizationFor(resource())).thenComparing(r -> r.topicPartition().toString()));
    sourceReplicas.addAll(broker.replicas());
    // Sort the replicas initially to avoid sorting it every time.
    PriorityQueue<CandidateBroker> candidateBrokerPQ = new PriorityQueue<>();
    for (Broker candidate : clusterModel.healthyBrokersOverThreshold(resource(), _balanceLowerThreshold)) {
        // Get candidate replicas on candidate broker to try swapping with -- sorted in the order of trial (descending load).
        double minSourceReplicaLoad = sourceReplicas.first().load().expectedUtilizationFor(resource());
        SortedSet<Replica> replicasToSwapWith = sortedCandidateReplicas(candidate, excludedTopics, minSourceReplicaLoad, false);
        CandidateBroker candidateBroker = new CandidateBroker(candidate, replicasToSwapWith, false);
        candidateBrokerPQ.add(candidateBroker);
    }
    while (!candidateBrokerPQ.isEmpty()) {
        CandidateBroker cb = candidateBrokerPQ.poll();
        SortedSet<Replica> candidateReplicasToSwapWith = cb.replicas();
        Replica swappedInReplica = null;
        Replica swappedOutReplica = null;
        for (Replica sourceReplica : sourceReplicas) {
            if (shouldExclude(sourceReplica, excludedTopics)) {
                continue;
            }
            // It does not make sense to swap replicas without utilization from a live broker.
            double sourceReplicaUtilization = sourceReplica.load().expectedUtilizationFor(resource());
            if (sourceReplicaUtilization == 0.0) {
                break;
            }
            // Try swapping the source with the candidate replicas. Get the swapped in replica if successful, null otherwise.
            Replica swappedIn = maybeApplySwapAction(clusterModel, sourceReplica, candidateReplicasToSwapWith, optimizedGoals);
            if (swappedIn != null) {
                if (isLoadAboveBalanceLowerLimit(broker)) {
                    // Successfully balanced this broker by swapping in.
                    return false;
                }
                // Add swapped in/out replica for updating the list of replicas in source broker.
                swappedInReplica = swappedIn;
                swappedOutReplica = sourceReplica;
                break;
            }
        }
        swapUpdate(swappedInReplica, swappedOutReplica, sourceReplicas, candidateReplicasToSwapWith, candidateBrokerPQ, cb);
    }
    return true;
}

Example 7 with Broker

use of com.linkedin.kafka.cruisecontrol.model.Broker in project cruise-control by linkedin.

the class ResourceDistributionGoal method rebalanceBySwappingLoadOut.

private boolean rebalanceBySwappingLoadOut(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, Set<String> excludedTopics) {
    if (!broker.isAlive()) {
        return true;
    }
    // Get the replicas to rebalance.
    SortedSet<Replica> sourceReplicas = new TreeSet<>((r1, r2) -> {
        int result = Double.compare(r2.load().expectedUtilizationFor(resource()), r1.load().expectedUtilizationFor(resource()));
        return result == 0 ? r1.topicPartition().toString().compareTo(r2.topicPartition().toString()) : result;
    });
    sourceReplicas.addAll(resource() == Resource.NW_OUT ? broker.leaderReplicas() : broker.replicas());
    // Sort the replicas initially to avoid sorting it every time.
    PriorityQueue<CandidateBroker> candidateBrokerPQ = new PriorityQueue<>();
    for (Broker candidate : clusterModel.healthyBrokersUnderThreshold(resource(), _balanceUpperThreshold).stream().filter(b -> !b.replicas().isEmpty()).collect(Collectors.toSet())) {
        // Get candidate replicas on candidate broker to try swapping with -- sorted in the order of trial (ascending load).
        double maxSourceReplicaLoad = sourceReplicas.first().load().expectedUtilizationFor(resource());
        SortedSet<Replica> replicasToSwapWith = sortedCandidateReplicas(candidate, excludedTopics, maxSourceReplicaLoad, true);
        CandidateBroker candidateBroker = new CandidateBroker(candidate, replicasToSwapWith, true);
        candidateBrokerPQ.add(candidateBroker);
    }
    while (!candidateBrokerPQ.isEmpty()) {
        CandidateBroker cb = candidateBrokerPQ.poll();
        SortedSet<Replica> candidateReplicasToSwapWith = cb.replicas();
        Replica swappedInReplica = null;
        Replica swappedOutReplica = null;
        for (Replica sourceReplica : sourceReplicas) {
            if (shouldExclude(sourceReplica, excludedTopics)) {
                continue;
            }
            // Try swapping the source with the candidate replicas. Get the swapped in replica if successful, null otherwise.
            Replica swappedIn = maybeApplySwapAction(clusterModel, sourceReplica, candidateReplicasToSwapWith, optimizedGoals);
            if (swappedIn != null) {
                if (isLoadUnderBalanceUpperLimit(broker)) {
                    // Successfully balanced this broker by swapping in.
                    return false;
                }
                // Add swapped in/out replica for updating the list of replicas in source broker.
                swappedInReplica = swappedIn;
                swappedOutReplica = sourceReplica;
                break;
            }
        }
        swapUpdate(swappedInReplica, swappedOutReplica, sourceReplicas, candidateReplicasToSwapWith, candidateBrokerPQ, cb);
    }
    return true;
}

Example 8 with Broker

use of com.linkedin.kafka.cruisecontrol.model.Broker in project cruise-control by linkedin.

the class TopicReplicaDistributionGoal method brokersToBalance.

/**
 * Get brokers that the rebalance process will go over to apply balancing actions to rep licas they contain.
 *
 * @param clusterModel The state of the cluster.
 * @return A collection of brokers that the rebalance process will go over to apply balancing actions to replicas
 * they contain.
 */
@Override
protected SortedSet<Broker> brokersToBalance(ClusterModel clusterModel) {
    if (!clusterModel.deadBrokers().isEmpty()) {
        return clusterModel.deadBrokers();
    }
    if (_currentRebalanceTopic == null) {
        return Collections.emptySortedSet();
    }
    // Brokers having over minimum number of replicas per broker for the current rebalance topic are eligible for balancing.
    SortedSet<Broker> brokersToBalance = new TreeSet<>();
    int minNumReplicasPerBroker = _replicaDistributionTargetByTopic.get(_currentRebalanceTopic).minNumReplicasPerBroker();
    brokersToBalance.addAll(clusterModel.brokers().stream().filter(broker -> broker.replicasOfTopicInBroker(_currentRebalanceTopic).size() > minNumReplicasPerBroker).collect(Collectors.toList()));
    return brokersToBalance;
}

Example 9 with Broker

use of com.linkedin.kafka.cruisecontrol.model.Broker in project cruise-control by linkedin.

the class KafkaAssignerDiskUsageDistributionGoal method optimize.

@Override
public boolean optimize(ClusterModel clusterModel, Set<Goal> optimizedGoals, Set<String> excludedTopics) {
    double meanDiskUsage = clusterModel.load().expectedUtilizationFor(DISK) / clusterModel.capacityFor(DISK);
    double upperThreshold = meanDiskUsage * (1 + balancePercentageWithMargin());
    double lowerThreshold = meanDiskUsage * Math.max(0, (1 - balancePercentageWithMargin()));
    Comparator<Broker> comparator = (b1, b2) -> {
        int result = Double.compare(diskUsage(b2), diskUsage(b1));
        return result == 0 ? Integer.compare(b1.id(), b2.id()) : result;
    };
    boolean improved;
    int numIterations = 0;
    do {
        List<Broker> brokers = new ArrayList<>();
        brokers.addAll(clusterModel.healthyBrokers());
        brokers.sort(comparator);
        improved = false;
        LOG.debug("Starting iteration {}", numIterations);
        for (Broker broker : brokers) {
            if (checkAndOptimize(broker, clusterModel, meanDiskUsage, lowerThreshold, upperThreshold, excludedTopics)) {
                improved = true;
            }
        }
        numIterations++;
    } while (improved);
    boolean succeeded = isOptimized(clusterModel, upperThreshold, lowerThreshold);
    LOG.debug("Finished optimization in {} iterations.", numIterations);
    return succeeded;
}

Example 10 with Broker

use of com.linkedin.kafka.cruisecontrol.model.Broker in project cruise-control by linkedin.

the class KafkaAssignerDiskUsageDistributionGoal method isOptimized.

/**
 * Check whether the cluster model still has brokers whose disk usage are above upper threshold or below lower
 * threshold.
 *
 * @param clusterModel the cluster model to check
 * @param upperThreshold the upper threshold of the disk usage.
 * @param lowerThreshold the lower threshold of the disk usage.
 *
 * @return true if all the brokers are within thresholds, false otherwise.
 */
private boolean isOptimized(ClusterModel clusterModel, double upperThreshold, double lowerThreshold) {
    // Check if any broker is out of the allowed usage range.
    Set<Broker> brokersAboveUpperThreshold = new HashSet<>();
    Set<Broker> brokersUnderLowerThreshold = new HashSet<>();
    for (Broker broker : clusterModel.healthyBrokers()) {
        double diskUsage = diskUsage(broker);
        if (diskUsage < lowerThreshold) {
            brokersUnderLowerThreshold.add(broker);
        } else if (diskUsage > upperThreshold) {
            brokersAboveUpperThreshold.add(broker);
        }
    }
    if (!brokersUnderLowerThreshold.isEmpty()) {
        StringJoiner joiner = new StringJoiner(", ");
        brokersUnderLowerThreshold.forEach(b -> joiner.add(String.format("%d:(%.3f)", b.id(), diskUsage(b))));
        LOG.warn("There are still {} brokers under the lower threshold of {}. The brokers are {}", brokersUnderLowerThreshold.size(), lowerThreshold, joiner.toString());
    }
    if (!brokersAboveUpperThreshold.isEmpty()) {
        StringJoiner joiner = new StringJoiner(", ");
        brokersAboveUpperThreshold.forEach(b -> joiner.add(String.format("%d:(%.3f)", b.id(), diskUsage(b))));
        LOG.warn("There are still {} brokers above the upper threshold of {}. The brokers are {}", brokersAboveUpperThreshold.size(), upperThreshold, joiner.toString());
    }
    return brokersUnderLowerThreshold.isEmpty() && brokersAboveUpperThreshold.isEmpty();
}