Examples with Broker com.linkedin.kafka.cruisecontrol.model.Broker used on opensource projects

Example 36 with Broker

use of com.linkedin.kafka.cruisecontrol.model.Broker in project cruise-control by linkedin.

the class TopicReplicaDistributionGoal method initGoalState.

/**
 * Initiates replica distribution target for each non-excluded topic in the given cluster.
 *
 * @param clusterModel The state of the cluster.
 * @param excludedTopics The topics that should be excluded from the optimization proposals.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, Set<String> excludedTopics) {
    _numRebalancedTopics = 0;
    _topicsToRebalance = new ArrayList<>(clusterModel.topics());
    if (clusterModel.deadBrokers().isEmpty()) {
        _topicsToRebalance.removeAll(excludedTopics);
    }
    if (_topicsToRebalance.isEmpty()) {
        LOG.warn("All topics are excluded from {}.", name());
        _currentRebalanceTopic = null;
    } else {
        _currentRebalanceTopic = _topicsToRebalance.get(_numRebalancedTopics);
    }
    _replicaDistributionTargetByTopic = new HashMap<>();
    Set<Broker> brokers = clusterModel.healthyBrokers();
    // Populate a map of replica distribution target by each non-excluded topic in the cluster.
    for (String topic : _topicsToRebalance) {
        ReplicaDistributionTarget replicaDistributionTarget = new ReplicaDistributionTarget(clusterModel.numTopicReplicas(topic), brokers);
        for (Broker broker : brokers) {
            replicaDistributionTarget.setBrokerEligibilityForReceivingReplica(broker.id(), broker.replicasOfTopicInBroker(topic).size());
        }
        _replicaDistributionTargetByTopic.put(topic, replicaDistributionTarget);
    }
}

Example 37 with Broker

use of com.linkedin.kafka.cruisecontrol.model.Broker in project cruise-control by linkedin.

the class KafkaAssignerDiskUsageDistributionGoal method checkAndOptimize.

/**
 * Optimize the broker if the disk usage of the broker is not within the required range.
 *
 * @param broker the broker to optimize
 * @param clusterModel the cluster model
 * @param meanDiskUsage the average disk usage of the cluster
 * @param lowerThreshold the lower limit of the disk usage for a broker
 * @param upperThreshold the upper limit of the disk usage for a broker
 * @param excludedTopics the topics to exclude from movement.
 *
 * @return true if an action has been taken to improve the disk usage of the broker, false when a broker cannot or
 * does not need to be improved further.
 */
private boolean checkAndOptimize(Broker broker, ClusterModel clusterModel, double meanDiskUsage, double lowerThreshold, double upperThreshold, Set<String> excludedTopics) {
    LOG.trace("Optimizing broker {}. BrokerDiskUsage = {}, meanDiskUsage = {}", broker, diskUsage(broker), meanDiskUsage);
    double brokerDiskUsage = diskUsage(broker);
    boolean improved = false;
    if (brokerDiskUsage > upperThreshold) {
        LOG.debug("Broker {} disk usage {} is above upper threshold of {}", broker.id(), brokerDiskUsage, upperThreshold);
        List<Broker> brokersAscend = clusterModel.sortedHealthyBrokersUnderThreshold(DISK, brokerDiskUsage);
        for (Broker toSwapWith : brokersAscend) {
            if (toSwapWith == broker || Math.abs(brokerSize(toSwapWith) - brokerSize(broker)) < BROKER_EQUALITY_DELTA) {
                continue;
            }
            if (swapReplicas(broker, toSwapWith, meanDiskUsage, clusterModel, excludedTopics)) {
                improved = true;
                break;
            }
        }
    } else if (brokerDiskUsage < lowerThreshold) {
        LOG.debug("Broker {} disk usage {} is below lower threshold of {}", broker.id(), brokerDiskUsage, lowerThreshold);
        List<Broker> brokersDescend = sortedBrokersAboveSizeDescend(clusterModel, brokerDiskUsage);
        for (Broker toSwapWith : brokersDescend) {
            if (toSwapWith == broker || Math.abs(brokerSize(toSwapWith) - brokerSize(broker)) < BROKER_EQUALITY_DELTA) {
                continue;
            }
            if (swapReplicas(broker, toSwapWith, meanDiskUsage, clusterModel, excludedTopics)) {
                improved = true;
                break;
            }
        }
    }
    return improved;
}

Example 38 with Broker

use of com.linkedin.kafka.cruisecontrol.model.Broker in project cruise-control by linkedin.

the class KafkaAssignerDiskUsageDistributionGoal method sortedBrokersAboveSizeDescend.

private List<Broker> sortedBrokersAboveSizeDescend(ClusterModel clusterModel, double aboveUsage) {
    List<Broker> sortedBrokersDescend = new ArrayList<>();
    List<Broker> sortedBrokersAscend = clusterModel.sortedHealthyBrokersUnderThreshold(DISK, Double.MAX_VALUE);
    for (int i = sortedBrokersAscend.size() - 1; i >= 0; i--) {
        Broker broker = sortedBrokersAscend.get(i);
        if (diskUsage(broker) <= aboveUsage) {
            break;
        }
        sortedBrokersDescend.add(sortedBrokersAscend.get(i));
    }
    return sortedBrokersDescend;
}

Example 39 with Broker

use of com.linkedin.kafka.cruisecontrol.model.Broker in project cruise-control by linkedin.

the class KafkaAssignerEvenRackAwareGoal method isReplicaMoveViolateRackAwareness.

private boolean isReplicaMoveViolateRackAwareness(ClusterModel clusterModel, Function<ClusterModel, Replica> sourceReplicaFunction, Function<ClusterModel, Broker> destinationBrokerFunction) {
    Replica sourceReplica = sourceReplicaFunction.apply(clusterModel);
    Broker destinationBroker = destinationBrokerFunction.apply(clusterModel);
    // Destination broker cannot be in a rack that violates rack awareness.
    Set<Broker> partitionBrokers = clusterModel.partition(sourceReplica.topicPartition()).partitionBrokers();
    partitionBrokers.remove(sourceReplica.broker());
    // Remove brokers in partition broker racks except the brokers in replica broker rack.
    for (Broker broker : partitionBrokers) {
        if (broker.rack().brokers().contains(destinationBroker)) {
            return true;
        }
    }
    return false;
}

Example 40 with Broker

use of com.linkedin.kafka.cruisecontrol.model.Broker in project cruise-control by linkedin.

the class AbstractGoal method getEligibleReplicasForSwap.

private SortedSet<Replica> getEligibleReplicasForSwap(ClusterModel clusterModel, Replica sourceReplica, SortedSet<Replica> candidateReplicasToSwapWith) {
    // CASE#1: All candidate replicas are eligible if any of the following is true:
    // (1) there are no new brokers in the cluster,
    // (2) the given candidate set contains no replicas,
    // (3) the intended swap is between replicas of new brokers,
    // (4) the intended swap is between a replica on a new broker, which originally was in the destination broker, and
    // any replica in the destination broker.
    Broker sourceBroker = sourceReplica.broker();
    Broker destinationBroker = candidateReplicasToSwapWith.isEmpty() ? null : candidateReplicasToSwapWith.first().broker();
    if (clusterModel.newBrokers().isEmpty() || destinationBroker == null || (sourceBroker.isNew() && (destinationBroker.isNew() || sourceReplica.originalBroker() == destinationBroker))) {
        return candidateReplicasToSwapWith;
    }
    // contains replicas that were originally in the source broker.
    if (destinationBroker.isNew()) {
        candidateReplicasToSwapWith.removeIf(replica -> replica.originalBroker() != sourceBroker);
        return candidateReplicasToSwapWith;
    }
    // CASE#3: No swap is possible between old brokers when there are new brokers in the cluster.
    return Collections.emptySortedSet();
}