Examples with Resource com.linkedin.kafka.cruisecontrol.common.Resource used on opensource projects

Example 21 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class ClusterModelStats method utilizationForResources.

/**
 * Generate statistics of utilization for resources in the given cluster.
 *
 * @param clusterModel The state of the cluster.
 */
private void utilizationForResources(ClusterModel clusterModel) {
    // Average, maximum, and standard deviation of utilization by resource.
    Map<Resource, Double> avgUtilizationByResource = new HashMap<>();
    Map<Resource, Double> maxUtilizationByResource = new HashMap<>();
    Map<Resource, Double> minUtilizationByResource = new HashMap<>();
    Map<Resource, Double> stDevUtilizationByResource = new HashMap<>();
    for (Resource resource : Resource.values()) {
        double balanceUpperThreshold = (clusterModel.load().expectedUtilizationFor(resource) / clusterModel.capacityFor(resource)) * _balancingConstraint.resourceBalancePercentage(resource);
        double balanceLowerThreshold = (clusterModel.load().expectedUtilizationFor(resource) / clusterModel.capacityFor(resource)) * Math.max(0, (2 - _balancingConstraint.resourceBalancePercentage(resource)));
        // Average utilization for the resource.
        double avgUtilization = clusterModel.load().expectedUtilizationFor(resource) / _numBrokers;
        avgUtilizationByResource.put(resource, avgUtilization);
        // Maximum, minimum, and standard deviation utilization for the resource.
        double hottestBrokerUtilization = 0.0;
        double coldestBrokerUtilization = Double.MAX_VALUE;
        double variance = 0.0;
        int numBalancedBrokers = 0;
        for (Broker broker : clusterModel.brokers()) {
            double utilization = resource.isHostResource() ? broker.host().load().expectedUtilizationFor(resource) : broker.load().expectedUtilizationFor(resource);
            double balanceUpperLimit = resource.isHostResource() ? broker.host().capacityFor(resource) * balanceUpperThreshold : broker.capacityFor(resource) * balanceUpperThreshold;
            double balanceLowerLimit = resource.isHostResource() ? broker.host().capacityFor(resource) * balanceLowerThreshold : broker.capacityFor(resource) * balanceLowerThreshold;
            if (utilization <= balanceUpperLimit && utilization >= balanceLowerLimit) {
                numBalancedBrokers++;
            }
            double brokerUtilization = broker.load().expectedUtilizationFor(resource);
            hottestBrokerUtilization = Math.max(hottestBrokerUtilization, brokerUtilization);
            coldestBrokerUtilization = Math.min(coldestBrokerUtilization, brokerUtilization);
            variance += (Math.pow(brokerUtilization - avgUtilization, 2) / _numBrokers);
        }
        _numBalancedBrokersByResource.put(resource, numBalancedBrokers);
        maxUtilizationByResource.put(resource, hottestBrokerUtilization);
        minUtilizationByResource.put(resource, coldestBrokerUtilization);
        stDevUtilizationByResource.put(resource, Math.sqrt(variance));
    }
    _resourceUtilizationStats.put(Statistic.AVG, avgUtilizationByResource);
    _resourceUtilizationStats.put(Statistic.MAX, maxUtilizationByResource);
    _resourceUtilizationStats.put(Statistic.MIN, minUtilizationByResource);
    _resourceUtilizationStats.put(Statistic.ST_DEV, stDevUtilizationByResource);
}

Example 22 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class ClusterModelStats method toString.

@Override
public String toString() {
    StringBuilder sb = new StringBuilder();
    sb.append(String.format("brokers:%d replicas:%d topics:%d%n", numBrokers(), numReplicasInCluster(), numTopics()));
    for (Statistic stat : Statistic.values()) {
        sb.append(String.format("%s:{", stat));
        for (Resource resource : Resource.values()) {
            sb.append(String.format("%s:%12.3f ", resource, resourceUtilizationStats().get(stat).get(resource)));
        }
        sb.append(String.format("potentialNwOut:%12.3f replicas:%s topicReplicas:%s}%n", potentialNwOutUtilizationStats().get(stat), replicaStats().get(stat), topicReplicaStats().get(stat)));
    }
    return sb.substring(0, sb.length() - 2);
}

Example 23 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class Host method setBrokerState.

/**
 * Set broker state and update the capacity
 */
void setBrokerState(int brokerId, Broker.State newState) {
    Broker broker = broker(brokerId);
    if (broker.isAlive() && newState == Broker.State.DEAD) {
        for (Resource r : Resource.values()) {
            _hostCapacity[r.id()] -= broker.capacityFor(r);
        }
        _aliveBrokers--;
    } else if (!broker.isAlive() && newState != Broker.State.DEAD) {
        for (Resource r : Resource.values()) {
            _hostCapacity[r.id()] += broker.capacityFor(r);
        }
        _aliveBrokers++;
    }
    broker.setState(newState);
}

Example 24 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class CapacityGoal method rebalanceForBroker.

/**
 * (1) REBALANCE BY LEADERSHIP MOVEMENT:
 * Perform leadership movement to ensure that the load on brokers and/or hosts (see {@link Resource#isHostResource()}
 * and {@link Resource#isBrokerResource()}) for the outbound network load and CPU is under the capacity limit.
 *
 * <p>
 * (2) REBALANCE BY REPLICA MOVEMENT:
 * Perform optimization via replica movement for the given resource to ensure rebalance: The load on brokers and/or
 * hosts (see {@link Resource#isHostResource()} and {@link Resource#isBrokerResource()}) for the given resource is
 * under the capacity limit.
 *
 * @param broker         Broker to be balanced.
 * @param clusterModel   The state of the cluster.
 * @param optimizedGoals Optimized goals.
 * @param excludedTopics The topics that should be excluded from the optimization action.
 */
@Override
protected void rebalanceForBroker(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, Set<String> excludedTopics) throws OptimizationFailureException {
    LOG.debug("balancing broker {}, optimized goals = {}", broker, optimizedGoals);
    Resource currentResource = resource();
    double capacityThreshold = _balancingConstraint.capacityThreshold(currentResource);
    double brokerCapacityLimit = broker.capacityFor(currentResource) * capacityThreshold;
    double hostCapacityLimit = broker.host().capacityFor(currentResource) * capacityThreshold;
    boolean isUtilizationOverLimit = isUtilizationOverLimit(broker, currentResource, brokerCapacityLimit, hostCapacityLimit);
    if (!isUtilizationOverLimit) {
        // The utilization of source broker and/or host for the current resource is already under the capacity limit.
        return;
    }
    // First try REBALANCE BY LEADERSHIP MOVEMENT:
    if (currentResource == Resource.NW_OUT || currentResource == Resource.CPU) {
        // Sort replicas by descending order of preference to relocate. Preference is based on resource cost.
        // Only leaders in the source broker are sorted.
        List<Replica> sortedLeadersInSourceBroker = broker.sortedLeadersFor(currentResource);
        for (Replica leader : sortedLeadersInSourceBroker) {
            if (shouldExclude(leader, excludedTopics)) {
                continue;
            }
            // Get followers of this leader and sort them in ascending order by their broker resource utilization.
            List<Replica> followers = clusterModel.partition(leader.topicPartition()).followers();
            clusterModel.sortReplicasInAscendingOrderByBrokerResourceUtilization(followers, currentResource);
            List<Broker> eligibleBrokers = followers.stream().map(Replica::broker).collect(Collectors.toList());
            Broker b = maybeApplyBalancingAction(clusterModel, leader, eligibleBrokers, ActionType.LEADERSHIP_MOVEMENT, optimizedGoals);
            if (b == null) {
                LOG.debug("Failed to move leader replica {} to any other brokers in {}", leader, eligibleBrokers);
            }
            isUtilizationOverLimit = isUtilizationOverLimit(broker, currentResource, brokerCapacityLimit, hostCapacityLimit);
            // Broker utilization has successfully been reduced under the capacity limit for the current resource.
            if (!isUtilizationOverLimit) {
                break;
            }
        }
    }
    // If leader movement did not work, move replicas.
    if (isUtilizationOverLimit) {
        // Get sorted healthy brokers under host and/or broker capacity limit (depending on the current resource).
        List<Broker> sortedHealthyBrokersUnderCapacityLimit = clusterModel.sortedHealthyBrokersUnderThreshold(currentResource, capacityThreshold);
        // be satisfied, throw an exception.
        for (Replica replica : broker.sortedReplicas(currentResource)) {
            if (shouldExclude(replica, excludedTopics)) {
                continue;
            }
            // Unless the target broker would go over the host- and/or broker-level capacity,
            // the movement will be successful.
            Broker b = maybeApplyBalancingAction(clusterModel, replica, sortedHealthyBrokersUnderCapacityLimit, ActionType.REPLICA_MOVEMENT, optimizedGoals);
            if (b == null) {
                LOG.debug("Failed to move replica {} to any broker in {}", replica, sortedHealthyBrokersUnderCapacityLimit);
            }
            // If capacity limit was not satisfied before, check if it is satisfied now.
            isUtilizationOverLimit = isUtilizationOverLimit(broker, currentResource, brokerCapacityLimit, hostCapacityLimit);
            // Broker utilization has successfully been reduced under the capacity limit for the current resource.
            if (!isUtilizationOverLimit) {
                break;
            }
        }
    }
    if (isUtilizationOverLimit) {
        if (!currentResource.isHostResource()) {
            // Utilization is above the capacity limit after all replicas in the given source broker were checked.
            throw new OptimizationFailureException("Violated capacity limit of " + brokerCapacityLimit + " via broker " + "utilization of " + broker.load().expectedUtilizationFor(currentResource) + " with broker id " + broker.id() + " for resource " + currentResource);
        } else {
            throw new OptimizationFailureException("Violated capacity limit of " + hostCapacityLimit + " via host " + "utilization of " + broker.host().load().expectedUtilizationFor(currentResource) + " with hostname " + broker.host().name() + " for resource " + currentResource);
        }
    }
}

Example 25 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class CapacityGoal method isUtilizationUnderLimitAfterAddingLoad.

/**
 * Check whether the additional load on the destination makes the host (for host resources) or broker (for broker
 * resources) go out of the capacity limit.
 *
 * @param destinationBroker Destination broker.
 * @param replicaUtilization Replica utilization for the given resource.
 * @return True if utilization is equal or above the capacity limit, false otherwise.
 */
private boolean isUtilizationUnderLimitAfterAddingLoad(Broker destinationBroker, double replicaUtilization) {
    Resource resource = resource();
    double capacityThreshold = _balancingConstraint.capacityThreshold(resource);
    // Host-level violation check.
    if (resource.isHostResource()) {
        double utilization = destinationBroker.host().load().expectedUtilizationFor(resource);
        double capacityLimit = destinationBroker.host().capacityFor(resource) * capacityThreshold;
        if (utilization + replicaUtilization >= capacityLimit) {
            // The utilization of the host for the resource is over the capacity limit.
            return false;
        }
    }
    // Broker-level violation check.
    if (resource.isBrokerResource()) {
        double utilization = destinationBroker.load().expectedUtilizationFor(resource);
        double capacityLimit = destinationBroker.capacityFor(resource) * capacityThreshold;
        return utilization + replicaUtilization < capacityLimit;
    }
    // Utilization would be under the limit after adding the load to the destination broker.
    return true;
}