Examples with Resource com.linkedin.kafka.cruisecontrol.common.Resource used on opensource projects

Example 1 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class ResourceDistributionGoal method rebalanceBySwappingLoadIn.

private boolean rebalanceBySwappingLoadIn(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, Set<String> excludedTopics) {
    if (!broker.isAlive() || broker.replicas().isEmpty()) {
        // Source broker is dead or has no replicas to swap.
        return true;
    }
    // Get the replicas to rebalance.
    SortedSet<Replica> sourceReplicas = new TreeSet<>(Comparator.comparingDouble((Replica r) -> r.load().expectedUtilizationFor(resource())).thenComparing(r -> r.topicPartition().toString()));
    sourceReplicas.addAll(broker.replicas());
    // Sort the replicas initially to avoid sorting it every time.
    PriorityQueue<CandidateBroker> candidateBrokerPQ = new PriorityQueue<>();
    for (Broker candidate : clusterModel.healthyBrokersOverThreshold(resource(), _balanceLowerThreshold)) {
        // Get candidate replicas on candidate broker to try swapping with -- sorted in the order of trial (descending load).
        double minSourceReplicaLoad = sourceReplicas.first().load().expectedUtilizationFor(resource());
        SortedSet<Replica> replicasToSwapWith = sortedCandidateReplicas(candidate, excludedTopics, minSourceReplicaLoad, false);
        CandidateBroker candidateBroker = new CandidateBroker(candidate, replicasToSwapWith, false);
        candidateBrokerPQ.add(candidateBroker);
    }
    while (!candidateBrokerPQ.isEmpty()) {
        CandidateBroker cb = candidateBrokerPQ.poll();
        SortedSet<Replica> candidateReplicasToSwapWith = cb.replicas();
        Replica swappedInReplica = null;
        Replica swappedOutReplica = null;
        for (Replica sourceReplica : sourceReplicas) {
            if (shouldExclude(sourceReplica, excludedTopics)) {
                continue;
            }
            // It does not make sense to swap replicas without utilization from a live broker.
            double sourceReplicaUtilization = sourceReplica.load().expectedUtilizationFor(resource());
            if (sourceReplicaUtilization == 0.0) {
                break;
            }
            // Try swapping the source with the candidate replicas. Get the swapped in replica if successful, null otherwise.
            Replica swappedIn = maybeApplySwapAction(clusterModel, sourceReplica, candidateReplicasToSwapWith, optimizedGoals);
            if (swappedIn != null) {
                if (isLoadAboveBalanceLowerLimit(broker)) {
                    // Successfully balanced this broker by swapping in.
                    return false;
                }
                // Add swapped in/out replica for updating the list of replicas in source broker.
                swappedInReplica = swappedIn;
                swappedOutReplica = sourceReplica;
                break;
            }
        }
        swapUpdate(swappedInReplica, swappedOutReplica, sourceReplicas, candidateReplicasToSwapWith, candidateBrokerPQ, cb);
    }
    return true;
}

Example 2 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class ResourceDistributionGoal method rebalanceBySwappingLoadOut.

private boolean rebalanceBySwappingLoadOut(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, Set<String> excludedTopics) {
    if (!broker.isAlive()) {
        return true;
    }
    // Get the replicas to rebalance.
    SortedSet<Replica> sourceReplicas = new TreeSet<>((r1, r2) -> {
        int result = Double.compare(r2.load().expectedUtilizationFor(resource()), r1.load().expectedUtilizationFor(resource()));
        return result == 0 ? r1.topicPartition().toString().compareTo(r2.topicPartition().toString()) : result;
    });
    sourceReplicas.addAll(resource() == Resource.NW_OUT ? broker.leaderReplicas() : broker.replicas());
    // Sort the replicas initially to avoid sorting it every time.
    PriorityQueue<CandidateBroker> candidateBrokerPQ = new PriorityQueue<>();
    for (Broker candidate : clusterModel.healthyBrokersUnderThreshold(resource(), _balanceUpperThreshold).stream().filter(b -> !b.replicas().isEmpty()).collect(Collectors.toSet())) {
        // Get candidate replicas on candidate broker to try swapping with -- sorted in the order of trial (ascending load).
        double maxSourceReplicaLoad = sourceReplicas.first().load().expectedUtilizationFor(resource());
        SortedSet<Replica> replicasToSwapWith = sortedCandidateReplicas(candidate, excludedTopics, maxSourceReplicaLoad, true);
        CandidateBroker candidateBroker = new CandidateBroker(candidate, replicasToSwapWith, true);
        candidateBrokerPQ.add(candidateBroker);
    }
    while (!candidateBrokerPQ.isEmpty()) {
        CandidateBroker cb = candidateBrokerPQ.poll();
        SortedSet<Replica> candidateReplicasToSwapWith = cb.replicas();
        Replica swappedInReplica = null;
        Replica swappedOutReplica = null;
        for (Replica sourceReplica : sourceReplicas) {
            if (shouldExclude(sourceReplica, excludedTopics)) {
                continue;
            }
            // Try swapping the source with the candidate replicas. Get the swapped in replica if successful, null otherwise.
            Replica swappedIn = maybeApplySwapAction(clusterModel, sourceReplica, candidateReplicasToSwapWith, optimizedGoals);
            if (swappedIn != null) {
                if (isLoadUnderBalanceUpperLimit(broker)) {
                    // Successfully balanced this broker by swapping in.
                    return false;
                }
                // Add swapped in/out replica for updating the list of replicas in source broker.
                swappedInReplica = swappedIn;
                swappedOutReplica = sourceReplica;
                break;
            }
        }
        swapUpdate(swappedInReplica, swappedOutReplica, sourceReplicas, candidateReplicasToSwapWith, candidateBrokerPQ, cb);
    }
    return true;
}

Example 3 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class AnalyzerUtils method getJsonStructure.

/*
   * Return an object that can be further used
   * to encode into JSON
   *
   * @param clusterModelStats Cluster model stats.
   */
public static Map<String, Object> getJsonStructure(ClusterModelStats clusterModelStats) {
    Map<String, Object> clusterStatsMap = new HashMap<>();
    clusterStatsMap.put("brokers", clusterModelStats.numBrokers());
    clusterStatsMap.put("replicas", clusterModelStats.numReplicasInCluster());
    clusterStatsMap.put("topics", clusterModelStats.numTopics());
    Map<Statistic, Map<Resource, Double>> resourceUtilizationStats = clusterModelStats.resourceUtilizationStats();
    Map<Statistic, Double> nwOutUtilizationStats = clusterModelStats.potentialNwOutUtilizationStats();
    Map<Statistic, Number> replicaStats = clusterModelStats.replicaStats();
    Map<Statistic, Number> topicReplicaStats = clusterModelStats.topicReplicaStats();
    Map<String, Object> statisticMap = new HashMap<>();
    for (Statistic stat : Statistic.values()) {
        Map<String, Double> resourceMap = new HashMap<>();
        for (Resource resource : Resource.values()) {
            resourceMap.put(resource.resource(), resourceUtilizationStats.get(stat).get(resource));
        }
        resourceMap.put("potentialNwOut", nwOutUtilizationStats.get(stat));
        resourceMap.put("replicas", replicaStats.get(stat).doubleValue());
        resourceMap.put("topicReplicas", topicReplicaStats.get(stat).doubleValue());
        statisticMap.put(stat.stat(), resourceMap);
    }
    clusterStatsMap.put("statistics", statisticMap);
    return clusterStatsMap;
}

Example 4 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class CapacityGoal method ensureUtilizationUnderCapacity.

/**
 * Ensure that for the resource, the utilization is under the capacity of the host/broker-level.
 * {@link Resource#_isBrokerResource} and {@link Resource#isHostResource()} determines the level of checks this
 * function performs.
 * @param clusterModel Cluster model.
 */
private void ensureUtilizationUnderCapacity(ClusterModel clusterModel) throws OptimizationFailureException {
    Resource resource = resource();
    double capacityThreshold = _balancingConstraint.capacityThreshold(resource);
    for (Broker broker : clusterModel.brokers()) {
        // Host-level violation check.
        if (resource.isHostResource()) {
            double utilization = broker.host().load().expectedUtilizationFor(resource);
            double capacityLimit = broker.host().capacityFor(resource) * capacityThreshold;
            if (!broker.host().replicas().isEmpty() && utilization > capacityLimit) {
                // The utilization of the host for the resource is over the capacity limit.
                throw new OptimizationFailureException(String.format("Optimization for goal %s failed because %s utilization " + "for host %s is %f which is above capacity limit %f.", name(), resource, broker.host().name(), utilization, capacityLimit));
            }
        }
        // Broker-level violation check.
        if (resource.isBrokerResource()) {
            double utilization = broker.load().expectedUtilizationFor(resource);
            double capacityLimit = broker.capacityFor(resource) * capacityThreshold;
            if (!broker.replicas().isEmpty() && utilization > capacityLimit) {
                // The utilization of the broker for the resource is over the capacity limit.
                throw new OptimizationFailureException(String.format("Optimization for goal %s failed because %s utilization " + "for broker %d is %f which is above capacity limit %f.", name(), resource, broker.id(), utilization, capacityLimit));
            }
        }
    }
}

Example 5 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class KafkaCruiseControlServlet method getPartitionLoad.

private boolean getPartitionLoad(HttpServletRequest request, HttpServletResponse response) throws Exception {
    Resource resource;
    Long startMs;
    Long endMs;
    boolean json = wantJSON(request);
    try {
        String resourceString = request.getParameter(RESOURCE_PARAM);
        try {
            if (resourceString == null) {
                resourceString = DEFAULT_PARTITION_LOAD_RESOURCE;
            }
            resource = Resource.valueOf(resourceString.toUpperCase());
        } catch (IllegalArgumentException iae) {
            String errorMsg = String.format("Invalid resource type %s. The resource type must be one of the following: " + "CPU, DISK, NW_IN, NW_OUT", resourceString);
            StringWriter sw = new StringWriter();
            iae.printStackTrace(new PrintWriter(sw));
            setErrorResponse(response, sw.toString(), errorMsg, SC_BAD_REQUEST, json);
            // Close session
            return true;
        }
        String startMsString = request.getParameter(START_MS_PARAM);
        String endMsString = request.getParameter(END_MS_PARAM);
        startMs = startMsString == null ? -1L : Long.parseLong(startMsString);
        endMs = endMsString == null ? System.currentTimeMillis() : Long.parseLong(endMsString);
    } catch (Exception e) {
        StringWriter sw = new StringWriter();
        e.printStackTrace(new PrintWriter(sw));
        setErrorResponse(response, sw.toString(), e.getMessage(), SC_BAD_REQUEST, json);
        // Close session
        return true;
    }
    ModelCompletenessRequirements requirements = new ModelCompletenessRequirements(1, 0.0, false);
    // Get cluster model asynchronously.
    ClusterModel clusterModel = getAndMaybeReturnProgress(request, response, () -> _asyncKafkaCruiseControl.clusterModel(startMs, endMs, requirements));
    if (clusterModel == null) {
        return false;
    }
    List<Partition> sortedPartitions = clusterModel.replicasSortedByUtilization(resource);
    OutputStream out = response.getOutputStream();
    String entriesString = request.getParameter(ENTRIES);
    Integer entries = entriesString == null ? Integer.MAX_VALUE : Integer.parseInt(entriesString);
    int numEntries = 0;
    if (!json) {
        int topicNameLength = clusterModel.topics().stream().mapToInt(String::length).max().orElse(20) + 5;
        setResponseCode(response, SC_OK);
        out.write(String.format("%" + topicNameLength + "s%10s%30s%20s%20s%20s%20s%n", "PARTITION", "LEADER", "FOLLOWERS", "CPU (%)", "DISK (MB)", "NW_IN (KB/s)", "NW_OUT (KB/s)").getBytes(StandardCharsets.UTF_8));
        for (Partition p : sortedPartitions) {
            if (++numEntries > entries) {
                break;
            }
            List<Integer> followers = p.followers().stream().map((replica) -> replica.broker().id()).collect(Collectors.toList());
            out.write(String.format("%" + topicNameLength + "s%10s%30s%19.6f%19.3f%19.3f%19.3f%n", p.leader().topicPartition(), p.leader().broker().id(), followers, p.leader().load().expectedUtilizationFor(Resource.CPU), p.leader().load().expectedUtilizationFor(Resource.DISK), p.leader().load().expectedUtilizationFor(Resource.NW_IN), p.leader().load().expectedUtilizationFor(Resource.NW_OUT)).getBytes(StandardCharsets.UTF_8));
        }
    } else {
        Map<String, Object> partitionMap = new HashMap<>();
        List<Object> partitionList = new ArrayList<>();
        List<String> header = new ArrayList<>(Arrays.asList("topic", "partition", "leader", "followers", "CPU", "DISK", "NW_IN", "NW_OUT"));
        partitionMap.put("version", JSON_VERSION);
        partitionMap.put("header", header);
        for (Partition p : sortedPartitions) {
            if (++numEntries > entries) {
                break;
            }
            List<Integer> followers = p.followers().stream().map((replica) -> replica.broker().id()).collect(Collectors.toList());
            List<Object> record = new ArrayList<>();
            record.add(p.leader().topicPartition().topic());
            record.add(p.leader().topicPartition().partition());
            record.add(p.leader().broker().id());
            record.add(followers);
            record.add(p.leader().load().expectedUtilizationFor(Resource.CPU));
            record.add(p.leader().load().expectedUtilizationFor(Resource.DISK));
            record.add(p.leader().load().expectedUtilizationFor(Resource.NW_IN));
            record.add(p.leader().load().expectedUtilizationFor(Resource.NW_OUT));
            partitionList.add(record);
        }
        partitionMap.put("records", partitionList);
        Gson gson = new Gson();
        String g = gson.toJson(partitionMap);
        setJSONResponseCode(response, SC_OK);
        response.setContentLength(g.length());
        out.write(g.getBytes(StandardCharsets.UTF_8));
    }
    out.flush();
    return true;
}