Examples with Disk com.linkedin.kafka.cruisecontrol.model.Disk used on opensource projects

Example 6 with Disk

use of com.linkedin.kafka.cruisecontrol.model.Disk in project cruise-control by linkedin.

the class IntraBrokerDiskUsageDistributionGoal method rebalanceByMovingLoadIn.

/**
 * Try to balance the underloaded disk by moving in replicas from other disks of the same broker.
 *
 * @param disk                 The disk to balance.
 * @param clusterModel         The current cluster model.
 * @param optimizedGoals       Optimized goals.
 * @param optimizationOptions  Options to take into account during optimization -- e.g. excluded topics.
 * @return {@code true} if the disk to balance is still underloaded, {@code false} otherwise.
 */
private boolean rebalanceByMovingLoadIn(Disk disk, ClusterModel clusterModel, Set<Goal> optimizedGoals, OptimizationOptions optimizationOptions) {
    Broker broker = disk.broker();
    double brokerUtilization = averageDiskUtilizationPercentage(broker);
    PriorityQueue<Disk> candidateDiskPQ = new PriorityQueue<>((d1, d2) -> Double.compare(diskUtilizationPercentage(d2), diskUtilizationPercentage(d1)));
    for (Disk candidateDisk : broker.disks()) {
        // Get candidate disk on broker to try moving load from -- sorted in the order of trial (descending load).
        if (candidateDisk.isAlive() && diskUtilizationPercentage(candidateDisk) > brokerUtilization) {
            candidateDiskPQ.add(candidateDisk);
        }
    }
    while (!candidateDiskPQ.isEmpty()) {
        Disk candidateDisk = candidateDiskPQ.poll();
        for (Iterator<Replica> iterator = candidateDisk.trackedSortedReplicas(replicaSortName(this, true, false)).sortedReplicas(true).iterator(); iterator.hasNext(); ) {
            Replica replica = iterator.next();
            Disk d = maybeMoveReplicaBetweenDisks(clusterModel, replica, Collections.singleton(disk), optimizedGoals);
            // has nothing to move in. In that case we will never re-enqueue that source disk.
            if (d != null) {
                if (diskUtilizationPercentage(disk) > _balanceLowerThresholdByBroker.get(broker)) {
                    return false;
                }
                iterator.remove();
                // we re-enqueue the source disk and switch to the next disk.
                if (!candidateDiskPQ.isEmpty() && diskUtilizationPercentage(candidateDisk) < diskUtilizationPercentage(candidateDiskPQ.peek())) {
                    candidateDiskPQ.add(candidateDisk);
                    break;
                }
            }
        }
    }
    return true;
}

Example 7 with Disk

use of com.linkedin.kafka.cruisecontrol.model.Disk in project cruise-control by linkedin.

the class IntraBrokerDiskUsageDistributionGoal method rebalanceBySwappingLoadIn.

/**
 * Try to balance the underloaded disk by swapping its replicas with replicas from other disks of the same broker.
 *
 * @param disk                 The disk to balance.
 * @param clusterModel         The current cluster model.
 * @param optimizedGoals       Optimized goals.
 * @param optimizationOptions  Options to take into account during optimization -- e.g. excluded topics.
 */
private void rebalanceBySwappingLoadIn(Disk disk, ClusterModel clusterModel, Set<Goal> optimizedGoals, OptimizationOptions optimizationOptions) {
    long swapStartTimeMs = System.currentTimeMillis();
    Broker broker = disk.broker();
    PriorityQueue<Disk> candidateDiskPQ = new PriorityQueue<>((d1, d2) -> Double.compare(diskUtilizationPercentage(d2), diskUtilizationPercentage(d1)));
    for (Disk candidateDisk : broker.disks()) {
        // Get candidate disk on broker to try to swap replica with -- sorted in the order of trial (descending load).
        if (candidateDisk.isAlive() && diskUtilizationPercentage(candidateDisk) > _balanceLowerThresholdByBroker.get(broker)) {
            candidateDiskPQ.add(candidateDisk);
        }
    }
    while (!candidateDiskPQ.isEmpty()) {
        Disk candidateDisk = candidateDiskPQ.poll();
        for (Replica sourceReplica : disk.trackedSortedReplicas(replicaSortName(this, false, false)).sortedReplicas(false)) {
            // Try swapping the source with the candidate replicas. Get the swapped in replica if successful, null otherwise.
            Replica swappedIn = maybeSwapReplicaBetweenDisks(clusterModel, sourceReplica, candidateDisk.trackedSortedReplicas(replicaSortName(this, true, false)).sortedReplicas(false), optimizedGoals);
            if (swappedIn != null) {
                if (diskUtilizationPercentage(disk) > _balanceLowerThresholdByBroker.get(broker)) {
                    // Successfully balanced this broker by swapping in.
                    return;
                }
                break;
            }
        }
        if (remainingPerDiskSwapTimeMs(swapStartTimeMs) <= 0) {
            LOG.debug("Swap load out timeout for disk {}.", disk.logDir());
            break;
        }
        if (diskUtilizationPercentage(candidateDisk) > _balanceLowerThresholdByBroker.get(broker)) {
            candidateDiskPQ.add(candidateDisk);
        }
    }
}

Example 8 with Disk

use of com.linkedin.kafka.cruisecontrol.model.Disk in project cruise-control by linkedin.

the class IntraBrokerDiskUsageDistributionGoal method rebalanceForBroker.

/**
 * (1) REBALANCE BY REPLICA MOVEMENT:
 * Perform optimization via replica movement between disks to ensure balance: The load on disks are within range.
 * (2) REBALANCE BY REPLICA SWAP:
 * Swap replicas to ensure balance without violating optimized goal requirements.
 * Note the optimization from this goal cannot be applied to offline replicas because Kafka does not support moving
 * replicas on bad disks to good disks within the same broker.
 *
 * @param broker              Broker to be balanced.
 * @param clusterModel        The state of the cluster.
 * @param optimizedGoals      Optimized goals.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void rebalanceForBroker(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, OptimizationOptions optimizationOptions) {
    double upperLimit = _balanceUpperThresholdByBroker.get(broker);
    double lowerLimit = _balanceLowerThresholdByBroker.get(broker);
    for (Disk disk : broker.disks()) {
        if (!disk.isAlive()) {
            continue;
        }
        if (diskUtilizationPercentage(disk) > upperLimit) {
            if (rebalanceByMovingLoadOut(disk, clusterModel, optimizedGoals, optimizationOptions)) {
                rebalanceBySwappingLoadOut(disk, clusterModel, optimizedGoals, optimizationOptions);
            }
        }
        if (diskUtilizationPercentage(disk) < lowerLimit) {
            if (rebalanceByMovingLoadIn(disk, clusterModel, optimizedGoals, optimizationOptions)) {
                rebalanceBySwappingLoadIn(disk, clusterModel, optimizedGoals, optimizationOptions);
            }
        }
    }
}

Example 9 with Disk

use of com.linkedin.kafka.cruisecontrol.model.Disk in project cruise-control by linkedin.

the class IntraBrokerDiskUsageDistributionGoal method updateGoalState.

/**
 * Update goal state.
 * Sanity check: After completion of balancing the resource, check whether there are disks whose utilization percentage is
 * out of range, finish and mark optimization status accordingly.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void updateGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) {
    List<String> disksAboveBalanceUpperLimit = new ArrayList<>();
    List<String> disksBelowBalanceLowerLimit = new ArrayList<>();
    for (Broker broker : brokersToBalance(clusterModel)) {
        double upperLimit = _balanceUpperThresholdByBroker.get(broker);
        double lowerLimit = _balanceLowerThresholdByBroker.get(broker);
        for (Disk disk : broker.disks()) {
            if (disk.isAlive()) {
                if (diskUtilizationPercentage(disk) > upperLimit) {
                    disksAboveBalanceUpperLimit.add(broker.id() + ":" + disk.logDir());
                }
                if (diskUtilizationPercentage(disk) < lowerLimit) {
                    disksBelowBalanceLowerLimit.add(broker.id() + ":" + disk.logDir());
                }
            }
        }
    }
    if (!disksAboveBalanceUpperLimit.isEmpty()) {
        LOG.warn("Disks {} are above balance upper limit after optimization.", disksAboveBalanceUpperLimit);
        _succeeded = false;
    }
    if (!disksBelowBalanceLowerLimit.isEmpty()) {
        LOG.warn("Disks {} are below balance lower limit after optimization.", disksBelowBalanceLowerLimit);
        _succeeded = false;
    }
    finish();
}

Example 10 with Disk

use of com.linkedin.kafka.cruisecontrol.model.Disk in project cruise-control by linkedin.

the class IntraBrokerDiskUsageDistributionGoal method actionAcceptance.

/**
 * Check whether given action is acceptable by this goal. An action is acceptable by this goal if it satisfies the
 * following:
 * (1) If source and destination disks were within the limit before the action, the corresponding limits cannot be
 * violated after the action.
 * (2) The action cannot increase the utilization difference between disks.
 *
 * @param action Action to be checked for acceptance.
 * @param clusterModel The state of the cluster.
 * @return {@link ActionAcceptance#ACCEPT} if the action is acceptable by this goal,
 *         {@link ActionAcceptance#REPLICA_REJECT} otherwise.
 */
@Override
public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) {
    double sourceUtilizationDelta = sourceUtilizationDelta(action, clusterModel);
    Broker broker = clusterModel.broker(action.sourceBrokerId());
    Disk sourceDisk = broker.disk(action.sourceBrokerLogdir());
    Disk destinationDisk = broker.disk(action.destinationBrokerLogdir());
    if (sourceUtilizationDelta == 0) {
        // No change in terms of load.
        return ACCEPT;
    }
    if (isChangeViolatingLimit(sourceUtilizationDelta, sourceDisk, destinationDisk)) {
        return REPLICA_REJECT;
    }
    return isGettingMoreBalanced(sourceDisk, destinationDisk, sourceUtilizationDelta) ? ACCEPT : REPLICA_REJECT;
}