Examples with Replica com.linkedin.kafka.cruisecontrol.model.Replica used on opensource projects

Example 11 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class AbstractGoal method maybeApplySwapAction.

/**
 * Attempt to swap the given source replica with a replica from the candidate replicas to swap with. The function
 * returns the swapped in replica if succeeded, null otherwise.
 * All the replicas in the given candidateReplicasToSwapWith must be from the same broker.
 *
 * @param clusterModel The state of the cluster.
 * @param sourceReplica Replica to be swapped with.
 * @param candidateReplicasToSwapWith Candidate replicas from the same destination broker to swap in the order of
 *                                    attempts to swap.
 * @param optimizedGoals Optimized goals.
 * @return True the swapped in replica if succeeded, null otherwise.
 */
Replica maybeApplySwapAction(ClusterModel clusterModel, Replica sourceReplica, SortedSet<Replica> candidateReplicasToSwapWith, Set<Goal> optimizedGoals) {
    SortedSet<Replica> eligibleReplicas = getEligibleReplicasForSwap(clusterModel, sourceReplica, candidateReplicasToSwapWith);
    if (eligibleReplicas.isEmpty()) {
        return null;
    }
    Broker destinationBroker = eligibleReplicas.first().broker();
    for (Replica destinationReplica : eligibleReplicas) {
        BalancingAction swapProposal = new BalancingAction(sourceReplica.topicPartition(), sourceReplica.broker().id(), destinationBroker.id(), ActionType.REPLICA_SWAP, destinationReplica.topicPartition());
        // 3. The movement is acceptable by the previously optimized goals.
        if (!legitMove(sourceReplica, destinationBroker, ActionType.REPLICA_MOVEMENT)) {
            LOG.trace("Swap from source to destination is not legit for {}.", swapProposal);
            return null;
        }
        if (!legitMove(destinationReplica, sourceReplica.broker(), ActionType.REPLICA_MOVEMENT)) {
            LOG.trace("Swap from destination to source is not legit for {}.", swapProposal);
            continue;
        }
        // The current goal is expected to know whether a swap is doable between given brokers.
        if (!selfSatisfied(clusterModel, swapProposal)) {
            // Unable to satisfy proposal for this eligible replica and the remaining eligible replicas in the list.
            LOG.trace("Unable to self-satisfy swap proposal {}.", swapProposal);
            return null;
        }
        ActionAcceptance acceptance = AnalyzerUtils.isProposalAcceptableForOptimizedGoals(optimizedGoals, swapProposal, clusterModel);
        LOG.trace("Trying to apply legit and self-satisfied swap {}, actionAcceptance = {}.", swapProposal, acceptance);
        if (acceptance == ACCEPT) {
            Broker sourceBroker = sourceReplica.broker();
            clusterModel.relocateReplica(sourceReplica.topicPartition(), sourceBroker.id(), destinationBroker.id());
            clusterModel.relocateReplica(destinationReplica.topicPartition(), destinationBroker.id(), sourceBroker.id());
            return destinationReplica;
        } else if (acceptance == BROKER_REJECT) {
            // Unable to swap the given source replica with any replicas in the destination broker.
            return null;
        }
    }
    return null;
}

Example 12 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class CapacityGoal method selfSatisfied.

/**
 * This is a hard goal; hence, the proposals are not limited to dead broker replicas in case of self-healing.
 * Check if requirements of this goal are not violated if this action is applied to the given cluster state,
 * false otherwise.
 *
 * @param clusterModel The state of the cluster.
 * @param action Action containing information about potential modification to the given cluster model.
 * @return True if requirements of this goal are not violated if this action is applied to the given cluster state,
 * false otherwise.
 */
@Override
protected boolean selfSatisfied(ClusterModel clusterModel, BalancingAction action) {
    Replica sourceReplica = clusterModel.broker(action.sourceBrokerId()).replica(action.topicPartition());
    Broker destinationBroker = clusterModel.broker(action.destinationBrokerId());
    // then the expected leadership CPU utilization would be the full CPU utilization of the leader.
    return isMovementAcceptableForCapacity(sourceReplica, destinationBroker);
}

Example 13 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class LeaderBytesInDistributionGoal method rebalanceForBroker.

@Override
protected void rebalanceForBroker(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, Set<String> excludedTopics) {
    double balanceThreshold = balanceThreshold(clusterModel, broker.id());
    if (broker.leadershipLoadForNwResources().expectedUtilizationFor(Resource.NW_IN) < balanceThreshold) {
        return;
    }
    List<Replica> leaderReplicasSortedByBytesIn = broker.replicas().stream().filter(Replica::isLeader).filter(r -> !shouldExclude(r, excludedTopics)).sorted((a, b) -> Double.compare(b.load().expectedUtilizationFor(Resource.NW_IN), a.load().expectedUtilizationFor(Resource.NW_IN))).collect(Collectors.toList());
    boolean overThreshold = true;
    Iterator<Replica> leaderReplicaIt = leaderReplicasSortedByBytesIn.iterator();
    while (overThreshold && leaderReplicaIt.hasNext()) {
        Replica leaderReplica = leaderReplicaIt.next();
        List<Replica> followers = clusterModel.partition(leaderReplica.topicPartition()).followers();
        List<Broker> eligibleBrokers = followers.stream().map(Replica::broker).sorted(Comparator.comparingDouble(a -> a.leadershipLoadForNwResources().expectedUtilizationFor(Resource.NW_IN))).collect(Collectors.toList());
        maybeApplyBalancingAction(clusterModel, leaderReplica, eligibleBrokers, ActionType.LEADERSHIP_MOVEMENT, optimizedGoals);
        overThreshold = broker.leadershipLoadForNwResources().expectedUtilizationFor(Resource.NW_IN) > balanceThreshold;
    }
    if (overThreshold) {
        _overLimitBrokerIds.add(broker.id());
    }
}

Example 14 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class LeaderBytesInDistributionGoal method actionAcceptance.

/**
 * An action is acceptable if it does not move the leader bytes in above the threshold for leader bytes in.
 *
 * @param action Action to be checked for acceptance.
 * @param clusterModel State of the cluster before application of the action.
 * @return {@link ActionAcceptance#ACCEPT} if the action is acceptable by this goal,
 * {@link ActionAcceptance#REPLICA_REJECT} otherwise.
 */
@Override
public ActionAcceptance actionAcceptance(BalancingAction action, ClusterModel clusterModel) {
    Replica sourceReplica = clusterModel.broker(action.sourceBrokerId()).replica(action.topicPartition());
    Broker destinationBroker = clusterModel.broker(action.destinationBrokerId());
    initMeanLeaderBytesIn(clusterModel);
    if (!sourceReplica.isLeader()) {
        switch(action.balancingAction()) {
            case REPLICA_SWAP:
                if (!destinationBroker.replica(action.destinationTopicPartition()).isLeader()) {
                    // No leadership bytes are being swapped between source and destination.
                    return ACCEPT;
                }
                break;
            case REPLICA_MOVEMENT:
                // No leadership bytes are being moved to destination.
                return ACCEPT;
            case LEADERSHIP_MOVEMENT:
                throw new IllegalStateException("Attempt to move leadership from the follower.");
            default:
                throw new IllegalArgumentException("Unsupported balancing action " + action.balancingAction() + " is provided.");
        }
    }
    double sourceReplicaUtilization = sourceReplica.load().expectedUtilizationFor(Resource.NW_IN);
    double newDestLeaderBytesIn;
    switch(action.balancingAction()) {
        case REPLICA_SWAP:
            double destinationReplicaUtilization = destinationBroker.replica(action.destinationTopicPartition()).load().expectedUtilizationFor(Resource.NW_IN);
            newDestLeaderBytesIn = destinationBroker.leadershipLoadForNwResources().expectedUtilizationFor(Resource.NW_IN) + sourceReplicaUtilization - destinationReplicaUtilization;
            Broker sourceBroker = clusterModel.broker(action.sourceBrokerId());
            double newSourceLeaderBytesIn = sourceBroker.leadershipLoadForNwResources().expectedUtilizationFor(Resource.NW_IN) + destinationReplicaUtilization - sourceReplicaUtilization;
            if (newSourceLeaderBytesIn > balanceThreshold(clusterModel, sourceBroker.id())) {
                return REPLICA_REJECT;
            }
            break;
        case REPLICA_MOVEMENT:
        case LEADERSHIP_MOVEMENT:
            newDestLeaderBytesIn = destinationBroker.leadershipLoadForNwResources().expectedUtilizationFor(Resource.NW_IN) + sourceReplicaUtilization;
            break;
        default:
            throw new IllegalArgumentException("Unsupported balancing action " + action.balancingAction() + " is provided.");
    }
    return !(newDestLeaderBytesIn > balanceThreshold(clusterModel, destinationBroker.id())) ? ACCEPT : REPLICA_REJECT;
}

Example 15 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class PotentialNwOutGoal method selfSatisfied.

/**
 * Check if the movement of potential outbound network utilization from the given source replica to given
 * destination broker is acceptable for this goal. The action is unacceptable if both of the following conditions
 * are met: (1) transfer of replica makes the potential network outbound utilization of the destination broker go
 * out of its allowed capacity, (2) broker containing the source replica is alive. For a swap action, this
 * consideration is bidirectional.
 *
 * @param clusterModel The state of the cluster.
 * @param action Action containing information about potential modification to the given cluster model.
 * @return True if requirements of this goal are not violated if this action is applied to the given cluster state,
 * false otherwise.
 */
@Override
protected boolean selfSatisfied(ClusterModel clusterModel, BalancingAction action) {
    Replica sourceReplica = clusterModel.broker(action.sourceBrokerId()).replica(action.topicPartition());
    ActionType actionType = action.balancingAction();
    Broker sourceBroker = sourceReplica.broker();
    // If the source broker is dead and currently self healing dead brokers only, then the action must be executed.
    if (!sourceBroker.isAlive() && _selfHealingDeadBrokersOnly && actionType != ActionType.REPLICA_SWAP) {
        return true;
    }
    Broker destinationBroker = clusterModel.broker(action.destinationBrokerId());
    double destinationBrokerUtilization = clusterModel.potentialLeadershipLoadFor(destinationBroker.id()).expectedUtilizationFor(Resource.NW_OUT);
    double destinationCapacity = destinationBroker.capacityFor(Resource.NW_OUT) * _balancingConstraint.capacityThreshold(Resource.NW_OUT);
    double sourceReplicaUtilization = clusterModel.partition(sourceReplica.topicPartition()).leader().load().expectedUtilizationFor(Resource.NW_OUT);
    if (actionType != ActionType.REPLICA_SWAP) {
        // Check whether replica or leadership transfer leads to violation of capacity limit requirement.
        return destinationCapacity >= destinationBrokerUtilization + sourceReplicaUtilization;
    }
    // Ensure that the destination capacity of self-satisfied for action type swap is not violated.
    double destinationReplicaUtilization = clusterModel.partition(action.destinationTopicPartition()).leader().load().expectedUtilizationFor(Resource.NW_OUT);
    if (destinationCapacity < destinationBrokerUtilization + sourceReplicaUtilization - destinationReplicaUtilization) {
        // Destination capacity would be violated due to swap.
        return false;
    }
    // Ensure that the source capacity of self-satisfied for action type swap is not violated.
    double sourceBrokerUtilization = clusterModel.potentialLeadershipLoadFor(sourceBroker.id()).expectedUtilizationFor(Resource.NW_OUT);
    double sourceCapacity = sourceBroker.capacityFor(Resource.NW_OUT) * _balancingConstraint.capacityThreshold(Resource.NW_OUT);
    return sourceCapacity >= sourceBrokerUtilization + destinationReplicaUtilization - sourceReplicaUtilization;
}