Examples with Replica com.linkedin.kafka.cruisecontrol.model.Replica used on opensource projects

Example 6 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class ResourceDistributionGoal method rebalanceBySwappingLoadOut.

private boolean rebalanceBySwappingLoadOut(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, Set<String> excludedTopics) {
    if (!broker.isAlive()) {
        return true;
    }
    // Get the replicas to rebalance.
    SortedSet<Replica> sourceReplicas = new TreeSet<>((r1, r2) -> {
        int result = Double.compare(r2.load().expectedUtilizationFor(resource()), r1.load().expectedUtilizationFor(resource()));
        return result == 0 ? r1.topicPartition().toString().compareTo(r2.topicPartition().toString()) : result;
    });
    sourceReplicas.addAll(resource() == Resource.NW_OUT ? broker.leaderReplicas() : broker.replicas());
    // Sort the replicas initially to avoid sorting it every time.
    PriorityQueue<CandidateBroker> candidateBrokerPQ = new PriorityQueue<>();
    for (Broker candidate : clusterModel.healthyBrokersUnderThreshold(resource(), _balanceUpperThreshold).stream().filter(b -> !b.replicas().isEmpty()).collect(Collectors.toSet())) {
        // Get candidate replicas on candidate broker to try swapping with -- sorted in the order of trial (ascending load).
        double maxSourceReplicaLoad = sourceReplicas.first().load().expectedUtilizationFor(resource());
        SortedSet<Replica> replicasToSwapWith = sortedCandidateReplicas(candidate, excludedTopics, maxSourceReplicaLoad, true);
        CandidateBroker candidateBroker = new CandidateBroker(candidate, replicasToSwapWith, true);
        candidateBrokerPQ.add(candidateBroker);
    }
    while (!candidateBrokerPQ.isEmpty()) {
        CandidateBroker cb = candidateBrokerPQ.poll();
        SortedSet<Replica> candidateReplicasToSwapWith = cb.replicas();
        Replica swappedInReplica = null;
        Replica swappedOutReplica = null;
        for (Replica sourceReplica : sourceReplicas) {
            if (shouldExclude(sourceReplica, excludedTopics)) {
                continue;
            }
            // Try swapping the source with the candidate replicas. Get the swapped in replica if successful, null otherwise.
            Replica swappedIn = maybeApplySwapAction(clusterModel, sourceReplica, candidateReplicasToSwapWith, optimizedGoals);
            if (swappedIn != null) {
                if (isLoadUnderBalanceUpperLimit(broker)) {
                    // Successfully balanced this broker by swapping in.
                    return false;
                }
                // Add swapped in/out replica for updating the list of replicas in source broker.
                swappedInReplica = swappedIn;
                swappedOutReplica = sourceReplica;
                break;
            }
        }
        swapUpdate(swappedInReplica, swappedOutReplica, sourceReplicas, candidateReplicasToSwapWith, candidateBrokerPQ, cb);
    }
    return true;
}

Example 7 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class TopicReplicaDistributionGoal method healCluster.

/**
 * Heal the given cluster without violating the requirements of optimized goals.
 *
 * @param clusterModel   The state of the cluster.
 * @param optimizedGoals Optimized goals.
 */
private void healCluster(ClusterModel clusterModel, Set<Goal> optimizedGoals) throws OptimizationFailureException {
    // Move self healed replicas (if their broker is overloaded or they reside at dead brokers) to eligible ones.
    for (Replica replica : clusterModel.selfHealingEligibleReplicas()) {
        String topic = replica.topicPartition().topic();
        ReplicaDistributionTarget replicaDistributionTarget = _replicaDistributionTargetByTopic.get(topic);
        replicaDistributionTarget.moveSelfHealingEligibleReplicaToEligibleBroker(clusterModel, replica, replica.broker().replicasOfTopicInBroker(topic).size(), optimizedGoals);
    }
}

Example 8 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class KafkaAssignerDiskUsageDistributionGoal method findReplicaToSwapWith.

/**
 * The function searches in a given sorted replica list until it finds a replica that is eligible to swap with the
 * specified replica.
 *
 * @param replica the specific replica to swap.
 * @param sortedReplicasToSearch  the sorted replica list.
 * @param targetSize the target size for the eligible replica.
 * @param minSize the min size for the eligible replica.
 * @param maxSize the max size for the eligible replica.
 * @param clusterModel the cluster model.
 *
 * @return the replica that can be swapped with the given replica, null otherwise.
 */
Replica findReplicaToSwapWith(Replica replica, List<ReplicaWrapper> sortedReplicasToSearch, double targetSize, double minSize, double maxSize, ClusterModel clusterModel) {
    int pos = findReplicaPos(sortedReplicasToSearch, targetSize, 0);
    int minPos = findReplicaPos(sortedReplicasToSearch, minSize, 1);
    int maxPos = findReplicaPos(sortedReplicasToSearch, maxSize, -1);
    if (minPos > maxPos) {
        // This check also ensures that both minPos and maxPos are within the valid index range for the given list [0, n-1].
        return null;
    }
    // It is possible that the target size is out of the range of minSize and maxSize. In that case, we make it become
    // the closest one in range.
    pos = Math.max(pos, minPos);
    pos = Math.min(pos, maxPos);
    // The following logic starts from pos and searches higher and lower position until it finds a replica that
    // is eligible to swap with.
    int low = pos;
    int high = pos;
    while (pos >= minPos && pos <= maxPos) {
        Replica toSwapWith = sortedReplicasToSearch.get(pos).replica();
        if (canSwap(replica, sortedReplicasToSearch.get(pos).replica(), clusterModel)) {
            // found the candidate.
            return toSwapWith;
        } else {
            // get the next position.
            pos = findNextPos(sortedReplicasToSearch, targetSize, low - 1, high + 1, minPos, maxPos);
            if (pos == low - 1) {
                low--;
            } else {
                high++;
            }
        }
    }
    return null;
}

Example 9 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class KafkaAssignerEvenRackAwareGoal method ensureRackAware.

/**
 * Sanity Check: Replicas are distributed in a rack-aware way.
 *
 * @param clusterModel The state of the cluster.
 * @param excludedTopics The topics that should be excluded from the optimization proposals.
 */
private void ensureRackAware(ClusterModel clusterModel, Set<String> excludedTopics) throws OptimizationFailureException {
    // Sanity check to confirm that the final distribution is rack aware.
    for (Replica leader : clusterModel.leaderReplicas()) {
        if (excludedTopics.contains(leader.topicPartition().topic())) {
            continue;
        }
        Set<String> replicaBrokersRackIds = new HashSet<>();
        Set<Broker> followerBrokers = new HashSet<>(clusterModel.partition(leader.topicPartition()).followerBrokers());
        // Add rack Id of replicas.
        for (Broker followerBroker : followerBrokers) {
            String followerRackId = followerBroker.rack().id();
            replicaBrokersRackIds.add(followerRackId);
        }
        replicaBrokersRackIds.add(leader.broker().rack().id());
        if (replicaBrokersRackIds.size() != (followerBrokers.size() + 1)) {
            throw new OptimizationFailureException("Optimization for goal " + name() + " failed for rack-awareness of " + "partition " + leader.topicPartition());
        }
    }
}

Example 10 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class KafkaAssignerEvenRackAwareGoal method maybeApplyMove.

/**
 * Apply the move to the first eligible destination broker selected from _healthyBrokerReplicaCountByPosition for the
 * relevant replica position.
 *
 * An eligible destination broker must reside in a rack that has no other replicas from the same partition, which has
 * a position smaller than the given replicaPosition.
 *
 * If the destination broker has:
 * (1) no other replica from the same partition, move the replica to there.
 * (2) a replica with a larger position AND the source broker is alive, swap positions.
 * (3) the conditions (1-2) are false AND the source broker is dead.
 * (4) the current replica under consideration, do nothing -- i.e. do not move replica or swap positions.
 *
 * @param clusterModel The state of the cluster.
 * @param partition The partition whose replica might be moved.
 * @param replicaPosition The position of the replica in the given partition.
 * @return true if a move is applied, false otherwise.
 */
private boolean maybeApplyMove(ClusterModel clusterModel, Partition partition, int replicaPosition) {
    // Racks with replica whose position is in [0, replicaPosition - 1] are ineligible for assignment.
    Set<String> ineligibleRackIds = new HashSet<>();
    for (int pos = 0; pos < replicaPosition; pos++) {
        Replica replica = replicaAtPosition(partition, pos);
        ineligibleRackIds.add(replica.broker().rack().id());
    }
    // Find an eligible destination and apply the relevant move.
    BrokerReplicaCount eligibleBrokerReplicaCount = null;
    for (final Iterator<BrokerReplicaCount> it = _healthyBrokerReplicaCountByPosition.get(replicaPosition).iterator(); it.hasNext(); ) {
        BrokerReplicaCount destinationBrokerReplicaCount = it.next();
        if (ineligibleRackIds.contains(destinationBrokerReplicaCount.broker().rack().id())) {
            continue;
        }
        // Get the replica in the destination broker (if any)
        Broker destinationBroker = destinationBrokerReplicaCount.broker();
        Replica destinationReplica = destinationBroker.replica(partition.topicPartition());
        Replica replicaAtPosition = partition.replicas().get(replicaPosition);
        if (destinationReplica == null) {
            // The destination broker has no replica from the source partition: move the source replica to the destination broker.
            LOG.trace("Destination broker {} has no other replica from the same partition, move the replica {} to there.", destinationBroker, replicaAtPosition);
            applyBalancingAction(clusterModel, replicaAtPosition, destinationBroker, ActionType.REPLICA_MOVEMENT);
        } else if (destinationBroker.id() != replicaAtPosition.broker().id() && replicaAtPosition.broker().isAlive()) {
            // The destination broker contains a replica from the source partition AND the destination broker is different
            // from the source replica broker AND the source broker is alive. Hence, we can safely swap replica positions.
            LOG.trace("Destination broker has a replica {} with a larger position than source replica {}, swap positions.", destinationReplica, replicaAtPosition);
            if (replicaPosition == 0) {
                // Transfer leadership -- i.e swap the position of leader with its follower in destination broker.
                applyBalancingAction(clusterModel, replicaAtPosition, destinationBroker, ActionType.LEADERSHIP_MOVEMENT);
            } else {
                // Swap the follower position of this replica with the follower position of destination replica.
                int destinationPos = followerPosition(partition, destinationBroker.id());
                partition.swapFollowerPositions(replicaPosition, destinationPos);
            }
        } else if (!replicaAtPosition.broker().isAlive()) {
            // The broker of source replica is dead. Hence, we have to move the source replica away from it. But, destination
            // broker contains a replica from the same source partition. This prevents moving the source replica to it.
            LOG.trace("Source broker {} is dead and either the destination broker {} is the same as the source, or has a " + "replica from the same partition.", replicaAtPosition.broker(), destinationBroker);
            // Unable apply any valid move.
            continue;
        }
        // Increment the replica count on the destination. Note that if the source and the destination brokers are the
        // same, then the source replica will simply stay in the same broker.
        eligibleBrokerReplicaCount = destinationBrokerReplicaCount;
        it.remove();
        break;
    }
    if (eligibleBrokerReplicaCount != null) {
        // Success: Increment the replica count on the destination.
        eligibleBrokerReplicaCount.incReplicaCount();
        _healthyBrokerReplicaCountByPosition.get(replicaPosition).add(eligibleBrokerReplicaCount);
        return true;
    }
    // Failure: Unable to apply any valid move -- i.e. optimization failed to place the source replica to a valid broker.
    return false;
}