Examples with Replica com.linkedin.kafka.cruisecontrol.model.Replica used on opensource projects

Example 16 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class RackAwareGoal method isReplicaMoveViolateRackAwareness.

private boolean isReplicaMoveViolateRackAwareness(ClusterModel clusterModel, Function<ClusterModel, Replica> sourceReplicaFunction, Function<ClusterModel, Broker> destinationBrokerFunction) {
    Replica sourceReplica = sourceReplicaFunction.apply(clusterModel);
    Broker destinationBroker = destinationBrokerFunction.apply(clusterModel);
    // Destination broker cannot be in a rack that violates rack awareness.
    Set<Broker> partitionBrokers = clusterModel.partition(sourceReplica.topicPartition()).partitionBrokers();
    partitionBrokers.remove(sourceReplica.broker());
    // Remove brokers in partition broker racks except the brokers in replica broker rack.
    for (Broker broker : partitionBrokers) {
        if (broker.rack().brokers().contains(destinationBroker)) {
            return true;
        }
    }
    return false;
}

Example 17 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class RackAwareGoal method rackAwareEligibleBrokers.

/**
 * Get a list of rack aware eligible brokers for the given replica in the given cluster. A broker is rack aware
 * eligible for a given replica if the broker resides in a rack where no other broker in the same rack contains a
 * replica from the same partition of the given replica.
 *
 * @param replica      Replica for which a set of rack aware eligible brokers are requested.
 * @param clusterModel The state of the cluster.
 * @return A list of rack aware eligible brokers for the given replica in the given cluster.
 */
private SortedSet<Broker> rackAwareEligibleBrokers(Replica replica, ClusterModel clusterModel) {
    // Populate partition rack ids.
    List<String> partitionRackIds = clusterModel.partition(replica.topicPartition()).partitionBrokers().stream().map(partitionBroker -> partitionBroker.rack().id()).collect(Collectors.toList());
    // Remove rack id of the given replica, but if there is any other replica from the partition residing in the
    // same cluster, keep its rack id in the list.
    partitionRackIds.remove(replica.broker().rack().id());
    SortedSet<Broker> rackAwareEligibleBrokers = new TreeSet<>((o1, o2) -> {
        return Integer.compare(o1.id(), o2.id());
    });
    for (Broker broker : clusterModel.healthyBrokers()) {
        if (!partitionRackIds.contains(broker.rack().id())) {
            rackAwareEligibleBrokers.add(broker);
        }
    }
    // Return eligible brokers.
    return rackAwareEligibleBrokers;
}

Example 18 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class ReplicaCapacityGoal method rebalanceForBroker.

/**
 * Rebalance the given broker without violating the constraints of the current goal and optimized goals.
 * @param broker         Broker to be balanced.
 * @param clusterModel   The state of the cluster.
 * @param optimizedGoals Optimized goals.
 * @param excludedTopics The topics that should be excluded from the optimization proposals.
 */
@Override
protected void rebalanceForBroker(Broker broker, ClusterModel clusterModel, Set<Goal> optimizedGoals, Set<String> excludedTopics) throws OptimizationFailureException {
    LOG.debug("balancing broker {}, optimized goals = {}", broker, optimizedGoals);
    for (Replica replica : new ArrayList<>(broker.replicas())) {
        if (broker.isAlive() && broker.replicas().size() <= _balancingConstraint.maxReplicasPerBroker()) {
            break;
        }
        if (shouldExclude(replica, excludedTopics)) {
            continue;
        }
        // The goal requirements are violated. Move replica to an eligible broker.
        List<Broker> eligibleBrokers = eligibleBrokers(replica, clusterModel).stream().map(BrokerReplicaCount::broker).collect(Collectors.toList());
        Broker b = maybeApplyBalancingAction(clusterModel, replica, eligibleBrokers, ActionType.REPLICA_MOVEMENT, optimizedGoals);
        if (b == null) {
            if (!broker.isAlive()) {
                // If the replica resides in a dead broker, throw an exception!
                throw new OptimizationFailureException(String.format("Failed to move dead broker replica %s of partition %s " + "to a broker in %s. Limit: %d for brokers: %s", replica, clusterModel.partition(replica.topicPartition()), eligibleBrokers, _balancingConstraint.maxReplicasPerBroker(), clusterModel.brokers()));
            }
            LOG.debug("Failed to move replica {} to any broker in {}.", replica, eligibleBrokers);
        }
    }
}

Example 19 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class KafkaAssignerDiskUsageDistributionGoalTest method testCanSwap.

@Test
public void testCanSwap() {
    KafkaAssignerDiskUsageDistributionGoal goal = new KafkaAssignerDiskUsageDistributionGoal();
    ClusterModel clusterModel = createClusterModel();
    Replica r1 = clusterModel.broker(0).replica(T0P0);
    Replica r2 = clusterModel.broker(1).replica(T2P0);
    assertTrue("Replicas in the same rack should be good to swap", goal.canSwap(r1, r2, clusterModel));
    assertTrue("Replicas in the same rack should be good to swap", goal.canSwap(r2, r1, clusterModel));
    r2 = clusterModel.broker(1).replica(T1P0);
    assertFalse("Should not be able to swap replica with different roles.", goal.canSwap(r1, r2, clusterModel));
    assertFalse("Should not be able to swap replica with different roles.", goal.canSwap(r2, r1, clusterModel));
    r2 = clusterModel.broker(2).replica(T2P1);
    assertFalse("Should not be able to put two replicas in the same broker", goal.canSwap(r1, r2, clusterModel));
    assertFalse("Should not be able to put two replicas in the same broker", goal.canSwap(r2, r1, clusterModel));
    r2 = clusterModel.broker(3).replica(T2P2);
    assertFalse("Should not be able to put two replicas in the same rack", goal.canSwap(r1, r2, clusterModel));
    assertFalse("Should not be able to put two replicas in the same rack", goal.canSwap(r2, r1, clusterModel));
    r1 = clusterModel.broker(3).replica(T0P2);
    r2 = clusterModel.broker(4).replica(T1P2);
    assertTrue("Should be able to swap", goal.canSwap(r1, r2, clusterModel));
    assertTrue("Should be able to swap", goal.canSwap(r2, r1, clusterModel));
}

Example 20 with Replica

use of com.linkedin.kafka.cruisecontrol.model.Replica in project cruise-control by linkedin.

the class OptimizationVerifier method verifyNewBrokers.

private static boolean verifyNewBrokers(ClusterModel clusterModel, BalancingConstraint constraint) {
    for (Broker broker : clusterModel.healthyBrokers()) {
        if (!broker.isNew()) {
            for (Replica replica : broker.replicas()) {
                if (replica.originalBroker() != broker) {
                    LOG.error("Broker {} is not a new broker but has received new replicas", broker.id());
                    return false;
                }
            }
        }
    }
    for (Broker broker : clusterModel.newBrokers()) {
        // We can only check the first resource.
        Resource r = constraint.resources().get(0);
        double utilizationLowerThreshold = clusterModel.load().expectedUtilizationFor(r) / clusterModel.capacityFor(r) * (2 - constraint.resourceBalancePercentage(r));
        double brokerUtilization = broker.load().expectedUtilizationFor(r) / broker.capacityFor(r);
        if (brokerUtilization < utilizationLowerThreshold) {
            LOG.error("Broker {} is still underutilized for resource {}. Broker utilization is {}, the " + "lower threshold is {}", broker, r, brokerUtilization, utilizationLowerThreshold);
            return false;
        }
    }
    return true;
}