Examples with ReplicaPlacementInfo com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo used on opensource projects

Example 26 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ReplicationThrottleHelperTest method testDoNotModifyExistingWildcardReplicaThrottles.

@Test
public void testDoNotModifyExistingWildcardReplicaThrottles() {
    createTopics();
    KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(zookeeper().connectionString(), "ReplicationThrottleHelperTestMetricGroup", "AddingThrottlesWithNoPreExistingThrottles", false);
    // Set replica throttle config values for both topics
    setWildcardThrottleReplicaForTopic(kafkaZkClient, TOPIC0);
    setWildcardThrottleReplicaForTopic(kafkaZkClient, TOPIC1);
    final long throttleRate = 100L;
    ReplicationThrottleHelper throttleHelper = new ReplicationThrottleHelper(kafkaZkClient, throttleRate);
    ExecutionProposal proposal = new ExecutionProposal(new TopicPartition(TOPIC0, 0), 100, new ReplicaPlacementInfo(0), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(2)));
    ExecutionProposal proposal2 = new ExecutionProposal(new TopicPartition(TOPIC0, 1), 100, new ReplicaPlacementInfo(0), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(3)), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(2)));
    throttleHelper.setThrottles(Arrays.asList(proposal, proposal2));
    ExecutionTask completedTask = completedTaskForProposal(0, proposal);
    ExecutionTask inProgressTask = inProgressTaskForProposal(1, proposal2);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 0, throttleRate);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 1, throttleRate);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 2, throttleRate);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 3, throttleRate);
    // Topic-level throttled replica config value should remain as "*"
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, ReplicationThrottleHelper.WILDCARD_ASTERISK);
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC1, ReplicationThrottleHelper.WILDCARD_ASTERISK);
    throttleHelper.clearThrottles(Collections.singletonList(completedTask), Collections.singletonList(inProgressTask));
    assertExpectedThrottledRateForBroker(kafkaZkClient, 0, throttleRate);
    // we expect broker 1 to be null since all replica movement related to it has completed.
    assertExpectedThrottledRateForBroker(kafkaZkClient, 1, null);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 2, throttleRate);
    // We expect broker 3 to have a throttle on it because there is an in-progress replica being moved
    assertExpectedThrottledRateForBroker(kafkaZkClient, 3, throttleRate);
    // Topic-level throttled replica config value should remain as "*"
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, ReplicationThrottleHelper.WILDCARD_ASTERISK);
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC1, ReplicationThrottleHelper.WILDCARD_ASTERISK);
    // passing an inProgress task that is not complete should have no effect.
    throttleHelper.clearThrottles(Collections.singletonList(completedTask), Collections.singletonList(inProgressTask));
    assertExpectedThrottledRateForBroker(kafkaZkClient, 0, throttleRate);
    // we expect broker 1 to be null since all replica movement related to it has completed.
    assertExpectedThrottledRateForBroker(kafkaZkClient, 1, null);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 2, throttleRate);
    // We expect broker 3 to have a throttle on it because there is an in-progress replica being moved
    assertExpectedThrottledRateForBroker(kafkaZkClient, 3, throttleRate);
    // Topic-level throttled replica config value should remain as "*"
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, ReplicationThrottleHelper.WILDCARD_ASTERISK);
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC1, ReplicationThrottleHelper.WILDCARD_ASTERISK);
    // Completing the in-progress task and the "*" should not be cleaned up.
    inProgressTask.completed(3);
    throttleHelper.clearThrottles(Arrays.asList(completedTask, inProgressTask), Collections.emptyList());
    Arrays.asList(0, 1, 2, 3).forEach(i -> assertExpectedThrottledRateForBroker(kafkaZkClient, i, null));
    // Topic-level throttled replica config value should remain as "*"
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, ReplicationThrottleHelper.WILDCARD_ASTERISK);
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC1, ReplicationThrottleHelper.WILDCARD_ASTERISK);
}

Example 27 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ReplicationThrottleHelperTest method testAddingThrottlesWithNoPreExistingThrottles.

@Test
public void testAddingThrottlesWithNoPreExistingThrottles() {
    createTopics();
    KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(zookeeper().connectionString(), "ReplicationThrottleHelperTestMetricGroup", "AddingThrottlesWithNoPreExistingThrottles", false);
    final long throttleRate = 100L;
    ReplicationThrottleHelper throttleHelper = new ReplicationThrottleHelper(kafkaZkClient, throttleRate);
    ExecutionProposal proposal = new ExecutionProposal(new TopicPartition(TOPIC0, 0), 100, new ReplicaPlacementInfo(0), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(2)));
    ExecutionTask task = completedTaskForProposal(0, proposal);
    throttleHelper.setThrottles(Collections.singletonList(proposal));
    assertExpectedThrottledRateForBroker(kafkaZkClient, 0, throttleRate);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 1, throttleRate);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 2, throttleRate);
    // No throttle on broker 3 because it's not involved in any of the execution proposals:
    assertExpectedThrottledRateForBroker(kafkaZkClient, 3, null);
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, "0:0,0:1,0:2");
    // We expect all throttles to be cleaned up
    throttleHelper.clearThrottles(Collections.singletonList(task), Collections.emptyList());
    Arrays.asList(0, 1, 2, 3).forEach(i -> assertExpectedThrottledRateForBroker(kafkaZkClient, i, null));
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, null);
}

Example 28 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ReplicationThrottleHelperTest method testClearThrottleOnNonExistentTopic.

@Test
public void testClearThrottleOnNonExistentTopic() {
    final long throttleRate = 100L;
    final int brokerId0 = 0;
    final int brokerId1 = 1;
    final int brokerId2 = 2;
    final int partitionId = 0;
    // A proposal to move a partition with 2 replicas from broker 0 and 1 to broker 0 and 2
    ExecutionProposal proposal = new ExecutionProposal(new TopicPartition(TOPIC0, partitionId), 100, new ReplicaPlacementInfo(brokerId0), Arrays.asList(new ReplicaPlacementInfo(brokerId0), new ReplicaPlacementInfo(brokerId1)), Arrays.asList(new ReplicaPlacementInfo(brokerId0), new ReplicaPlacementInfo(brokerId2)));
    // Case 1: a situation where Topic0 does not exist. Hence no property is returned upon read.
    KafkaZkClient mockKafkaZkClient = prepareMockKafkaZkClient(new Properties());
    ExecutionTask mockCompleteTask = prepareMockCompleteTask(proposal);
    EasyMock.replay(mockCompleteTask, mockKafkaZkClient);
    ReplicationThrottleHelper throttleHelper = new ReplicationThrottleHelper(mockKafkaZkClient, throttleRate);
    throttleHelper.clearThrottles(Collections.singletonList(mockCompleteTask), Collections.emptyList());
    EasyMock.verify(mockKafkaZkClient, mockCompleteTask);
    // Case 2: a situation where Topic0 gets deleted after its configs were read.
    Properties topicConfigProps = new Properties();
    String throttledReplicas = brokerId0 + "," + brokerId1;
    topicConfigProps.put(ReplicationThrottleHelper.LEADER_THROTTLED_REPLICAS, throttledReplicas);
    topicConfigProps.put(ReplicationThrottleHelper.FOLLOWER_THROTTLED_REPLICAS, throttledReplicas);
    mockKafkaZkClient = prepareMockKafkaZkClient(topicConfigProps);
    EasyMock.expect(mockKafkaZkClient.topicExists(TOPIC0)).andReturn(false).times(2);
    mockCompleteTask = prepareMockCompleteTask(proposal);
    EasyMock.replay(mockCompleteTask, mockKafkaZkClient);
    throttleHelper = new ReplicationThrottleHelper(mockKafkaZkClient, throttleRate);
    // Expect no exception
    throttleHelper.clearThrottles(Collections.singletonList(mockCompleteTask), Collections.emptyList());
    EasyMock.verify(mockKafkaZkClient, mockCompleteTask);
}

Example 29 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutionTaskPlanner method removeInterBrokerReplicaActionForExecution.

private void removeInterBrokerReplicaActionForExecution(ExecutionTask task) {
    int sourceBroker = task.proposal().oldLeader().brokerId();
    _interPartMoveTasksByBrokerId.get(sourceBroker).remove(task);
    for (ReplicaPlacementInfo destinationBroker : task.proposal().replicasToAdd()) {
        _interPartMoveTasksByBrokerId.get(destinationBroker.brokerId()).remove(task);
    }
    _remainingInterBrokerReplicaMovements.remove(task);
}

Example 30 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutionTaskPlanner method getInterBrokerReplicaMovementTasks.

/**
 * Get a list of executable inter-broker replica movements that comply with the concurrency constraint
 * and partitions in move constraint provided.
 *
 * @param readyBrokers The brokers that is ready to execute more movements.
 * @param inProgressPartitions Topic partitions of replicas that are already in progress. This is needed because the
 *                             controller does not allow updating the ongoing replica reassignment for a partition
 *                             whose replica is being reassigned.
 * @param maxInterBrokerPartitionMovements Maximum cap for number of partitions to move at any time
 * @return A list of movements that is executable for the ready brokers.
 */
public List<ExecutionTask> getInterBrokerReplicaMovementTasks(Map<Integer, Integer> readyBrokers, Set<TopicPartition> inProgressPartitions, int maxInterBrokerPartitionMovements) {
    LOG.trace("Getting inter-broker replica movement tasks for brokers with concurrency {}", readyBrokers);
    List<ExecutionTask> executableReplicaMovements = new ArrayList<>();
    SortedSet<Integer> interPartMoveBrokerIds = new TreeSet<>(_interPartMoveBrokerComparator);
    List<Integer> interPartMoveBrokerIdsList = new ArrayList<>(_interPartMoveTasksByBrokerId.keySet().size());
    /*
     * The algorithm avoids unfair situation where the available movement slots of a broker is completely taken
     * by another broker. It checks the proposals in a round-robin manner that makes sure each ready broker gets
     * chances to make progress.
     */
    boolean newTaskAdded = true;
    interPartMoveBrokerIds.addAll(_interPartMoveTasksByBrokerId.keySet());
    Set<Integer> brokerInvolved = new HashSet<>();
    Set<TopicPartition> partitionsInvolved = new HashSet<>();
    int numInProgressPartitions = inProgressPartitions.size();
    boolean maxPartitionMovesReached = false;
    while (newTaskAdded && !maxPartitionMovesReached) {
        newTaskAdded = false;
        brokerInvolved.clear();
        // To avoid ConcurrentModificationException on interPartMoveBrokerId when we remove its elements,
        // preserving the order then iterate on another list interPartMoveBrokerIdsList.
        interPartMoveBrokerIdsList.clear();
        interPartMoveBrokerIdsList.addAll(interPartMoveBrokerIds);
        for (int brokerId : interPartMoveBrokerIdsList) {
            // If max partition moves limit reached, no need to check other brokers
            if (maxPartitionMovesReached) {
                break;
            }
            // If this broker has already involved in this round, skip it.
            if (brokerInvolved.contains(brokerId)) {
                continue;
            }
            // Check the available balancing proposals of this broker to see if we can find one ready to execute.
            SortedSet<ExecutionTask> proposalsForBroker = _interPartMoveTasksByBrokerId.get(brokerId);
            LOG.trace("Execution task for broker {} are {}", brokerId, proposalsForBroker);
            for (ExecutionTask task : proposalsForBroker) {
                // Break if max cap reached
                if (numInProgressPartitions >= maxInterBrokerPartitionMovements) {
                    LOG.trace("In progress Partitions {} reached/exceeded Max partitions to move in cluster {}. " + "Not adding anymore tasks.", numInProgressPartitions, maxInterBrokerPartitionMovements);
                    maxPartitionMovesReached = true;
                    break;
                }
                // Skip this proposal if either source broker or destination broker of this proposal has already
                // involved in this round.
                int sourceBroker = task.proposal().oldLeader().brokerId();
                Set<Integer> destinationBrokers = task.proposal().replicasToAdd().stream().mapToInt(ReplicaPlacementInfo::brokerId).boxed().collect(Collectors.toSet());
                if (brokerInvolved.contains(sourceBroker) || KafkaCruiseControlUtils.containsAny(brokerInvolved, destinationBrokers)) {
                    continue;
                }
                TopicPartition tp = task.proposal().topicPartition();
                // Check if the proposal is executable.
                if (isExecutableProposal(task.proposal(), readyBrokers) && !inProgressPartitions.contains(tp) && !partitionsInvolved.contains(tp)) {
                    partitionsInvolved.add(tp);
                    executableReplicaMovements.add(task);
                    // Record the brokers as involved in this round and stop involving them again in this round.
                    brokerInvolved.add(sourceBroker);
                    brokerInvolved.addAll(destinationBrokers);
                    // The first task of each involved broker might have changed.
                    // Let's remove the brokers before the tasks change, then add them again later by comparing their new first tasks.
                    interPartMoveBrokerIds.remove(sourceBroker);
                    interPartMoveBrokerIds.removeAll(destinationBrokers);
                    // Remove the proposal from the execution plan.
                    removeInterBrokerReplicaActionForExecution(task);
                    interPartMoveBrokerIds.add(sourceBroker);
                    interPartMoveBrokerIds.addAll(destinationBrokers);
                    // Decrement the slots for both source and destination brokers
                    readyBrokers.put(sourceBroker, readyBrokers.get(sourceBroker) - 1);
                    for (int broker : destinationBrokers) {
                        readyBrokers.put(broker, readyBrokers.get(broker) - 1);
                    }
                    // Mark proposal added to true so we will have another round of check.
                    newTaskAdded = true;
                    numInProgressPartitions++;
                    LOG.debug("Found ready task {} for broker {}. Broker concurrency state: {}", task, brokerId, readyBrokers);
                    // We can stop the check for proposals for this broker because we have found a proposal.
                    break;
                }
            }
        }
    }
    return executableReplicaMovements;
}