Examples with ReplicaPlacementInfo com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo used on opensource projects

Example 1 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutionUtils method submitReplicaReassignmentTasks.

/**
 * Submits the given inter-broker replica reassignment tasks for execution using the given admin client.
 *
 * @param adminClient The adminClient to submit new inter-broker replica reassignments.
 * @param tasks Inter-broker replica reassignment tasks to execute.
 * @return The {@link AlterPartitionReassignmentsResult result} of reassignment request -- cannot be {@code null}.
 */
public static AlterPartitionReassignmentsResult submitReplicaReassignmentTasks(AdminClient adminClient, List<ExecutionTask> tasks) {
    if (validateNotNull(tasks, "Tasks to execute cannot be null.").isEmpty()) {
        throw new IllegalArgumentException("Tasks to execute cannot be empty.");
    }
    // Update the ongoing replica reassignments in case the task status has changed.
    Map<TopicPartition, Optional<NewPartitionReassignment>> newReassignments = new HashMap<>();
    for (ExecutionTask task : tasks) {
        TopicPartition tp = task.proposal().topicPartition();
        List<Integer> newReplicas = new ArrayList<>(task.proposal().newReplicas().size());
        for (ReplicaPlacementInfo replicaPlacementInfo : task.proposal().newReplicas()) {
            newReplicas.add(replicaPlacementInfo.brokerId());
        }
        switch(task.state()) {
            case ABORTING:
            case ABORTED:
            case DEAD:
                // A task in one of these states should not have a corresponding partition movement -- cancel it.
                newReassignments.put(tp, cancelReassignmentValue());
                LOG.debug("The ongoing reassignment will be cancelled for task {}.", task);
                break;
            case COMPLETED:
                // No action needed.
                LOG.debug("Task {} has already been completed.", task);
                break;
            case IN_PROGRESS:
                // No need to check whether the topic exists or being deleted as the server response will indicate those.
                // Likewise, no need to check if there is already an ongoing execution for the partition, because if one
                // exists, it will be modified to execute the desired task.
                newReassignments.put(tp, reassignmentValue(newReplicas));
                LOG.debug("Task {} will be executed.", task);
                break;
            default:
                throw new IllegalStateException(String.format("Unrecognized task state %s.", task.state()));
        }
    }
    if (newReassignments.isEmpty()) {
        throw new IllegalArgumentException("All tasks submitted for replica reassignment are already completed.");
    }
    return adminClient.alterPartitionReassignments(newReassignments);
}

Example 2 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutionProposal method hashCode.

@Override
public int hashCode() {
    int result = _tp.hashCode();
    result = 31 * result + _oldLeader.hashCode();
    for (ReplicaPlacementInfo replica : _oldReplicas) {
        result = 31 * result + replica.hashCode();
    }
    for (ReplicaPlacementInfo replica : _newReplicas) {
        result = 31 * replica.hashCode();
    }
    return result;
}

Example 3 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ReplicationThrottleHelperTest method testIsNoOpWhenThrottleIsNull.

@Test
public void testIsNoOpWhenThrottleIsNull() {
    KafkaZkClient mockKafkaZkClient = EasyMock.strictMock(KafkaZkClient.class);
    EasyMock.replay(mockKafkaZkClient);
    // Test would fail on any unexpected interactions with the kafkaZkClient
    ReplicationThrottleHelper throttleHelper = new ReplicationThrottleHelper(mockKafkaZkClient, null);
    ExecutionProposal proposal = new ExecutionProposal(new TopicPartition("topic", 0), 100, new ReplicaPlacementInfo(0), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(2)));
    ExecutionTask task = completedTaskForProposal(0, proposal);
    throttleHelper.setThrottles(Collections.singletonList(proposal));
    throttleHelper.clearThrottles(Collections.singletonList(task), Collections.emptyList());
}

Example 4 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ReplicationThrottleHelperTest method testAddingThrottlesWithPreExistingThrottles.

@Test
public void testAddingThrottlesWithPreExistingThrottles() {
    createTopics();
    KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(zookeeper().connectionString(), "ReplicationThrottleHelperTestMetricGroup", "AddingThrottlesWithNoPreExistingThrottles", false);
    final long throttleRate = 100L;
    ReplicationThrottleHelper throttleHelper = new ReplicationThrottleHelper(kafkaZkClient, throttleRate);
    ExecutionProposal proposal = new ExecutionProposal(new TopicPartition(TOPIC0, 0), 100, new ReplicaPlacementInfo(0), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(2)));
    ExecutionTask task = completedTaskForProposal(0, proposal);
    // Broker 0 has an existing leader and follower throttle; we expect these to be preserved.
    Properties broker0Config = new Properties();
    long preExistingBroker0ThrottleRate = 200L;
    broker0Config.setProperty(ReplicationThrottleHelper.LEADER_THROTTLED_RATE, String.valueOf(preExistingBroker0ThrottleRate));
    broker0Config.setProperty(ReplicationThrottleHelper.FOLLOWER_THROTTLED_RATE, String.valueOf(preExistingBroker0ThrottleRate));
    ExecutorUtils.changeBrokerConfig(new AdminZkClient(kafkaZkClient), 0, broker0Config);
    // Partition 1 (which is not involved in any execution proposal) has pre-existing throttled
    // replicas (on both leaders and followers); we expect these configurations to be merged
    // with our new throttled replicas.
    Properties topic0Config = kafkaZkClient.getEntityConfigs(ConfigType.Topic(), TOPIC0);
    topic0Config.setProperty(ReplicationThrottleHelper.LEADER_THROTTLED_REPLICAS, "1:0,1:1");
    topic0Config.setProperty(ReplicationThrottleHelper.FOLLOWER_THROTTLED_REPLICAS, "1:0,1:1");
    ExecutorUtils.changeTopicConfig(new AdminZkClient(kafkaZkClient), TOPIC0, topic0Config);
    // Topic 1 is not involved in any execution proposal. It has pre-existing throttled replicas.
    Properties topic1Config = kafkaZkClient.getEntityConfigs(ConfigType.Topic(), TOPIC1);
    topic1Config.setProperty(ReplicationThrottleHelper.LEADER_THROTTLED_REPLICAS, "1:1");
    topic1Config.setProperty(ReplicationThrottleHelper.FOLLOWER_THROTTLED_REPLICAS, "1:1");
    ExecutorUtils.changeTopicConfig(new AdminZkClient(kafkaZkClient), TOPIC1, topic1Config);
    throttleHelper.setThrottles(Collections.singletonList(proposal));
    assertExpectedThrottledRateForBroker(kafkaZkClient, 0, preExistingBroker0ThrottleRate);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 1, throttleRate);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 2, throttleRate);
    // No throttle on broker 3 because it's not involved in any of the execution proposals:
    assertExpectedThrottledRateForBroker(kafkaZkClient, 3, null);
    // Existing throttled replicas are merged with new throttled replicas for topic 0:
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, "0:0,0:1,0:2,1:0,1:1");
    // Existing throttled replicas are unchanged for topic 1:
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC1, "1:1");
    throttleHelper.clearThrottles(Collections.singletonList(task), Collections.emptyList());
    // We expect all throttles related to replica movement to be removed. Specifically,
    // any throttles related to partitions which were not moved will remain.
    // However, we do expect the broker throttles to be removed.
    throttleHelper.clearThrottles(Collections.singletonList(task), Collections.emptyList());
    Arrays.asList(0, 1, 2, 3).forEach(i -> assertExpectedThrottledRateForBroker(kafkaZkClient, i, null));
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, "1:0,1:1");
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC1, "1:1");
}

Example 5 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ReplicationThrottleHelperTest method testDoNotRemoveThrottlesForInProgressTasks.

@Test
public void testDoNotRemoveThrottlesForInProgressTasks() {
    createTopics();
    KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(zookeeper().connectionString(), "ReplicationThrottleHelperTestMetricGroup", "AddingThrottlesWithNoPreExistingThrottles", false);
    final long throttleRate = 100L;
    ReplicationThrottleHelper throttleHelper = new ReplicationThrottleHelper(kafkaZkClient, throttleRate);
    ExecutionProposal proposal = new ExecutionProposal(new TopicPartition(TOPIC0, 0), 100, new ReplicaPlacementInfo(0), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(2)));
    ExecutionProposal proposal2 = new ExecutionProposal(new TopicPartition(TOPIC0, 1), 100, new ReplicaPlacementInfo(0), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(3)), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(2)));
    throttleHelper.setThrottles(Arrays.asList(proposal, proposal2));
    ExecutionTask completedTask = completedTaskForProposal(0, proposal);
    ExecutionTask inProgressTask = inProgressTaskForProposal(1, proposal2);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 0, throttleRate);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 1, throttleRate);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 2, throttleRate);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 3, throttleRate);
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, "0:0,0:1,0:2,1:0,1:2,1:3");
    throttleHelper.clearThrottles(Collections.singletonList(completedTask), Collections.singletonList(inProgressTask));
    assertExpectedThrottledRateForBroker(kafkaZkClient, 0, throttleRate);
    // we expect broker 1 to be null since all replica movement related to it has completed.
    assertExpectedThrottledRateForBroker(kafkaZkClient, 1, null);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 2, throttleRate);
    // We expect broker 3 to have a throttle on it because there is an in-progress replica being moved
    assertExpectedThrottledRateForBroker(kafkaZkClient, 3, throttleRate);
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, "1:0,1:2,1:3");
    // passing an inProgress task that is not complete should have no effect.
    throttleHelper.clearThrottles(Collections.singletonList(completedTask), Collections.singletonList(inProgressTask));
    assertExpectedThrottledRateForBroker(kafkaZkClient, 0, throttleRate);
    // we expect broker 1 to be null since all replica movement related to it has completed.
    assertExpectedThrottledRateForBroker(kafkaZkClient, 1, null);
    assertExpectedThrottledRateForBroker(kafkaZkClient, 2, throttleRate);
    // We expect broker 3 to have a throttle on it because there is an in-progress replica being moved
    assertExpectedThrottledRateForBroker(kafkaZkClient, 3, throttleRate);
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, "1:0,1:2,1:3");
    // Completing the in-progress task and clearing the throttles should clean everything up.
    inProgressTask.completed(3);
    throttleHelper.clearThrottles(Arrays.asList(completedTask, inProgressTask), Collections.emptyList());
    Arrays.asList(0, 1, 2, 3).forEach(i -> assertExpectedThrottledRateForBroker(kafkaZkClient, i, null));
    assertExpectedThrottledReplicas(kafkaZkClient, TOPIC0, null);
}