Examples with ReplicaPlacementInfo com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo used on opensource projects

Example 6 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ReplicationThrottleHelperTest method testSetThrottleOnNonExistentTopic.

@Test
public void testSetThrottleOnNonExistentTopic() {
    final long throttleRate = 100L;
    final int brokerId0 = 0;
    final int brokerId1 = 1;
    final int brokerId2 = 2;
    final int partitionId = 0;
    // A proposal to move a partition with 2 replicas from broker 0 and 1 to broker 0 and 2
    ExecutionProposal proposal = new ExecutionProposal(new TopicPartition(TOPIC0, partitionId), 100, new ReplicaPlacementInfo(brokerId0), Arrays.asList(new ReplicaPlacementInfo(brokerId0), new ReplicaPlacementInfo(brokerId1)), Arrays.asList(new ReplicaPlacementInfo(brokerId0), new ReplicaPlacementInfo(brokerId2)));
    KafkaZkClient mockKafkaZkClient = EasyMock.mock(KafkaZkClient.class);
    prepareKafkaZkClientMockWithBrokerConfigs(mockKafkaZkClient, 2);
    // Case 1: a situation where Topic0 does not exist. Hence no property is returned upon read.
    EasyMock.expect(mockKafkaZkClient.getEntityConfigs(ConfigType.Topic(), TOPIC0)).andReturn(new Properties()).times(2);
    EasyMock.expect(mockKafkaZkClient.topicExists(TOPIC0)).andReturn(false).times(4);
    EasyMock.replay(mockKafkaZkClient);
    ReplicationThrottleHelper throttleHelper = new ReplicationThrottleHelper(mockKafkaZkClient, throttleRate);
    throttleHelper.setThrottles(Collections.singletonList(proposal));
    EasyMock.verify(mockKafkaZkClient);
    // Case 2: a situation where Topic0 gets deleted after its configs were read. Change configs should not fail.
    EasyMock.reset(mockKafkaZkClient);
    prepareKafkaZkClientMockWithBrokerConfigs(mockKafkaZkClient, 2);
    Properties topicConfigProps = new Properties();
    String throttledReplicas = brokerId0 + "," + brokerId1;
    topicConfigProps.put(ReplicationThrottleHelper.LEADER_THROTTLED_REPLICAS, throttledReplicas);
    topicConfigProps.put(ReplicationThrottleHelper.FOLLOWER_THROTTLED_REPLICAS, throttledReplicas);
    EasyMock.expect(mockKafkaZkClient.getEntityConfigs(ConfigType.Topic(), TOPIC0)).andReturn(topicConfigProps).times(2);
    EasyMock.expect(mockKafkaZkClient.topicExists(TOPIC0)).andReturn(false).times(4);
    EasyMock.replay(mockKafkaZkClient);
    // Expect no exception
    throttleHelper.setThrottles(Collections.singletonList(proposal));
    EasyMock.verify(mockKafkaZkClient);
}

Example 7 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutorTest method testSubmitReplicaReassignmentTasksWithDeadTaskAndNoReassignmentInProgress.

@Test
public void testSubmitReplicaReassignmentTasksWithDeadTaskAndNoReassignmentInProgress() throws InterruptedException {
    AdminClient adminClient = KafkaCruiseControlUtils.createAdminClient(Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, broker(0).plaintextAddr()));
    TopicPartition tp = new TopicPartition(TOPIC0, 0);
    ExecutionProposal proposal = new ExecutionProposal(tp, 100, new ReplicaPlacementInfo(0), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(2)));
    ExecutionTask task = new ExecutionTask(0, proposal, ExecutionTask.TaskType.INTER_BROKER_REPLICA_ACTION, EXECUTION_ALERTING_THRESHOLD_MS);
    task.inProgress(MOCK_CURRENT_TIME);
    task.kill(MOCK_CURRENT_TIME);
    AlterPartitionReassignmentsResult result = ExecutionUtils.submitReplicaReassignmentTasks(adminClient, Collections.singletonList(task));
    assertEquals(1, result.values().size());
    assertTrue(verifyFutureError(result.values().get(tp), Errors.NO_REASSIGNMENT_IN_PROGRESS.exception().getClass()));
}

Example 8 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutorTest method populateProposals.

/**
 * Proposal#1: [TPO] move from original broker to the other one -- e.g. 0 -> 1
 * Proposal#2: [TP1] change order and leader -- e.g. [0, 1] -> [1, 0]
 * @param proposalToExecute Proposals to execute.
 * @param proposalToVerify Proposals to verify.
 * @param topicSize Topic size in bytes.
 */
private void populateProposals(List<ExecutionProposal> proposalToExecute, List<ExecutionProposal> proposalToVerify, long topicSize) throws InterruptedException {
    Map<String, TopicDescription> topicDescriptions = createTopics((int) topicSize);
    int initialLeader0 = topicDescriptions.get(TOPIC0).partitions().get(0).leader().id();
    int initialLeader1 = topicDescriptions.get(TOPIC1).partitions().get(0).leader().id();
    // Valid proposals
    ExecutionProposal proposal0 = new ExecutionProposal(TP0, topicSize, new ReplicaPlacementInfo(initialLeader0), Collections.singletonList(new ReplicaPlacementInfo(initialLeader0)), Collections.singletonList(new ReplicaPlacementInfo(initialLeader0 == 0 ? 1 : 0)));
    ExecutionProposal proposal1 = new ExecutionProposal(TP1, 0, new ReplicaPlacementInfo(initialLeader1), Arrays.asList(new ReplicaPlacementInfo(initialLeader1), new ReplicaPlacementInfo(initialLeader1 == 0 ? 1 : 0)), Arrays.asList(new ReplicaPlacementInfo(initialLeader1 == 0 ? 1 : 0), new ReplicaPlacementInfo(initialLeader1)));
    // Invalid proposals, the targeting topics of these proposals does not exist.
    ExecutionProposal proposal2 = new ExecutionProposal(TP2, 0, new ReplicaPlacementInfo(initialLeader0), Collections.singletonList(new ReplicaPlacementInfo(initialLeader0)), Collections.singletonList(new ReplicaPlacementInfo(initialLeader0 == 0 ? 1 : 0)));
    ExecutionProposal proposal3 = new ExecutionProposal(TP3, 0, new ReplicaPlacementInfo(initialLeader1), Arrays.asList(new ReplicaPlacementInfo(initialLeader1), new ReplicaPlacementInfo(initialLeader1 == 0 ? 1 : 0)), Arrays.asList(new ReplicaPlacementInfo(initialLeader1 == 0 ? 1 : 0), new ReplicaPlacementInfo(initialLeader1)));
    proposalToExecute.addAll(Arrays.asList(proposal0, proposal1, proposal2, proposal3));
    proposalToVerify.addAll(Arrays.asList(proposal0, proposal1));
}

Example 9 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutorTest method testSubmitPreferredLeaderElection.

@Test
public void testSubmitPreferredLeaderElection() throws InterruptedException, ExecutionException {
    Map<String, TopicDescription> topicDescriptions = createTopics(0);
    AdminClient adminClient = KafkaCruiseControlUtils.createAdminClient(Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, broker(0).plaintextAddr()));
    // Case 1: Handle the successful case.
    ReplicaPlacementInfo oldLeader = new ReplicaPlacementInfo(topicDescriptions.get(TP1.topic()).partitions().get(PARTITION).leader().id());
    List<ReplicaPlacementInfo> replicas = new ArrayList<>();
    for (Node node : topicDescriptions.get(TP1.topic()).partitions().get(PARTITION).replicas()) {
        replicas.add(new ReplicaPlacementInfo(node.id()));
    }
    List<ReplicaPlacementInfo> newReplicas = new ArrayList<>(replicas.size());
    for (int i = replicas.size() - 1; i >= 0; i--) {
        newReplicas.add(new ReplicaPlacementInfo(replicas.get(i).brokerId()));
    }
    // Reorder replicas before leadership election to change the preferred replica.
    ExecutionTask swapTask = new ExecutionTask(0, new ExecutionProposal(TP1, 0, oldLeader, replicas, newReplicas), ExecutionTask.TaskType.INTER_BROKER_REPLICA_ACTION, EXECUTION_ALERTING_THRESHOLD_MS);
    swapTask.inProgress(MOCK_CURRENT_TIME);
    AlterPartitionReassignmentsResult swapResult = ExecutionUtils.submitReplicaReassignmentTasks(adminClient, Collections.singletonList(swapTask));
    assertEquals(1, swapResult.values().size());
    // Can retrieve the future if it is successful.
    swapResult.values().get(TP1).get();
    // Leadership task to transfer the leadership to the preferred replica.
    ExecutionTask task = new ExecutionTask(0, new ExecutionProposal(TP1, 0, oldLeader, newReplicas, newReplicas), ExecutionTask.TaskType.LEADER_ACTION, EXECUTION_ALERTING_THRESHOLD_MS);
    task.inProgress(MOCK_CURRENT_TIME);
    ElectLeadersResult result = ExecutionUtils.submitPreferredLeaderElection(adminClient, Collections.singletonList(task));
    assertTrue(result.partitions().get().get(TP1).isEmpty());
    // Case 2: Handle "election not needed" -- i.e. the leader is already the preferred replica for TP0.
    oldLeader = new ReplicaPlacementInfo(topicDescriptions.get(TP0.topic()).partitions().get(PARTITION).leader().id());
    replicas.clear();
    for (Node node : topicDescriptions.get(TP0.topic()).partitions().get(PARTITION).replicas()) {
        replicas.add(new ReplicaPlacementInfo(node.id()));
    }
    task = new ExecutionTask(0, new ExecutionProposal(TP0, 0, oldLeader, replicas, replicas), ExecutionTask.TaskType.LEADER_ACTION, EXECUTION_ALERTING_THRESHOLD_MS);
    task.inProgress(MOCK_CURRENT_TIME);
    result = ExecutionUtils.submitPreferredLeaderElection(adminClient, Collections.singletonList(task));
    assertTrue(result.partitions().get().get(TP0).isPresent());
    assertEquals(Errors.ELECTION_NOT_NEEDED.exception().getClass(), result.partitions().get().get(TP0).get().getClass());
    // Case 3: Handle "unknown topic or partition" -- i.e. the partition TP2 does not exist, maybe it existed at some point and then deleted
    task = new ExecutionTask(0, new ExecutionProposal(TP2, 0, oldLeader, replicas, replicas), ExecutionTask.TaskType.LEADER_ACTION, EXECUTION_ALERTING_THRESHOLD_MS);
    task.inProgress(MOCK_CURRENT_TIME);
    result = ExecutionUtils.submitPreferredLeaderElection(adminClient, Collections.singletonList(task));
    assertTrue(result.partitions().get().get(TP2).isPresent());
    assertEquals(Errors.UNKNOWN_TOPIC_OR_PARTITION.exception().getClass(), result.partitions().get().get(TP2).get().getClass());
    // Case 4: Handle the scenario, where we tried to elect the preferred leader but it is offline.
    _brokers.get(0).shutdown();
    task = new ExecutionTask(0, new ExecutionProposal(TP0, 0, oldLeader, replicas, replicas), ExecutionTask.TaskType.LEADER_ACTION, EXECUTION_ALERTING_THRESHOLD_MS);
    task.inProgress(MOCK_CURRENT_TIME);
    result = ExecutionUtils.submitPreferredLeaderElection(adminClient, Collections.singletonList(task));
    assertTrue(result.partitions().get().get(TP0).isPresent());
    assertEquals(Errors.PREFERRED_LEADER_NOT_AVAILABLE.exception().getClass(), result.partitions().get().get(TP0).get().getClass());
}

Example 10 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutorTest method executeAndVerifyProposals.

private void executeAndVerifyProposals(KafkaZkClient kafkaZkClient, Collection<ExecutionProposal> proposalsToExecute, Collection<ExecutionProposal> proposalsToCheck, boolean completeWithError, Long replicationThrottle, boolean verifyProgress, boolean isTriggeredByUserRequest) throws OngoingExecutionException {
    KafkaCruiseControlConfig configs = new KafkaCruiseControlConfig(getExecutorProperties());
    UserTaskManager.UserTaskInfo mockUserTaskInfo = getMockUserTaskInfo();
    UserTaskManager mockUserTaskManager = isTriggeredByUserRequest ? getMockUserTaskManager(RANDOM_UUID, mockUserTaskInfo, Collections.singletonList(completeWithError)) : null;
    ExecutorNotifier mockExecutorNotifier = EasyMock.mock(ExecutorNotifier.class);
    LoadMonitor mockLoadMonitor = getMockLoadMonitor();
    Capture<String> captureMessage = Capture.newInstance(CaptureType.FIRST);
    AnomalyDetectorManager mockAnomalyDetectorManager = getMockAnomalyDetector(RANDOM_UUID);
    if (completeWithError) {
        mockExecutorNotifier.sendAlert(EasyMock.capture(captureMessage));
    } else {
        mockExecutorNotifier.sendNotification(EasyMock.capture(captureMessage));
    }
    if (isTriggeredByUserRequest) {
        EasyMock.replay(mockUserTaskInfo, mockUserTaskManager, mockExecutorNotifier, mockLoadMonitor, mockAnomalyDetectorManager);
    } else {
        EasyMock.replay(mockUserTaskInfo, mockExecutorNotifier, mockLoadMonitor, mockAnomalyDetectorManager);
    }
    Executor executor = new Executor(configs, new SystemTime(), new MetricRegistry(), null, mockExecutorNotifier, mockAnomalyDetectorManager);
    executor.setUserTaskManager(mockUserTaskManager);
    Map<TopicPartition, Integer> replicationFactors = new HashMap<>();
    for (ExecutionProposal proposal : proposalsToCheck) {
        TopicPartition tp = new TopicPartition(proposal.topic(), proposal.partitionId());
        replicationFactors.put(tp, proposal.oldReplicas().size());
    }
    executor.setGeneratingProposalsForExecution(RANDOM_UUID, ExecutorTest.class::getSimpleName, isTriggeredByUserRequest);
    executor.executeProposals(proposalsToExecute, Collections.emptySet(), null, mockLoadMonitor, null, null, null, null, null, null, replicationThrottle, isTriggeredByUserRequest, RANDOM_UUID, false, false);
    if (verifyProgress) {
        verifyOngoingPartitionReassignments(Collections.singleton(TP0));
    }
    waitUntilTrue(() -> (!executor.hasOngoingExecution() && executor.state().state() == ExecutorState.State.NO_TASK_IN_PROGRESS), "Proposal execution did not finish within the time limit", EXECUTION_DEADLINE_MS, EXECUTION_REGULAR_CHECK_MS);
    // Check notification is sent after execution has finished.
    String notification = captureMessage.getValue();
    assertTrue(notification.contains(RANDOM_UUID));
    for (ExecutionProposal proposal : proposalsToCheck) {
        TopicPartition tp = new TopicPartition(proposal.topic(), proposal.partitionId());
        int expectedReplicationFactor = replicationFactors.get(tp);
        assertEquals("Replication factor for partition " + tp + " should be " + expectedReplicationFactor, expectedReplicationFactor, kafkaZkClient.getReplicasForPartition(tp).size());
        if (proposal.hasReplicaAction()) {
            for (ReplicaPlacementInfo r : proposal.newReplicas()) {
                assertTrue("The partition should have moved for " + tp, kafkaZkClient.getReplicasForPartition(tp).contains(r.brokerId()));
            }
        }
        assertEquals("The leader should have moved for " + tp, proposal.newLeader().brokerId(), kafkaZkClient.getLeaderForPartition(tp).get());
    }
    if (isTriggeredByUserRequest) {
        EasyMock.verify(mockUserTaskInfo, mockUserTaskManager, mockExecutorNotifier, mockLoadMonitor, mockAnomalyDetectorManager);
    } else {
        EasyMock.verify(mockUserTaskInfo, mockExecutorNotifier, mockLoadMonitor, mockAnomalyDetectorManager);
    }
}