Examples with ReplicaPlacementInfo com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo used on opensource projects

Example 21 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutionTaskPlannerTest method testGetIntraBrokerPartitionMovementTasks.

@Test
public void testGetIntraBrokerPartitionMovementTasks() {
    ReplicaPlacementInfo r0d0 = new ReplicaPlacementInfo(0, "d0");
    ReplicaPlacementInfo r0d1 = new ReplicaPlacementInfo(0, "d1");
    ReplicaPlacementInfo r1d0 = new ReplicaPlacementInfo(1, "d0");
    ReplicaPlacementInfo r1d1 = new ReplicaPlacementInfo(1, "d1");
    List<ExecutionProposal> proposals = Collections.singletonList(new ExecutionProposal(new TopicPartition(TOPIC2, 0), 4, r0d0, Arrays.asList(r0d0, r1d1), Arrays.asList(r1d0, r0d1)));
    TopicPartitionReplica tpr0 = new TopicPartitionReplica(TOPIC2, 0, 0);
    TopicPartitionReplica tpr1 = new TopicPartitionReplica(TOPIC2, 0, 1);
    // Mock adminClient
    AdminClient mockAdminClient = EasyMock.mock(AdminClient.class);
    try {
        // Reflectively set constructors from package private to public.
        Constructor<DescribeReplicaLogDirsResult> constructor1 = DescribeReplicaLogDirsResult.class.getDeclaredConstructor(Map.class);
        constructor1.setAccessible(true);
        Constructor<ReplicaLogDirInfo> constructor2 = ReplicaLogDirInfo.class.getDeclaredConstructor(String.class, long.class, String.class, long.class);
        constructor2.setAccessible(true);
        Map<TopicPartitionReplica, KafkaFuture<ReplicaLogDirInfo>> futureByReplica = new HashMap<>();
        futureByReplica.put(tpr0, completedFuture(constructor2.newInstance("d0", 0L, null, -1L)));
        futureByReplica.put(tpr1, completedFuture(constructor2.newInstance("d1", 0L, null, -1L)));
        EasyMock.expect(mockAdminClient.describeReplicaLogDirs(anyObject())).andReturn(constructor1.newInstance(futureByReplica)).anyTimes();
        EasyMock.replay(mockAdminClient);
    } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
    // Let it go.
    }
    Set<PartitionInfo> partitions = new HashSet<>();
    Node[] isrArray = generateExpectedReplicas(proposals.get(0));
    partitions.add(new PartitionInfo(proposals.get(0).topicPartition().topic(), proposals.get(0).topicPartition().partition(), isrArray[0], isrArray, isrArray));
    Cluster expectedCluster = new Cluster(null, _expectedNodes, partitions, Collections.emptySet(), Collections.emptySet());
    StrategyOptions strategyOptions = new StrategyOptions.Builder(expectedCluster).build();
    ExecutionTaskPlanner planner = new ExecutionTaskPlanner(mockAdminClient, new KafkaCruiseControlConfig(KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties()));
    planner.addExecutionProposals(proposals, strategyOptions, null);
    assertEquals(1, planner.remainingLeadershipMovements().size());
    assertEquals(2, planner.remainingIntraBrokerReplicaMovements().size());
    planner.clear();
    assertEquals(0, planner.remainingLeadershipMovements().size());
    assertEquals(0, planner.remainingIntraBrokerReplicaMovements().size());
    EasyMock.verify(mockAdminClient);
}

Example 22 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutorTest method testSubmitReplicaReassignmentTasksWithEmptyNullOrAllCompletedTasks.

@Test
public void testSubmitReplicaReassignmentTasksWithEmptyNullOrAllCompletedTasks() {
    AdminClient mockAdminClient = EasyMock.mock(AdminClient.class);
    // 1. Verify with an empty task list.
    assertThrows(IllegalArgumentException.class, () -> ExecutionUtils.submitReplicaReassignmentTasks(mockAdminClient, Collections.emptyList()));
    // 2. Verify with null tasks.
    assertThrows(IllegalArgumentException.class, () -> ExecutionUtils.submitReplicaReassignmentTasks(mockAdminClient, null));
    // 2. Verify with all completed tasks.
    TopicPartition tp = new TopicPartition(TOPIC0, 0);
    ExecutionProposal proposal = new ExecutionProposal(tp, 100, new ReplicaPlacementInfo(0), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(2)));
    ExecutionTask task = new ExecutionTask(0, proposal, ExecutionTask.TaskType.INTER_BROKER_REPLICA_ACTION, EXECUTION_ALERTING_THRESHOLD_MS);
    task.inProgress(MOCK_CURRENT_TIME);
    task.completed(MOCK_CURRENT_TIME);
    assertThrows(IllegalArgumentException.class, () -> ExecutionUtils.submitReplicaReassignmentTasks(mockAdminClient, Collections.singletonList(task)));
}

Example 23 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutorTest method testTimeoutAndExecutionStop.

@Test
public void testTimeoutAndExecutionStop() throws InterruptedException, OngoingExecutionException {
    createTopics(0);
    // The proposal tries to move the leader. We fake the replica list to be unchanged so there is no replica
    // movement, but only leader movement.
    ExecutionProposal proposal = new ExecutionProposal(TP1, 0, new ReplicaPlacementInfo(1), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)));
    KafkaCruiseControlConfig configs = new KafkaCruiseControlConfig(getExecutorProperties());
    Time time = new MockTime();
    MetadataClient mockMetadataClient = EasyMock.mock(MetadataClient.class);
    // Fake the metadata to never change so the leader movement will timeout.
    Node node0 = new Node(0, "host0", 100);
    Node node1 = new Node(1, "host1", 100);
    Node[] replicas = new Node[2];
    replicas[0] = node0;
    replicas[1] = node1;
    PartitionInfo partitionInfo = new PartitionInfo(TP1.topic(), TP1.partition(), node1, replicas, replicas);
    Cluster cluster = new Cluster("id", Arrays.asList(node0, node1), Collections.singleton(partitionInfo), Collections.emptySet(), Collections.emptySet());
    MetadataClient.ClusterAndGeneration clusterAndGeneration = new MetadataClient.ClusterAndGeneration(cluster, 0);
    EasyMock.expect(mockMetadataClient.refreshMetadata()).andReturn(clusterAndGeneration).anyTimes();
    EasyMock.expect(mockMetadataClient.cluster()).andReturn(clusterAndGeneration.cluster()).anyTimes();
    LoadMonitor mockLoadMonitor = getMockLoadMonitor();
    AnomalyDetectorManager mockAnomalyDetectorManager = getMockAnomalyDetector(RANDOM_UUID);
    UserTaskManager.UserTaskInfo mockUserTaskInfo = getMockUserTaskInfo();
    // This tests runs two consecutive executions. First one completes w/o error, but the second one with error.
    UserTaskManager mockUserTaskManager = getMockUserTaskManager(RANDOM_UUID, mockUserTaskInfo, Arrays.asList(false, true));
    EasyMock.replay(mockMetadataClient, mockLoadMonitor, mockAnomalyDetectorManager, mockUserTaskInfo, mockUserTaskManager);
    Collection<ExecutionProposal> proposalsToExecute = Collections.singletonList(proposal);
    Executor executor = new Executor(configs, time, new MetricRegistry(), mockMetadataClient, null, mockAnomalyDetectorManager);
    executor.setUserTaskManager(mockUserTaskManager);
    executor.setGeneratingProposalsForExecution(RANDOM_UUID, ExecutorTest.class::getSimpleName, true);
    executor.executeProposals(proposalsToExecute, Collections.emptySet(), null, mockLoadMonitor, null, null, null, null, null, null, null, true, RANDOM_UUID, false, false);
    waitUntilTrue(() -> (executor.state().state() == ExecutorState.State.LEADER_MOVEMENT_TASK_IN_PROGRESS && !executor.inExecutionTasks().isEmpty()), "Leader movement task did not start within the time limit", EXECUTION_DEADLINE_MS, EXECUTION_SHORT_CHECK_MS);
    // Sleep over ExecutorConfig#DEFAULT_LEADER_MOVEMENT_TIMEOUT_MS with some margin for inter-thread synchronization.
    time.sleep(ExecutorConfig.DEFAULT_LEADER_MOVEMENT_TIMEOUT_MS + 1L);
    // The execution should finish.
    waitUntilTrue(() -> (!executor.hasOngoingExecution() && executor.state().state() == ExecutorState.State.NO_TASK_IN_PROGRESS), "Proposal execution did not finish within the time limit", EXECUTION_DEADLINE_MS, EXECUTION_REGULAR_CHECK_MS);
    // The proposal tries to move replicas.
    proposal = new ExecutionProposal(TP1, 0, new ReplicaPlacementInfo(1), Arrays.asList(new ReplicaPlacementInfo(0), new ReplicaPlacementInfo(1)), Arrays.asList(new ReplicaPlacementInfo(1), new ReplicaPlacementInfo(0)));
    Collection<ExecutionProposal> newProposalsToExecute = Collections.singletonList(proposal);
    // Expect exception in case of UUID mismatch between UNKNOWN_UUID and RANDOM_UUID.
    executor.setGeneratingProposalsForExecution(UNKNOWN_UUID, ExecutorTest.class::getSimpleName, true);
    assertThrows(IllegalStateException.class, () -> executor.executeProposals(newProposalsToExecute, Collections.emptySet(), null, mockLoadMonitor, null, null, null, null, null, null, null, true, RANDOM_UUID, false, false));
    executor.failGeneratingProposalsForExecution(UNKNOWN_UUID);
    // Now successfully start the execution..
    executor.setGeneratingProposalsForExecution(RANDOM_UUID, ExecutorTest.class::getSimpleName, true);
    executor.executeProposals(newProposalsToExecute, Collections.emptySet(), null, mockLoadMonitor, null, null, null, null, null, null, null, true, RANDOM_UUID, false, false);
    waitUntilTrue(() -> (executor.state().state() == ExecutorState.State.INTER_BROKER_REPLICA_MOVEMENT_TASK_IN_PROGRESS), "Inter-broker replica movement task did not start within the time limit", EXECUTION_DEADLINE_MS, EXECUTION_SHORT_CHECK_MS);
    // Stop execution.
    executor.userTriggeredStopExecution(false);
    // The execution should finish.
    waitUntilTrue(() -> (!executor.hasOngoingExecution() && executor.state().state() == ExecutorState.State.NO_TASK_IN_PROGRESS), "Proposal execution did not finish within the time limit", EXECUTION_DEADLINE_MS, EXECUTION_REGULAR_CHECK_MS);
    EasyMock.verify(mockMetadataClient, mockLoadMonitor, mockAnomalyDetectorManager, mockUserTaskInfo, mockUserTaskManager);
}

Example 24 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutorTest method testSubmitReplicaReassignmentTasksWithInProgressTaskAndNonExistingTopic.

@Test
public void testSubmitReplicaReassignmentTasksWithInProgressTaskAndNonExistingTopic() throws InterruptedException {
    AdminClient adminClient = KafkaCruiseControlUtils.createAdminClient(Collections.singletonMap(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, broker(0).plaintextAddr()));
    ExecutionProposal proposal = new ExecutionProposal(TP0, 100, new ReplicaPlacementInfo(0), Collections.singletonList(new ReplicaPlacementInfo(0)), Collections.singletonList(new ReplicaPlacementInfo(1)));
    ExecutionTask task = new ExecutionTask(0, proposal, ExecutionTask.TaskType.INTER_BROKER_REPLICA_ACTION, EXECUTION_ALERTING_THRESHOLD_MS);
    task.inProgress(MOCK_CURRENT_TIME);
    AlterPartitionReassignmentsResult result = ExecutionUtils.submitReplicaReassignmentTasks(adminClient, Collections.singletonList(task));
    assertEquals(1, result.values().size());
    // Expect topic deletion -- i.e. or it has never been created.
    assertTrue(verifyFutureError(result.values().get(TP0), Errors.UNKNOWN_TOPIC_OR_PARTITION.exception().getClass()));
}

Example 25 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExecutorTest method testBrokerDiesBeforeMovingPartition.

@Test
public void testBrokerDiesBeforeMovingPartition() throws Exception {
    KafkaZkClient kafkaZkClient = KafkaCruiseControlUtils.createKafkaZkClient(zookeeper().connectionString(), "ExecutorTestMetricGroup", "BrokerDiesBeforeMovingPartition", false);
    try {
        Map<String, TopicDescription> topicDescriptions = createTopics((int) PRODUCE_SIZE_IN_BYTES);
        // initialLeader0 will be alive after killing a broker in cluster.
        int initialLeader0 = topicDescriptions.get(TOPIC0).partitions().get(0).leader().id();
        int initialLeader1 = topicDescriptions.get(TOPIC1).partitions().get(0).leader().id();
        // Kill broker before starting the reassignment.
        _brokers.get(initialLeader0 == 0 ? 1 : 0).shutdown();
        ExecutionProposal proposal0 = new ExecutionProposal(TP0, PRODUCE_SIZE_IN_BYTES, new ReplicaPlacementInfo(initialLeader0), Collections.singletonList(new ReplicaPlacementInfo(initialLeader0)), Collections.singletonList(new ReplicaPlacementInfo(initialLeader0 == 0 ? 1 : 0)));
        ExecutionProposal proposal1 = new ExecutionProposal(TP1, 0, new ReplicaPlacementInfo(initialLeader1), Arrays.asList(new ReplicaPlacementInfo(initialLeader1), new ReplicaPlacementInfo(initialLeader1 == 0 ? 1 : 0)), Arrays.asList(new ReplicaPlacementInfo(initialLeader1 == 0 ? 1 : 0), new ReplicaPlacementInfo(initialLeader1)));
        Collection<ExecutionProposal> proposalsToExecute = Arrays.asList(proposal0, proposal1);
        executeAndVerifyProposals(kafkaZkClient, proposalsToExecute, Collections.emptyList(), true, null, false, false);
        // We are doing the rollback. -- The leadership should be on the alive broker.
        assertEquals(initialLeader0, kafkaZkClient.getLeaderForPartition(TP0).get());
        assertEquals(initialLeader0, kafkaZkClient.getLeaderForPartition(TP1).get());
    } finally {
        KafkaCruiseControlUtils.closeKafkaZkClientWithTimeout(kafkaZkClient);
    }
}