Examples with StrategyOptions com.linkedin.kafka.cruisecontrol.executor.strategy.StrategyOptions used on opensource projects

Example 1 with StrategyOptions

use of com.linkedin.kafka.cruisecontrol.executor.strategy.StrategyOptions in project cruise-control by linkedin.

the class ExecutionTaskPlannerTest method testGetInterBrokerPartitionMovementWithMinIsrTasks.

@Test
public void testGetInterBrokerPartitionMovementWithMinIsrTasks() {
    List<ExecutionProposal> proposals = new ArrayList<>();
    proposals.add(_rf4PartitionMovement0);
    proposals.add(_rf4PartitionMovement1);
    proposals.add(_rf4PartitionMovement2);
    proposals.add(_rf4PartitionMovement3);
    // Test PrioritizeOneAboveMinIsrWithOfflineReplicasStrategy execution strategies.
    // Create prioritizeOneAboveMinIsrMovementPlanner, chain after prioritizeMinIsr strategy
    Properties prioritizeOneAboveMinIsrMovementProps = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    prioritizeOneAboveMinIsrMovementProps.setProperty(ExecutorConfig.DEFAULT_REPLICA_MOVEMENT_STRATEGIES_CONFIG, String.format("%s,%s", PrioritizeMinIsrWithOfflineReplicasStrategy.class.getName(), PrioritizeOneAboveMinIsrWithOfflineReplicasStrategy.class.getName()));
    ExecutionTaskPlanner prioritizeOneAboveMinIsrMovementPlanner = new ExecutionTaskPlanner(null, new KafkaCruiseControlConfig(prioritizeOneAboveMinIsrMovementProps));
    Set<PartitionInfo> partitions = new HashSet<>();
    partitions.add(generatePartitionInfo(_rf4PartitionMovement0, false));
    partitions.add(generatePartitionInfoWithUrpHavingOfflineReplica(_rf4PartitionMovement1, 1));
    partitions.add(generatePartitionInfoWithUrpHavingOfflineReplica(_rf4PartitionMovement2, 3));
    partitions.add(generatePartitionInfoWithUrpHavingOfflineReplica(_rf4PartitionMovement3, 2));
    Cluster expectedCluster = new Cluster(null, _rf4ExpectedNodes, partitions, Collections.emptySet(), Collections.emptySet());
    // Setting topic min ISR to 2
    Map<String, MinIsrWithTime> minIsrWithTimeByTopic = Collections.singletonMap(TOPIC3, new MinIsrWithTime((short) 2, 0));
    StrategyOptions strategyOptions = new StrategyOptions.Builder(expectedCluster).minIsrWithTimeByTopic(minIsrWithTimeByTopic).build();
    Map<Integer, Integer> readyBrokers = new HashMap<>();
    readyBrokers.put(0, 5);
    readyBrokers.put(1, 6);
    readyBrokers.put(2, 6);
    readyBrokers.put(3, 6);
    readyBrokers.put(4, 5);
    readyBrokers.put(5, 6);
    prioritizeOneAboveMinIsrMovementPlanner.addExecutionProposals(proposals, strategyOptions, null);
    List<ExecutionTask> partitionMovementTasks = prioritizeOneAboveMinIsrMovementPlanner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _rf4PartitionMovement2, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _rf4PartitionMovement3, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _rf4PartitionMovement1, partitionMovementTasks.get(2).proposal());
    assertEquals("Fourth task", _rf4PartitionMovement0, partitionMovementTasks.get(3).proposal());
}

Example 2 with StrategyOptions

use of com.linkedin.kafka.cruisecontrol.executor.strategy.StrategyOptions in project cruise-control by linkedin.

the class ExecutionTaskManagerTest method testStateChangeSequences.

@Test
public void testStateChangeSequences() {
    TopicPartition tp = new TopicPartition("topic", 0);
    List<List<ExecutionTaskState>> testSequences = new ArrayList<>();
    // Completed successfully.
    testSequences.add(Arrays.asList(ExecutionTaskState.IN_PROGRESS, ExecutionTaskState.COMPLETED));
    // Rollback succeeded.
    testSequences.add(Arrays.asList(ExecutionTaskState.IN_PROGRESS, ExecutionTaskState.ABORTING, ExecutionTaskState.ABORTED));
    // Rollback failed.
    testSequences.add(Arrays.asList(ExecutionTaskState.IN_PROGRESS, ExecutionTaskState.ABORTING, ExecutionTaskState.DEAD));
    // Cannot rollback.
    testSequences.add(Arrays.asList(ExecutionTaskState.IN_PROGRESS, ExecutionTaskState.DEAD));
    ReplicaPlacementInfo r0 = new ReplicaPlacementInfo(0);
    ReplicaPlacementInfo r1 = new ReplicaPlacementInfo(1);
    ReplicaPlacementInfo r2 = new ReplicaPlacementInfo(2);
    // Make sure the proposal does not involve leader movement.
    ExecutionProposal proposal = new ExecutionProposal(tp, 10, r2, Arrays.asList(r0, r2), Arrays.asList(r2, r1));
    StrategyOptions strategyOptions = new StrategyOptions.Builder(generateExpectedCluster(proposal)).build();
    for (List<ExecutionTaskState> sequence : testSequences) {
        taskManager.clear();
        taskManager.setExecutionModeForTaskTracker(false);
        taskManager.addExecutionProposals(Collections.singletonList(proposal), Collections.emptySet(), strategyOptions, null);
        taskManager.setRequestedInterBrokerPartitionMovementConcurrency(null);
        taskManager.setRequestedIntraBrokerPartitionMovementConcurrency(null);
        taskManager.setRequestedLeadershipMovementConcurrency(null);
        List<ExecutionTask> tasks = taskManager.getInterBrokerReplicaMovementTasks();
        assertEquals(1, tasks.size());
        ExecutionTask task = tasks.get(0);
        verifyStateChangeSequence(sequence, task, taskManager);
    }
    // Verify that the movement concurrency matches the default configuration
    for (ConcurrencyType concurrencyType : ConcurrencyType.cachedValues()) {
        assertEquals(MOCK_DEFAULT_CONCURRENCY.get(concurrencyType).intValue(), taskManager.movementConcurrency(concurrencyType));
    }
}

Example 3 with StrategyOptions

use of com.linkedin.kafka.cruisecontrol.executor.strategy.StrategyOptions in project cruise-control by linkedin.

the class ExecutionTaskPlannerTest method testGetLeaderMovementTasks.

@Test
public void testGetLeaderMovementTasks() {
    List<ExecutionProposal> proposals = new ArrayList<>();
    proposals.add(_leaderMovement1);
    proposals.add(_leaderMovement2);
    proposals.add(_leaderMovement3);
    proposals.add(_leaderMovement4);
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    props.setProperty(ExecutorConfig.DEFAULT_REPLICA_MOVEMENT_STRATEGIES_CONFIG, "");
    ExecutionTaskPlanner planner = new ExecutionTaskPlanner(null, new KafkaCruiseControlConfig(props));
    Set<PartitionInfo> partitions = new HashSet<>();
    for (ExecutionProposal proposal : proposals) {
        partitions.add(generatePartitionInfo(proposal, false));
    }
    Cluster expectedCluster = new Cluster(null, _expectedNodes, partitions, Collections.emptySet(), Collections.emptySet());
    StrategyOptions strategyOptions = new StrategyOptions.Builder(expectedCluster).build();
    planner.addExecutionProposals(proposals, strategyOptions, null);
    List<ExecutionTask> leaderMovementTasks = planner.getLeadershipMovementTasks(2);
    assertEquals("2 of the leader movements should return in one batch", 2, leaderMovementTasks.size());
    assertEquals(4, leaderMovementTasks.get(0).executionId());
    assertEquals(leaderMovementTasks.get(0).proposal(), _leaderMovement1);
    assertEquals(5, leaderMovementTasks.get(1).executionId());
    assertEquals(leaderMovementTasks.get(1).proposal(), _leaderMovement2);
    leaderMovementTasks = planner.getLeadershipMovementTasks(2);
    assertEquals("2 of the leader movements should return in one batch", 2, leaderMovementTasks.size());
    assertEquals(6, leaderMovementTasks.get(0).executionId());
    assertEquals(leaderMovementTasks.get(0).proposal(), _leaderMovement3);
    assertEquals(7, leaderMovementTasks.get(1).executionId());
    assertEquals(leaderMovementTasks.get(1).proposal(), _leaderMovement4);
}

Example 4 with StrategyOptions

use of com.linkedin.kafka.cruisecontrol.executor.strategy.StrategyOptions in project cruise-control by linkedin.

the class ExecutionTaskPlannerTest method testGetIntraBrokerPartitionMovementTasks.

@Test
public void testGetIntraBrokerPartitionMovementTasks() {
    ReplicaPlacementInfo r0d0 = new ReplicaPlacementInfo(0, "d0");
    ReplicaPlacementInfo r0d1 = new ReplicaPlacementInfo(0, "d1");
    ReplicaPlacementInfo r1d0 = new ReplicaPlacementInfo(1, "d0");
    ReplicaPlacementInfo r1d1 = new ReplicaPlacementInfo(1, "d1");
    List<ExecutionProposal> proposals = Collections.singletonList(new ExecutionProposal(new TopicPartition(TOPIC2, 0), 4, r0d0, Arrays.asList(r0d0, r1d1), Arrays.asList(r1d0, r0d1)));
    TopicPartitionReplica tpr0 = new TopicPartitionReplica(TOPIC2, 0, 0);
    TopicPartitionReplica tpr1 = new TopicPartitionReplica(TOPIC2, 0, 1);
    // Mock adminClient
    AdminClient mockAdminClient = EasyMock.mock(AdminClient.class);
    try {
        // Reflectively set constructors from package private to public.
        Constructor<DescribeReplicaLogDirsResult> constructor1 = DescribeReplicaLogDirsResult.class.getDeclaredConstructor(Map.class);
        constructor1.setAccessible(true);
        Constructor<ReplicaLogDirInfo> constructor2 = ReplicaLogDirInfo.class.getDeclaredConstructor(String.class, long.class, String.class, long.class);
        constructor2.setAccessible(true);
        Map<TopicPartitionReplica, KafkaFuture<ReplicaLogDirInfo>> futureByReplica = new HashMap<>();
        futureByReplica.put(tpr0, completedFuture(constructor2.newInstance("d0", 0L, null, -1L)));
        futureByReplica.put(tpr1, completedFuture(constructor2.newInstance("d1", 0L, null, -1L)));
        EasyMock.expect(mockAdminClient.describeReplicaLogDirs(anyObject())).andReturn(constructor1.newInstance(futureByReplica)).anyTimes();
        EasyMock.replay(mockAdminClient);
    } catch (InstantiationException | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
    // Let it go.
    }
    Set<PartitionInfo> partitions = new HashSet<>();
    Node[] isrArray = generateExpectedReplicas(proposals.get(0));
    partitions.add(new PartitionInfo(proposals.get(0).topicPartition().topic(), proposals.get(0).topicPartition().partition(), isrArray[0], isrArray, isrArray));
    Cluster expectedCluster = new Cluster(null, _expectedNodes, partitions, Collections.emptySet(), Collections.emptySet());
    StrategyOptions strategyOptions = new StrategyOptions.Builder(expectedCluster).build();
    ExecutionTaskPlanner planner = new ExecutionTaskPlanner(mockAdminClient, new KafkaCruiseControlConfig(KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties()));
    planner.addExecutionProposals(proposals, strategyOptions, null);
    assertEquals(1, planner.remainingLeadershipMovements().size());
    assertEquals(2, planner.remainingIntraBrokerReplicaMovements().size());
    planner.clear();
    assertEquals(0, planner.remainingLeadershipMovements().size());
    assertEquals(0, planner.remainingIntraBrokerReplicaMovements().size());
    EasyMock.verify(mockAdminClient);
}

Example 5 with StrategyOptions

use of com.linkedin.kafka.cruisecontrol.executor.strategy.StrategyOptions in project cruise-control by linkedin.

the class ExecutionTaskPlannerTest method testClear.

@Test
public void testClear() {
    List<ExecutionProposal> proposals = new ArrayList<>();
    proposals.add(_leaderMovement1);
    proposals.add(_partitionMovement0);
    ExecutionTaskPlanner planner = new ExecutionTaskPlanner(null, new KafkaCruiseControlConfig(KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties()));
    Set<PartitionInfo> partitions = new HashSet<>();
    partitions.add(generatePartitionInfo(_leaderMovement1, false));
    partitions.add(generatePartitionInfo(_partitionMovement0, false));
    Cluster expectedCluster = new Cluster(null, _expectedNodes, partitions, Collections.emptySet(), Collections.emptySet());
    StrategyOptions strategyOptions = new StrategyOptions.Builder(expectedCluster).build();
    planner.addExecutionProposals(proposals, strategyOptions, null);
    assertEquals(2, planner.remainingLeadershipMovements().size());
    assertEquals(2, planner.remainingInterBrokerReplicaMovements().size());
    planner.clear();
    assertEquals(0, planner.remainingLeadershipMovements().size());
    assertEquals(0, planner.remainingInterBrokerReplicaMovements().size());
}