Examples with StrategyOptions com.linkedin.kafka.cruisecontrol.executor.strategy.StrategyOptions used on opensource projects

Example 6 with StrategyOptions

use of com.linkedin.kafka.cruisecontrol.executor.strategy.StrategyOptions in project cruise-control by linkedin.

the class ExecutionTaskPlannerTest method testDynamicConfigReplicaMovementStrategy.

@Test
public void testDynamicConfigReplicaMovementStrategy() {
    List<ExecutionProposal> proposals = new ArrayList<>();
    proposals.add(_partitionMovement0);
    proposals.add(_partitionMovement1);
    proposals.add(_partitionMovement2);
    proposals.add(_partitionMovement3);
    ExecutionTaskPlanner planner = new ExecutionTaskPlanner(null, new KafkaCruiseControlConfig(KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties()));
    Set<PartitionInfo> partitions = new HashSet<>();
    partitions.add(generatePartitionInfo(_partitionMovement0, true));
    partitions.add(generatePartitionInfo(_partitionMovement1, false));
    partitions.add(generatePartitionInfo(_partitionMovement2, true));
    partitions.add(generatePartitionInfo(_partitionMovement3, false));
    Cluster expectedCluster = new Cluster(null, _expectedNodes, partitions, Collections.emptySet(), Collections.emptySet());
    StrategyOptions strategyOptions = new StrategyOptions.Builder(expectedCluster).build();
    Map<Integer, Integer> readyBrokers = new HashMap<>();
    readyBrokers.put(0, 8);
    readyBrokers.put(1, 8);
    readyBrokers.put(2, 8);
    readyBrokers.put(3, 8);
    planner.addExecutionProposals(proposals, strategyOptions, null);
    List<ExecutionTask> partitionMovementTasks = planner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _partitionMovement0, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _partitionMovement2, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _partitionMovement1, partitionMovementTasks.get(2).proposal());
    planner.addExecutionProposals(proposals, strategyOptions, new PostponeUrpReplicaMovementStrategy());
    partitionMovementTasks = planner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _partitionMovement1, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _partitionMovement3, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _partitionMovement0, partitionMovementTasks.get(2).proposal());
    planner.addExecutionProposals(proposals, strategyOptions, new PrioritizeLargeReplicaMovementStrategy());
    partitionMovementTasks = planner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _partitionMovement1, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _partitionMovement3, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _partitionMovement2, partitionMovementTasks.get(2).proposal());
    planner.addExecutionProposals(proposals, strategyOptions, new PrioritizeSmallReplicaMovementStrategy());
    partitionMovementTasks = planner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _partitionMovement0, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _partitionMovement2, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _partitionMovement3, partitionMovementTasks.get(2).proposal());
    assertEquals("Fourth task", _partitionMovement1, partitionMovementTasks.get(3).proposal());
}

Example 7 with StrategyOptions

use of com.linkedin.kafka.cruisecontrol.executor.strategy.StrategyOptions in project cruise-control by linkedin.

the class ExecutionTaskPlannerTest method testGetInterBrokerPartitionMovementTasks.

@Test
public void testGetInterBrokerPartitionMovementTasks() {
    List<ExecutionProposal> proposals = new ArrayList<>();
    proposals.add(_partitionMovement0);
    proposals.add(_partitionMovement1);
    proposals.add(_partitionMovement2);
    proposals.add(_partitionMovement3);
    // Test different execution strategies.
    ExecutionTaskPlanner basePlanner = new ExecutionTaskPlanner(null, new KafkaCruiseControlConfig(KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties()));
    // Create postponeUrpPlanner
    Properties postponeUrpProps = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    postponeUrpProps.setProperty(ExecutorConfig.DEFAULT_REPLICA_MOVEMENT_STRATEGIES_CONFIG, PostponeUrpReplicaMovementStrategy.class.getName());
    ExecutionTaskPlanner postponeUrpPlanner = new ExecutionTaskPlanner(null, new KafkaCruiseControlConfig(postponeUrpProps));
    // Create prioritizeLargeMovementPlanner
    Properties prioritizeLargeMovementProps = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    prioritizeLargeMovementProps.setProperty(ExecutorConfig.DEFAULT_REPLICA_MOVEMENT_STRATEGIES_CONFIG, String.format("%s,%s", PrioritizeLargeReplicaMovementStrategy.class.getName(), BaseReplicaMovementStrategy.class.getName()));
    ExecutionTaskPlanner prioritizeLargeMovementPlanner = new ExecutionTaskPlanner(null, new KafkaCruiseControlConfig(prioritizeLargeMovementProps));
    // Create prioritizeSmallMovementPlanner
    Properties prioritizeSmallMovementProps = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    prioritizeSmallMovementProps.setProperty(ExecutorConfig.DEFAULT_REPLICA_MOVEMENT_STRATEGIES_CONFIG, String.format("%s,%s", PrioritizeSmallReplicaMovementStrategy.class.getName(), BaseReplicaMovementStrategy.class.getName()));
    ExecutionTaskPlanner prioritizeSmallMovementPlanner = new ExecutionTaskPlanner(null, new KafkaCruiseControlConfig(prioritizeSmallMovementProps));
    // Create smallUrpMovementPlanner
    Properties smallUrpMovementProps = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    smallUrpMovementProps.setProperty(ExecutorConfig.DEFAULT_REPLICA_MOVEMENT_STRATEGIES_CONFIG, String.format("%s,%s", PrioritizeSmallReplicaMovementStrategy.class.getName(), PostponeUrpReplicaMovementStrategy.class.getName()));
    ExecutionTaskPlanner smallUrpMovementPlanner = new ExecutionTaskPlanner(null, new KafkaCruiseControlConfig(smallUrpMovementProps));
    // Create contradictingMovementPlanner containing both small and large replica movements
    Properties contradictingMovementProps = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    contradictingMovementProps.setProperty(ExecutorConfig.DEFAULT_REPLICA_MOVEMENT_STRATEGIES_CONFIG, String.format("%s,%s,%s", PrioritizeSmallReplicaMovementStrategy.class.getName(), PostponeUrpReplicaMovementStrategy.class.getName(), PrioritizeLargeReplicaMovementStrategy.class.getName()));
    ExecutionTaskPlanner contradictingMovementPlanner = new ExecutionTaskPlanner(null, new KafkaCruiseControlConfig(contradictingMovementProps));
    // Create prioritizeMinIsrMovementPlanner
    Properties prioritizeMinIsrMovementProps = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    prioritizeMinIsrMovementProps.setProperty(ExecutorConfig.DEFAULT_REPLICA_MOVEMENT_STRATEGIES_CONFIG, PrioritizeMinIsrWithOfflineReplicasStrategy.class.getName());
    ExecutionTaskPlanner prioritizeMinIsrMovementPlanner = new ExecutionTaskPlanner(null, new KafkaCruiseControlConfig(prioritizeMinIsrMovementProps));
    Set<PartitionInfo> partitions = new HashSet<>();
    partitions.add(generatePartitionInfoWithUrpHavingOfflineReplica(_partitionMovement0, true));
    partitions.add(generatePartitionInfo(_partitionMovement1, false));
    partitions.add(generatePartitionInfoWithUrpHavingOfflineReplica(_partitionMovement2, true));
    partitions.add(generatePartitionInfo(_partitionMovement3, false));
    Cluster expectedCluster = new Cluster(null, _expectedNodes, partitions, Collections.emptySet(), Collections.emptySet());
    // This ensures that the _partitionMovement0 and _partitionMovement2 are AtMinISR, while the other partitions are not.
    Map<String, MinIsrWithTime> minIsrWithTimeByTopic = Collections.singletonMap(TOPIC2, new MinIsrWithTime((short) (_partitionMovement0.oldReplicas().size() - 1), 0));
    StrategyOptions strategyOptions = new StrategyOptions.Builder(expectedCluster).minIsrWithTimeByTopic(minIsrWithTimeByTopic).build();
    Map<Integer, Integer> readyBrokers = new HashMap<>();
    readyBrokers.put(0, 14);
    readyBrokers.put(1, 14);
    readyBrokers.put(2, 14);
    readyBrokers.put(3, 14);
    basePlanner.addExecutionProposals(proposals, strategyOptions, null);
    List<ExecutionTask> partitionMovementTasks = basePlanner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _partitionMovement0, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _partitionMovement2, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _partitionMovement1, partitionMovementTasks.get(2).proposal());
    postponeUrpPlanner.addExecutionProposals(proposals, strategyOptions, null);
    partitionMovementTasks = postponeUrpPlanner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _partitionMovement1, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _partitionMovement3, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _partitionMovement0, partitionMovementTasks.get(2).proposal());
    prioritizeLargeMovementPlanner.addExecutionProposals(proposals, strategyOptions, null);
    partitionMovementTasks = prioritizeLargeMovementPlanner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _partitionMovement1, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _partitionMovement3, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _partitionMovement2, partitionMovementTasks.get(2).proposal());
    prioritizeSmallMovementPlanner.addExecutionProposals(proposals, strategyOptions, null);
    partitionMovementTasks = prioritizeSmallMovementPlanner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _partitionMovement0, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _partitionMovement2, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _partitionMovement3, partitionMovementTasks.get(2).proposal());
    assertEquals("Fourth task", _partitionMovement1, partitionMovementTasks.get(3).proposal());
    smallUrpMovementPlanner.addExecutionProposals(proposals, strategyOptions, null);
    partitionMovementTasks = smallUrpMovementPlanner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _partitionMovement3, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _partitionMovement1, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _partitionMovement0, partitionMovementTasks.get(2).proposal());
    assertEquals("Fourth task", _partitionMovement2, partitionMovementTasks.get(3).proposal());
    contradictingMovementPlanner.addExecutionProposals(proposals, strategyOptions, null);
    partitionMovementTasks = contradictingMovementPlanner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _partitionMovement3, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _partitionMovement1, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _partitionMovement0, partitionMovementTasks.get(2).proposal());
    assertEquals("Fourth task", _partitionMovement2, partitionMovementTasks.get(3).proposal());
    prioritizeMinIsrMovementPlanner.addExecutionProposals(proposals, strategyOptions, null);
    partitionMovementTasks = prioritizeMinIsrMovementPlanner.getInterBrokerReplicaMovementTasks(readyBrokers, Collections.emptySet(), _defaultPartitionsMaxCap);
    assertEquals("First task", _partitionMovement0, partitionMovementTasks.get(0).proposal());
    assertEquals("Second task", _partitionMovement2, partitionMovementTasks.get(1).proposal());
    assertEquals("Third task", _partitionMovement1, partitionMovementTasks.get(2).proposal());
    assertEquals("Fourth task", _partitionMovement3, partitionMovementTasks.get(3).proposal());
}

Example 8 with StrategyOptions

use of com.linkedin.kafka.cruisecontrol.executor.strategy.StrategyOptions in project cruise-control by linkedin.

the class Executor method initProposalExecution.

private synchronized void initProposalExecution(Collection<ExecutionProposal> proposals, Collection<Integer> brokersToSkipConcurrencyCheck, Integer requestedInterBrokerPartitionMovementConcurrency, Integer requestedMaxInterBrokerPartitionMovements, Integer requestedIntraBrokerPartitionMovementConcurrency, Integer requestedLeadershipMovementConcurrency, Long requestedExecutionProgressCheckIntervalMs, ReplicaMovementStrategy replicaMovementStrategy, boolean isTriggeredByUserRequest, LoadMonitor loadMonitor) {
    _executorState = ExecutorState.initializeProposalExecution(_uuid, _reasonSupplier.get(), recentlyDemotedBrokers(), recentlyRemovedBrokers(), isTriggeredByUserRequest);
    _executionTaskManager.setExecutionModeForTaskTracker(_isKafkaAssignerMode);
    // Get a snapshot of (1) cluster and (2) minIsr with time by topic name.
    StrategyOptions strategyOptions = new StrategyOptions.Builder(_metadataClient.refreshMetadata().cluster()).minIsrWithTimeByTopic(_topicMinIsrCache.minIsrWithTimeByTopic()).build();
    _executionTaskManager.addExecutionProposals(proposals, brokersToSkipConcurrencyCheck, strategyOptions, replicaMovementStrategy);
    _concurrencyAdjuster.initAdjustment(loadMonitor, requestedInterBrokerPartitionMovementConcurrency, requestedLeadershipMovementConcurrency);
    setRequestedIntraBrokerPartitionMovementConcurrency(requestedIntraBrokerPartitionMovementConcurrency);
    setRequestedMaxInterBrokerPartitionMovements(requestedMaxInterBrokerPartitionMovements);
    setRequestedExecutionProgressCheckIntervalMs(requestedExecutionProgressCheckIntervalMs);
}