Examples with ReplicaPlacementInfo com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo used on opensource projects

Example 31 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class AbstractReplicaMovementStrategy method applyStrategy.

@Override
public Map<Integer, SortedSet<ExecutionTask>> applyStrategy(Set<ExecutionTask> replicaMovementTasks, StrategyOptions strategyOptions) {
    Map<Integer, SortedSet<ExecutionTask>> tasksByBrokerId = new HashMap<>();
    for (ExecutionTask task : replicaMovementTasks) {
        ExecutionProposal proposal = task.proposal();
        // Add the task to source broker's execution plan
        SortedSet<ExecutionTask> sourceBrokerTaskSet = tasksByBrokerId.computeIfAbsent(proposal.oldLeader().brokerId(), k -> new TreeSet<>(taskComparator(strategyOptions)));
        if (!sourceBrokerTaskSet.add(task)) {
            throw new IllegalStateException("Replica movement strategy " + this.getClass().getSimpleName() + " failed to determine order of tasks.");
        }
        // Add the task to destination brokers' execution plan
        for (ReplicaPlacementInfo destinationBroker : proposal.replicasToAdd()) {
            SortedSet<ExecutionTask> destinationBrokerTaskSet = tasksByBrokerId.computeIfAbsent(destinationBroker.brokerId(), k -> new TreeSet<>(taskComparator(strategyOptions)));
            if (!destinationBrokerTaskSet.add(task)) {
                throw new IllegalStateException("Replica movement strategy " + this.getClass().getSimpleName() + " failed to determine order of tasks.");
            }
        }
    }
    return tasksByBrokerId;
}

Example 32 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class GoalViolationDetector method optimizeForGoal.

protected boolean optimizeForGoal(ClusterModel clusterModel, Goal goal, GoalViolations goalViolations, Set<Integer> excludedBrokersForLeadership, Set<Integer> excludedBrokersForReplicaMove, boolean checkPartitionsWithRFGreaterThanNumRacks) throws KafkaCruiseControlException {
    if (clusterModel.topics().isEmpty()) {
        LOG.info("Skipping goal violation detection because the cluster model does not have any topic.");
        return false;
    }
    Map<TopicPartition, List<ReplicaPlacementInfo>> initReplicaDistribution = clusterModel.getReplicaDistribution();
    Map<TopicPartition, ReplicaPlacementInfo> initLeaderDistribution = clusterModel.getLeaderDistribution();
    try {
        OptimizationOptions options = _optimizationOptionsGenerator.optimizationOptionsForGoalViolationDetection(clusterModel, excludedTopics(clusterModel), excludedBrokersForLeadership, excludedBrokersForReplicaMove);
        if (checkPartitionsWithRFGreaterThanNumRacks) {
            _hasPartitionsWithRFGreaterThanNumRacks = clusterModel.maxReplicationFactor() > clusterModel.numAliveRacksAllowedReplicaMoves(options);
        }
        goal.optimize(clusterModel, Collections.emptySet(), options);
    } catch (OptimizationFailureException ofe) {
        // An OptimizationFailureException indicates (1) a hard goal violation that cannot be fixed typically due to
        // lack of physical hardware (e.g. insufficient number of racks to satisfy rack awareness, insufficient number
        // of brokers to satisfy Replica Capacity Goal, or insufficient number of resources to satisfy resource
        // capacity goals), or (2) a failure to move offline replicas away from dead brokers/disks.
        goalViolations.addViolation(goal.name(), false);
        return true;
    }
    boolean hasDiff = AnalyzerUtils.hasDiff(initReplicaDistribution, initLeaderDistribution, clusterModel);
    LOG.trace("{} generated {} proposals", goal.name(), hasDiff ? "some" : "no");
    if (hasDiff) {
        // A goal violation that can be optimized by applying the generated proposals.
        goalViolations.addViolation(goal.name(), true);
        return true;
    } else {
        // The goal is already satisfied.
        return false;
    }
}

Example 33 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class LowResourceUtilizationTest method test.

@Test
public void test() throws OptimizationFailureException {
    ClusterModel clusterModel = createSmallClusterModel();
    Map<TopicPartition, List<ReplicaPlacementInfo>> initReplicaDistribution = clusterModel.getReplicaDistribution();
    Map<TopicPartition, ReplicaPlacementInfo> initLeaderDistribution = clusterModel.getLeaderDistribution();
    // Before the optimization, goals are expected to be undecided wrt their provision status.
    assertEquals(ProvisionStatus.UNDECIDED, _resourceDistributionGoal.provisionResponse().status());
    assertTrue("Failed to optimize " + _resourceDistributionGoal.name(), _resourceDistributionGoal.optimize(clusterModel, Collections.emptySet(), new OptimizationOptions(Collections.emptySet(), Collections.emptySet(), Collections.emptySet())));
    // Since all optimizations succeed, it is guaranteed that all brokers have resource utilization under low utilization threshold.
    assertEquals(ProvisionStatus.OVER_PROVISIONED, _resourceDistributionGoal.provisionResponse().status());
    boolean hasDiff = AnalyzerUtils.hasDiff(initReplicaDistribution, initLeaderDistribution, clusterModel);
    if (_expectRebalance) {
        assertTrue(hasDiff);
    } else {
        assertFalse(hasDiff);
    }
}

Example 34 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ReplicationFactorChangeTest method test.

@Test
public void test() throws Exception {
    prepareContext();
    Map<TopicPartition, List<ReplicaPlacementInfo>> initReplicaDistribution = _clusterModel.getReplicaDistribution();
    Map<TopicPartition, ReplicaPlacementInfo> initLeaderDistribution = _clusterModel.getLeaderDistribution();
    _clusterModel.createOrDeleteReplicas(Collections.singletonMap(_replicationFactor, _topics), _brokersByRack, _rackByBroker, _cluster);
    // Before the optimization, goals are expected to be undecided wrt their provision status.
    assertEquals(ProvisionStatus.UNDECIDED, _goal.provisionResponse().status());
    if (_exceptionClass == null) {
        if (_expectedToOptimize) {
            assertTrue("Replication factor change test with goal " + _goal.name() + " failed.", _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
        } else {
            assertFalse("Replication factor change test with goal " + _goal.name() + " should not succeed.", _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
        }
        // The cluster cannot be underprovisioned, because _exceptionClass was null.
        assertNotEquals(ProvisionStatus.UNDER_PROVISIONED, _goal.provisionResponse().status());
        Set<ExecutionProposal> goalProposals = AnalyzerUtils.getDiff(initReplicaDistribution, initLeaderDistribution, _clusterModel, true);
        for (ExecutionProposal proposal : goalProposals) {
            // Replication factor change should only be applied to specified topics.
            if (!_topics.contains(proposal.topic())) {
                fail("Replication factor change should not apply to topic %s." + proposal.topic());
            }
            if (proposal.newReplicas().size() != _replicationFactor) {
                fail(String.format("Topic partition %s's replication factor is not changed to %d.", proposal.topicPartition(), _replicationFactor));
            }
            // Increase replication factor should not touch the existing replicas.
            if (_replicationFactor >= proposal.oldReplicas().size() && !proposal.replicasToRemove().isEmpty()) {
                fail(String.format("Increasing topic partition %s's replication factor to %d should not move existing replicas.", proposal.topicPartition(), _replicationFactor));
            }
        }
        // Ensure all the specified topic has target replication factor.
        for (String topic : _topics) {
            for (PartitionInfo partitioninfo : _cluster.partitionsForTopic(topic)) {
                TopicPartition tp = new TopicPartition(topic, partitioninfo.partition());
                if (_clusterModel.partition(tp).replicas().size() != _replicationFactor) {
                    fail(String.format("Topic partition %s's replication factor is not changed to %d", tp, _replicationFactor));
                }
            }
        }
    } else {
        assertThrows(_exceptionClass, () -> _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
    }
}

Example 35 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExcludedBrokersForReplicaMoveTest method test.

@Test
public void test() throws Exception {
    // Before the optimization, goals are expected to be undecided wrt their provision status.
    assertEquals(ProvisionStatus.UNDECIDED, _goal.provisionResponse().status());
    if (_exceptionClass == null) {
        Map<TopicPartition, List<ReplicaPlacementInfo>> initReplicaDistribution = _clusterModel.getReplicaDistribution();
        Map<TopicPartition, ReplicaPlacementInfo> initLeaderDistribution = _clusterModel.getLeaderDistribution();
        Set<Integer> excludedBrokersForReplicaMove = _optimizationOptions.excludedBrokersForReplicaMove();
        if (_expectedToOptimize) {
            assertTrue("Failed to optimize " + _goal.name() + " with excluded brokers for replica move.", _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
        } else {
            assertFalse("Optimized " + _goal.name() + " with excluded brokers for replicaMove " + excludedBrokersForReplicaMove, _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
        }
        // The cluster cannot be underprovisioned, because _exceptionClass was null.
        assertNotEquals(ProvisionStatus.UNDER_PROVISIONED, _goal.provisionResponse().status());
        // Generated proposals cannot move replicas to the excluded brokers for replica move.
        if (!excludedBrokersForReplicaMove.isEmpty()) {
            Set<ExecutionProposal> goalProposals = AnalyzerUtils.getDiff(initReplicaDistribution, initLeaderDistribution, _clusterModel);
            assertEquals(_expectedToGenerateProposals, !goalProposals.isEmpty());
            for (ExecutionProposal proposal : goalProposals) {
                if (proposal.hasReplicaAction() && violatesExcludedBrokersForReplicaMove(excludedBrokersForReplicaMove, proposal)) {
                    fail(String.format("A replica move in %s to an excluded broker for replica move %s.", proposal, excludedBrokersForReplicaMove));
                }
            }
        }
    } else {
        _expected.expect(_exceptionClass);
        assertTrue("Failed to optimize with excluded brokers for replica move.", _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
        assertEquals(ProvisionStatus.UNDER_PROVISIONED, _goal.provisionResponse().status());
    }
}