Examples with ReplicaPlacementInfo com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo used on opensource projects

Example 16 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExcludedBrokersForLeadershipTest method test.

@Test
public void test() throws Exception {
    // Before the optimization, goals are expected to be undecided wrt their provision status.
    assertEquals(ProvisionStatus.UNDECIDED, _goal.provisionResponse().status());
    if (_exceptionClass == null) {
        Map<TopicPartition, List<ReplicaPlacementInfo>> initReplicaDistribution = _clusterModel.getReplicaDistribution();
        Map<TopicPartition, ReplicaPlacementInfo> initLeaderDistribution = _clusterModel.getLeaderDistribution();
        Set<Integer> excludedBrokersForLeadership = _optimizationOptions.excludedBrokersForLeadership();
        if (_expectedToOptimize) {
            assertTrue("Failed to optimize " + _goal.name() + " with excluded brokers for leadership.", _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
        } else {
            assertFalse("Optimized " + _goal.name() + " with excluded brokers for leadership " + excludedBrokersForLeadership, _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
        }
        // The cluster cannot be underprovisioned, because _exceptionClass was null.
        assertNotEquals(ProvisionStatus.UNDER_PROVISIONED, _goal.provisionResponse().status());
        // Generated proposals cannot move leadership to the excluded brokers for leadership.
        if (!excludedBrokersForLeadership.isEmpty()) {
            Set<ExecutionProposal> goalProposals = AnalyzerUtils.getDiff(initReplicaDistribution, initLeaderDistribution, _clusterModel);
            for (ExecutionProposal proposal : goalProposals) {
                if (proposal.hasLeaderAction() && excludedBrokersForLeadership.contains(proposal.newLeader().brokerId()) && _clusterModel.broker(proposal.oldLeader().brokerId()).isAlive()) {
                    fail(String.format("Leadership move in %s from an online replica to an excluded broker for leadership %s.", proposal, excludedBrokersForLeadership));
                }
            }
        }
    } else {
        _expected.expect(_exceptionClass);
        assertTrue("Failed to optimize with excluded brokers for leadership.", _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
        assertEquals(ProvisionStatus.UNDER_PROVISIONED, _goal.provisionResponse().status());
    }
}

Example 17 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class ExcludedTopicsTest method test.

@Test
public void test() throws Exception {
    // Before the optimization, goals are expected to be undecided wrt their provision status.
    assertEquals(ProvisionStatus.UNDECIDED, _goal.provisionResponse().status());
    if (_exceptionClass == null) {
        Map<TopicPartition, List<ReplicaPlacementInfo>> initReplicaDistribution = _clusterModel.getReplicaDistribution();
        Map<TopicPartition, ReplicaPlacementInfo> initLeaderDistribution = _clusterModel.getLeaderDistribution();
        Set<String> excludedTopics = _optimizationOptions.excludedTopics();
        if (_expectedToOptimize) {
            assertTrue("Excluded Topics Test failed to optimize " + _goal.name() + " with excluded topics " + excludedTopics, _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
        } else {
            assertFalse("Excluded Topics Test optimized " + _goal.name() + " with excluded topics " + excludedTopics, _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
        }
        // The cluster cannot be underprovisioned, because _exceptionClass was null.
        assertNotEquals(ProvisionStatus.UNDER_PROVISIONED, _goal.provisionResponse().status());
        // Generated proposals cannot have the excluded topic.
        if (!excludedTopics.isEmpty()) {
            Set<ExecutionProposal> goalProposals = AnalyzerUtils.getDiff(initReplicaDistribution, initLeaderDistribution, _clusterModel);
            assertEquals(_expectedToGenerateProposals, !goalProposals.isEmpty());
            for (ExecutionProposal proposal : goalProposals) {
                if (excludedTopics.contains(proposal.topic())) {
                    for (ReplicaPlacementInfo r : proposal.replicasToRemove()) {
                        if (_clusterModel.broker(r.brokerId()).isAlive()) {
                            fail(String.format("Proposal %s contains excluded topic %s, but the broker %d is still alive.", proposal, proposal.topic(), r.brokerId()));
                        }
                    }
                }
            }
        }
    } else {
        _expected.expect(_exceptionClass);
        assertTrue("Excluded Topics Test failed to optimize with excluded topics.", _goal.optimize(_clusterModel, Collections.emptySet(), _optimizationOptions));
        assertEquals(ProvisionStatus.UNDER_PROVISIONED, _goal.provisionResponse().status());
    }
}

Example 18 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class AnalyzerUtils method hasDiff.

/**
 * Get whether there is any diff represented by a set of balancing proposals to move from the initial to final distribution.
 *
 * @param initialReplicaDistribution Initial distribution of replicas over the cluster.
 * @param initialLeaderDistribution Initial distribution of the leaders.
 * @param optimizedClusterModel The optimized cluster model.
 * @return The diff represented by the set of balancing proposals to move from initial to final distribution.
 */
public static boolean hasDiff(Map<TopicPartition, List<ReplicaPlacementInfo>> initialReplicaDistribution, Map<TopicPartition, ReplicaPlacementInfo> initialLeaderDistribution, ClusterModel optimizedClusterModel) {
    Map<TopicPartition, List<ReplicaPlacementInfo>> finalReplicaDistribution = optimizedClusterModel.getReplicaDistribution();
    sanityCheckReplicaDistribution(initialReplicaDistribution, finalReplicaDistribution, false);
    boolean hasDiff = false;
    for (Map.Entry<TopicPartition, List<ReplicaPlacementInfo>> entry : initialReplicaDistribution.entrySet()) {
        TopicPartition tp = entry.getKey();
        List<ReplicaPlacementInfo> initialReplicas = entry.getValue();
        List<ReplicaPlacementInfo> finalReplicas = finalReplicaDistribution.get(tp);
        if (!finalReplicas.equals(initialReplicas)) {
            hasDiff = true;
            break;
        } else {
            Replica finalLeader = optimizedClusterModel.partition(tp).leader();
            ReplicaPlacementInfo finalLeaderPlacementInfo = new ReplicaPlacementInfo(finalLeader.broker().id(), finalLeader.disk() == null ? null : finalLeader.disk().logDir());
            if (!initialLeaderDistribution.get(tp).equals(finalLeaderPlacementInfo)) {
                hasDiff = true;
                break;
            }
        // The partition has no change.
        }
    }
    return hasDiff;
}

Example 19 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class AnalyzerUtils method getDiff.

/**
 * Get the diff represented by the set of balancing proposals to move from initial to final distribution.
 *
 * @param initialReplicaDistribution Initial distribution of replicas over the cluster.
 * @param initialLeaderDistribution Initial distribution of the leaders.
 * @param optimizedClusterModel The optimized cluster model.
 * @param skipReplicationFactorChangeCheck Whether skip sanity check of topic partition's replication factor change before
 *                                         and after optimization.
 * @return The diff represented by the set of balancing proposals to move from initial to final distribution.
 */
public static Set<ExecutionProposal> getDiff(Map<TopicPartition, List<ReplicaPlacementInfo>> initialReplicaDistribution, Map<TopicPartition, ReplicaPlacementInfo> initialLeaderDistribution, ClusterModel optimizedClusterModel, boolean skipReplicationFactorChangeCheck) {
    Map<TopicPartition, List<ReplicaPlacementInfo>> finalReplicaDistribution = optimizedClusterModel.getReplicaDistribution();
    sanityCheckReplicaDistribution(initialReplicaDistribution, finalReplicaDistribution, skipReplicationFactorChangeCheck);
    // Generate a set of execution proposals to represent the diff between initial and final distribution.
    Set<ExecutionProposal> diff = new HashSet<>();
    for (Map.Entry<TopicPartition, List<ReplicaPlacementInfo>> entry : initialReplicaDistribution.entrySet()) {
        TopicPartition tp = entry.getKey();
        List<ReplicaPlacementInfo> initialReplicas = entry.getValue();
        List<ReplicaPlacementInfo> finalReplicas = finalReplicaDistribution.get(tp);
        Replica finalLeader = optimizedClusterModel.partition(tp).leader();
        ReplicaPlacementInfo finalLeaderPlacementInfo = new ReplicaPlacementInfo(finalLeader.broker().id(), finalLeader.disk() == null ? null : finalLeader.disk().logDir());
        // The partition has no change.
        if (finalReplicas.equals(initialReplicas) && initialLeaderDistribution.get(tp).equals(finalLeaderPlacementInfo)) {
            continue;
        }
        // We need to adjust the final broker list order to ensure the final leader is the first replica.
        if (finalLeaderPlacementInfo != finalReplicas.get(0)) {
            int leaderPos = finalReplicas.indexOf(finalLeaderPlacementInfo);
            finalReplicas.set(leaderPos, finalReplicas.get(0));
            finalReplicas.set(0, finalLeaderPlacementInfo);
        }
        double partitionSize = optimizedClusterModel.partition(tp).leader().load().expectedUtilizationFor(Resource.DISK);
        diff.add(new ExecutionProposal(tp, (int) partitionSize, initialLeaderDistribution.get(tp), initialReplicas, finalReplicas));
    }
    return diff;
}

Example 20 with ReplicaPlacementInfo

use of com.linkedin.kafka.cruisecontrol.model.ReplicaPlacementInfo in project cruise-control by linkedin.

the class GoalOptimizer method optimizations.

/**
 * Depending the existence of dead/broken/decommissioned brokers in the given cluster:
 * (1) Re-balance: Generates proposals to update the state of the cluster to achieve a final balanced state.
 * (2) Self-healing: Generates proposals to move replicas away from decommissioned brokers and broken disks.
 * Returns a map from goal names to stats. Initial stats are returned under goal name "init".
 *
 * Assumptions:
 * <ul>
 *   <li>The cluster model cannot be null.</li>
 *   <li>At least one goal has been provided in goalsByPriority.</li>
 *   <li>There is at least one alive broker in the cluster.</li>
 * </ul>
 *
 * @param clusterModel The state of the cluster over which the balancing proposal will be applied. Function execution
 *                     updates the cluster state with balancing proposals. If the cluster model is specified, the
 *                     cached proposal will be ignored.
 * @param goalsByPriority the goals ordered by priority.
 * @param operationProgress to report the job progress.
 * @param initReplicaDistributionForProposalGeneration The initial replica distribution of the cluster. This is only
 *                                                     needed if the passed in clusterModel is not the original cluster
 *                                                     model so that initial replica distribution can not be deducted
 *                                                     from that cluster model, otherwise it is null. One case explicitly
 *                                                     specifying initial replica distribution needed is to increase/decrease
 *                                                     specific topic partition's replication factor, in this case some
 *                                                     replicas are tentatively deleted/added in cluster model before
 *                                                     passing it in to generate proposals.
 * @param optimizationOptions Optimization options.
 * @return Results of optimization containing the proposals and stats.
 */
public OptimizerResult optimizations(ClusterModel clusterModel, List<Goal> goalsByPriority, OperationProgress operationProgress, Map<TopicPartition, List<ReplicaPlacementInfo>> initReplicaDistributionForProposalGeneration, OptimizationOptions optimizationOptions) throws KafkaCruiseControlException {
    LOG.trace("Cluster before optimization is {}", clusterModel);
    BrokerStats brokerStatsBeforeOptimization = clusterModel.brokerStats(null);
    Map<TopicPartition, List<ReplicaPlacementInfo>> initReplicaDistribution = clusterModel.getReplicaDistribution();
    Map<TopicPartition, ReplicaPlacementInfo> initLeaderDistribution = clusterModel.getLeaderDistribution();
    boolean isSelfHealing = !clusterModel.selfHealingEligibleReplicas().isEmpty();
    // Set of balancing proposals that will be applied to the given cluster state to satisfy goals (leadership
    // transfer AFTER partition transfer.)
    Set<Goal> optimizedGoals = new HashSet<>();
    Set<String> violatedGoalNamesBeforeOptimization = new HashSet<>();
    Set<String> violatedGoalNamesAfterOptimization = new HashSet<>();
    LinkedHashMap<Goal, ClusterModelStats> statsByGoalPriority = new LinkedHashMap<>(goalsByPriority.size());
    Map<TopicPartition, List<ReplicaPlacementInfo>> preOptimizedReplicaDistribution = null;
    Map<TopicPartition, ReplicaPlacementInfo> preOptimizedLeaderDistribution = null;
    ProvisionResponse provisionResponse = new ProvisionResponse(ProvisionStatus.UNDECIDED);
    Map<String, Duration> optimizationDurationByGoal = new HashMap<>();
    for (Goal goal : goalsByPriority) {
        preOptimizedReplicaDistribution = preOptimizedReplicaDistribution == null ? initReplicaDistribution : clusterModel.getReplicaDistribution();
        preOptimizedLeaderDistribution = preOptimizedLeaderDistribution == null ? initLeaderDistribution : clusterModel.getLeaderDistribution();
        OptimizationForGoal step = new OptimizationForGoal(goal.name());
        operationProgress.addStep(step);
        LOG.debug("Optimizing goal {}", goal.name());
        long startTimeMs = _time.milliseconds();
        boolean succeeded = goal.optimize(clusterModel, optimizedGoals, optimizationOptions);
        optimizedGoals.add(goal);
        statsByGoalPriority.put(goal, clusterModel.getClusterStats(_balancingConstraint, optimizationOptions));
        optimizationDurationByGoal.put(goal.name(), Duration.ofMillis(_time.milliseconds() - startTimeMs));
        boolean hasDiff = AnalyzerUtils.hasDiff(preOptimizedReplicaDistribution, preOptimizedLeaderDistribution, clusterModel);
        if (hasDiff || !succeeded) {
            violatedGoalNamesBeforeOptimization.add(goal.name());
        }
        if (!succeeded) {
            violatedGoalNamesAfterOptimization.add(goal.name());
        }
        LOG.debug("[{}/{}] Generated {} proposals for {}{}.", optimizedGoals.size(), _goalsByPriority.size(), hasDiff ? "some" : "no", isSelfHealing ? "self-healing " : "", goal.name());
        step.done();
        if (LOG.isDebugEnabled()) {
            LOG.debug("Broker level stats after optimization: {}", clusterModel.brokerStats(null));
        }
        provisionResponse.aggregate(goal.provisionResponse());
    }
    // Broker level stats in the final cluster state.
    if (LOG.isTraceEnabled()) {
        LOG.trace("Broker level stats after optimization: {}%n", clusterModel.brokerStats(null));
    }
    // Skip replication factor change check here since in above iteration we already check for each goal it does not change
    // any partition's replication factor.
    Set<ExecutionProposal> proposals = AnalyzerUtils.getDiff(initReplicaDistributionForProposalGeneration != null ? initReplicaDistributionForProposalGeneration : initReplicaDistribution, initLeaderDistribution, clusterModel, true);
    return new OptimizerResult(statsByGoalPriority, violatedGoalNamesBeforeOptimization, violatedGoalNamesAfterOptimization, proposals, brokerStatsBeforeOptimization, clusterModel, optimizationOptions, balancednessCostByGoal(goalsByPriority, _priorityWeight, _strictnessWeight), optimizationDurationByGoal, provisionResponse);
}