Examples with KafkaCruiseControlException com.linkedin.kafka.cruisecontrol.exception.KafkaCruiseControlException used on opensource projects

Example 1 with KafkaCruiseControlException

use of com.linkedin.kafka.cruisecontrol.exception.KafkaCruiseControlException in project cruise-control by linkedin.

the class KafkaCruiseControl method addBrokers.

/**
 * Add brokers
 * @param brokerIds the broker ids.
 * @param dryRun whether it is a dry run or not.
 * @param throttleAddedBrokers whether throttle the brokers that are being added.
 * @param goals the goals to be met when adding the brokers. When empty all goals will be used.
 * @param requirements The cluster model completeness requirements.
 * @param operationProgress The progress of the job to update.
 * @return The optimization result.
 * @throws KafkaCruiseControlException when any exception occurred during the broker addition.
 */
public GoalOptimizer.OptimizerResult addBrokers(Collection<Integer> brokerIds, boolean dryRun, boolean throttleAddedBrokers, List<String> goals, ModelCompletenessRequirements requirements, OperationProgress operationProgress) throws KafkaCruiseControlException {
    try (AutoCloseable ignored = _loadMonitor.acquireForModelGeneration(operationProgress)) {
        Map<Integer, Goal> goalsByPriority = goalsByPriority(goals);
        ModelCompletenessRequirements modelCompletenessRequirements = modelCompletenessRequirements(goalsByPriority.values()).weaker(requirements);
        ClusterModel clusterModel = _loadMonitor.clusterModel(_time.milliseconds(), modelCompletenessRequirements, operationProgress);
        brokerIds.forEach(id -> clusterModel.setBrokerState(id, Broker.State.NEW));
        GoalOptimizer.OptimizerResult result = getOptimizationProposals(clusterModel, goalsByPriority, operationProgress);
        if (!dryRun) {
            executeProposals(result.goalProposals(), throttleAddedBrokers ? Collections.emptyList() : brokerIds);
        }
        return result;
    } catch (KafkaCruiseControlException kcce) {
        throw kcce;
    } catch (Exception e) {
        throw new KafkaCruiseControlException(e);
    }
}

Example 2 with KafkaCruiseControlException

use of com.linkedin.kafka.cruisecontrol.exception.KafkaCruiseControlException in project cruise-control by linkedin.

the class KafkaCruiseControl method decommissionBrokers.

/**
 * Decommission a broker.
 *
 * @param brokerIds The brokers to decommission.
 * @param dryRun throw
 * @param throttleDecommissionedBroker whether throttle the brokers that are being decommissioned.
 * @param goals the goals to be met when decommissioning the brokers. When empty all goals will be used.
 * @param requirements The cluster model completeness requirements.
 * @param operationProgress the progress to report.
 * @return the optimization result.
 *
 * @throws KafkaCruiseControlException when any exception occurred during the decommission process.
 */
public GoalOptimizer.OptimizerResult decommissionBrokers(Collection<Integer> brokerIds, boolean dryRun, boolean throttleDecommissionedBroker, List<String> goals, ModelCompletenessRequirements requirements, OperationProgress operationProgress) throws KafkaCruiseControlException {
    Map<Integer, Goal> goalsByPriority = goalsByPriority(goals);
    ModelCompletenessRequirements modelCompletenessRequirements = modelCompletenessRequirements(goalsByPriority.values()).weaker(requirements);
    try (AutoCloseable ignored = _loadMonitor.acquireForModelGeneration(operationProgress)) {
        ClusterModel clusterModel = _loadMonitor.clusterModel(_time.milliseconds(), modelCompletenessRequirements, operationProgress);
        brokerIds.forEach(id -> clusterModel.setBrokerState(id, Broker.State.DEAD));
        GoalOptimizer.OptimizerResult result = getOptimizationProposals(clusterModel, goalsByPriority, operationProgress);
        if (!dryRun) {
            executeProposals(result.goalProposals(), throttleDecommissionedBroker ? Collections.emptyList() : brokerIds);
        }
        return result;
    } catch (KafkaCruiseControlException kcce) {
        throw kcce;
    } catch (Exception e) {
        throw new KafkaCruiseControlException(e);
    }
}

Example 3 with KafkaCruiseControlException

use of com.linkedin.kafka.cruisecontrol.exception.KafkaCruiseControlException in project cruise-control by linkedin.

the class KafkaCruiseControl method getOptimizationProposals.

/**
 * Optimize a cluster workload model.
 * @param goals a list of goals to optimize. When empty all goals will be used.
 * @param requirements the model completeness requirements to enforce when generating the propsoals.
 * @param operationProgress the progress of the job to report.
 * @return The optimization result.
 * @throws KafkaCruiseControlException
 */
public GoalOptimizer.OptimizerResult getOptimizationProposals(List<String> goals, ModelCompletenessRequirements requirements, OperationProgress operationProgress) throws KafkaCruiseControlException {
    GoalOptimizer.OptimizerResult result;
    Map<Integer, Goal> goalsByPriority = goalsByPriority(goals);
    ModelCompletenessRequirements modelCompletenessRequirements = modelCompletenessRequirements(goalsByPriority.values()).weaker(requirements);
    // There are a few cases that we cannot use the cached best proposals:
    // 1. When users specified goals.
    // 2. When provided requirements contains a weaker requirement than what is used by the cached proposal.
    ModelCompletenessRequirements requirementsForCache = _goalOptimizer.modelCompletenessRequirementsForPrecomputing();
    boolean hasWeakerRequirement = requirementsForCache.minMonitoredPartitionsPercentage() > modelCompletenessRequirements.minMonitoredPartitionsPercentage() || requirementsForCache.minRequiredNumWindows() > modelCompletenessRequirements.minRequiredNumWindows() || (requirementsForCache.includeAllTopics() && !modelCompletenessRequirements.includeAllTopics());
    if ((goals != null && !goals.isEmpty()) || hasWeakerRequirement) {
        try (AutoCloseable ignored = _loadMonitor.acquireForModelGeneration(operationProgress)) {
            // The cached proposals are computed with ignoreMinMonitoredPartitions = true. So if user provided a different
            // setting, we need to generate a new model.
            ClusterModel clusterModel = _loadMonitor.clusterModel(-1, _time.milliseconds(), modelCompletenessRequirements, operationProgress);
            result = getOptimizationProposals(clusterModel, goalsByPriority, operationProgress);
        } catch (KafkaCruiseControlException kcce) {
            throw kcce;
        } catch (Exception e) {
            throw new KafkaCruiseControlException(e);
        }
    } else {
        result = getOptimizationProposals(operationProgress);
    }
    return result;
}

Example 4 with KafkaCruiseControlException

use of com.linkedin.kafka.cruisecontrol.exception.KafkaCruiseControlException in project cruise-control by linkedin.

the class GoalViolationDetector method run.

@Override
public void run() {
    long now = _time.milliseconds();
    if (_loadMonitor.clusterModelGeneration().equals(_lastCheckedModelGeneration)) {
        LOG.debug("Skipping goal violation detection because the model generation hasn't changed. Current model generation {}", _loadMonitor.clusterModelGeneration());
        return;
    }
    AutoCloseable clusterModelSemaphore = null;
    try {
        LoadMonitorTaskRunner.LoadMonitorTaskRunnerState loadMonitorTaskRunnerState = _loadMonitor.taskRunnerState();
        if (!ViolationUtils.isLoadMonitorReady(loadMonitorTaskRunnerState)) {
            LOG.info("Skipping goal violation detection because load monitor is in {} state.", loadMonitorTaskRunnerState);
            return;
        }
        GoalViolations goalViolations = new GoalViolations();
        boolean newModelNeeded = true;
        ClusterModel clusterModel = null;
        for (Map.Entry<Integer, Goal> entry : _goals.entrySet()) {
            Goal goal = entry.getValue();
            if (_loadMonitor.meetCompletenessRequirements(goal.clusterModelCompletenessRequirements())) {
                LOG.debug("Detecting if {} is violated.", entry.getValue().name());
                // Because the model generation could be slow, We only get new cluster model if needed.
                if (newModelNeeded) {
                    if (clusterModelSemaphore != null) {
                        clusterModelSemaphore.close();
                    }
                    clusterModelSemaphore = _loadMonitor.acquireForModelGeneration(new OperationProgress());
                    // Make cluster model null before generating a new cluster model so the current one can be GCed.
                    clusterModel = null;
                    clusterModel = _loadMonitor.clusterModel(now, goal.clusterModelCompletenessRequirements(), new OperationProgress());
                }
                int priority = entry.getKey();
                newModelNeeded = optimizeForGoal(clusterModel, priority, goal, goalViolations);
            } else {
                LOG.debug("Skipping goal violation detection for {} because load completeness requirement is not met.", goal);
            }
        }
        if (clusterModel != null) {
            _lastCheckedModelGeneration = clusterModel.generation();
        }
        if (!goalViolations.violations().isEmpty()) {
            _anomalies.add(goalViolations);
        }
    } catch (NotEnoughValidWindowsException nevwe) {
        LOG.debug("Skipping goal violation detection because there are not enough valid windows.");
    } catch (KafkaCruiseControlException kcce) {
        LOG.warn("Goal violation detector received exception", kcce);
    } catch (Exception e) {
        LOG.error("Unexpected exception", e);
    } finally {
        if (clusterModelSemaphore != null) {
            try {
                clusterModelSemaphore.close();
            } catch (Exception e) {
                LOG.error("Received exception when closing auto closable semaphore", e);
            }
        }
        LOG.debug("Goal violation detection finished.");
    }
}