Examples with ProvisionRecommendation com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation used on opensource projects

Example 21 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class PartitionProvisionerTest method assertProvisionPartitionIncreaseConstructsCorrectResponse.

private void assertProvisionPartitionIncreaseConstructsCorrectResponse(KafkaCruiseControlUtils.CompletionType partitionIncreaseCompletion, ProvisionerState.State expectedState, String expectedSummary, boolean hasBrokerRecommendation) throws ExecutionException, InterruptedException, TimeoutException {
    int recommendedPartitionCount = expectedSummary.contains("Ignored") ? MOCK_IGNORED_PARTITION_COUNT : MOCK_PARTITION_COUNT;
    ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numPartitions(recommendedPartitionCount).topicPattern(MOCK_TOPIC_PATTERN).build();
    Map<String, ProvisionRecommendation> provisionRecommendation;
    if (hasBrokerRecommendation) {
        provisionRecommendation = Map.of(RECOMMENDER_UP, recommendation, RECOMMENDER_TO_EXECUTE, BROKER_REC_TO_EXECUTE);
    } else {
        provisionRecommendation = Collections.singletonMap(RECOMMENDER_UP, recommendation);
    }
    Map<String, KafkaFuture<TopicDescription>> describeTopicsValues = Collections.singletonMap(MOCK_TOPIC, MOCK_TOPIC_DESCRIPTION_FUTURE);
    EasyMock.expect(MOCK_ADMIN_CLIENT.describeTopics(Collections.singletonList(MOCK_TOPIC))).andReturn(MOCK_DESCRIBE_TOPICS_RESULT);
    EasyMock.expect(MOCK_DESCRIBE_TOPICS_RESULT.values()).andReturn(describeTopicsValues);
    if (partitionIncreaseCompletion == COMPLETED) {
        EasyMock.expect(MOCK_TOPIC_DESCRIPTION_FUTURE.get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)).andReturn(MOCK_TOPIC_DESCRIPTION);
        // Create partitions: for this test, we ignore the fact that the mock cluster has one node -- i.e. in reality a request to increase
        // partition count to two would fail in a cluster with one node.
        EasyMock.expect(MOCK_ADMIN_CLIENT.createPartitions(Collections.singletonMap(MOCK_TOPIC, EasyMock.anyObject()))).andReturn(MOCK_CREATE_PARTITIONS_RESULT);
        Map<String, KafkaFuture<Void>> createPartitionsResultValues = Collections.singletonMap(MOCK_TOPIC, MOCK_CREATE_PARTITIONS_FUTURE);
        EasyMock.expect(MOCK_CREATE_PARTITIONS_RESULT.values()).andReturn(createPartitionsResultValues);
        EasyMock.expect(MOCK_CREATE_PARTITIONS_FUTURE.get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)).andReturn(null);
    } else if (partitionIncreaseCompletion == KafkaCruiseControlUtils.CompletionType.COMPLETED_WITH_ERROR) {
        EasyMock.expect(MOCK_TOPIC_DESCRIPTION_FUTURE.get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)).andThrow(new ExecutionException(new InvalidTopicException()));
    } else {
        EasyMock.expect(MOCK_TOPIC_DESCRIPTION_FUTURE.get(CLIENT_REQUEST_TIMEOUT_MS, TimeUnit.MILLISECONDS)).andReturn(MOCK_TOPIC_DESCRIPTION);
    }
    EasyMock.replay(MOCK_ADMIN_CLIENT, MOCK_DESCRIBE_TOPICS_RESULT, MOCK_TOPIC_DESCRIPTION_FUTURE, MOCK_CREATE_PARTITIONS_FUTURE, MOCK_CREATE_PARTITIONS_RESULT);
    ProvisionerState results = _provisioner.rightsize(provisionRecommendation, RIGHTSIZE_OPTIONS);
    assertEquals(expectedState, results.state());
    assertEquals(expectedSummary, results.summary());
}

Example 22 with ProvisionRecommendation

use of com.linkedin.kafka.cruisecontrol.analyzer.ProvisionRecommendation in project cruise-control by linkedin.

the class ResourceDistributionGoal method initGoalState.

/**
 * (1) Initialize the current resource to be balanced or self healed.
 * (2) Set the flag which indicates whether the self healing failed to relocate all replicas away from dead brokers
 * in its initial attempt. Since self healing has not been executed yet, this flag is false.
 *
 * @param clusterModel The state of the cluster.
 * @param optimizationOptions Options to take into account during optimization.
 */
@Override
protected void initGoalState(ClusterModel clusterModel, OptimizationOptions optimizationOptions) throws OptimizationFailureException {
    _brokersAllowedReplicaMove = GoalUtils.aliveBrokersNotExcludedForReplicaMove(clusterModel, optimizationOptions);
    if (_brokersAllowedReplicaMove.isEmpty()) {
        // Handle the case when all alive brokers are excluded from replica moves.
        ProvisionRecommendation recommendation = new ProvisionRecommendation.Builder(ProvisionStatus.UNDER_PROVISIONED).numBrokers(clusterModel.maxReplicationFactor()).build();
        throw new OptimizationFailureException(String.format("[%s] All alive brokers are excluded from replica moves.", name()), recommendation);
    }
    _fixOfflineReplicasOnly = false;
    double resourceUtilization = clusterModel.load().expectedUtilizationFor(resource());
    double capacity = clusterModel.capacityWithAllowedReplicaMovesFor(resource(), optimizationOptions);
    // Cluster utilization excludes the capacity of brokers excluded for replica moves.
    double avgUtilizationPercentage = resourceUtilization / capacity;
    _balanceUpperThreshold = GoalUtils.computeResourceUtilizationBalanceThreshold(avgUtilizationPercentage, resource(), _balancingConstraint, optimizationOptions.isTriggeredByGoalViolation(), BALANCE_MARGIN, false);
    _balanceLowerThreshold = GoalUtils.computeResourceUtilizationBalanceThreshold(avgUtilizationPercentage, resource(), _balancingConstraint, optimizationOptions.isTriggeredByGoalViolation(), BALANCE_MARGIN, true);
    // Identify whether the cluster is in a low utilization state.
    _isLowUtilization = avgUtilizationPercentage <= _balancingConstraint.lowUtilizationThreshold(resource());
    if (_isLowUtilization) {
        // Identify a typical broker capacity to be used in recommendations in case the cluster is over-provisioned.
        // To make sure that the recommended brokers to drop in an heterogeneous cluster is a positive number, the typical broker must have
        // the max capacity.
        int typicalBrokerId = brokerIdWithMaxCapacity(clusterModel);
        double typicalCapacity = clusterModel.broker(typicalBrokerId).capacityFor(resource());
        // Any capacity greater than the allowed capacity may yield over-provisioning.
        double allowedCapacity = resourceUtilization / _balancingConstraint.lowUtilizationThreshold(resource());
        int allowedNumBrokers = (int) (allowedCapacity / typicalCapacity);
        int numBrokersToDrop = Math.max(_brokersAllowedReplicaMove.size() - allowedNumBrokers, 1);
        _overProvisionedRecommendation = new ProvisionRecommendation.Builder(ProvisionStatus.OVER_PROVISIONED).numBrokers(numBrokersToDrop).typicalBrokerCapacity(typicalCapacity).typicalBrokerId(typicalBrokerId).resource(resource()).build();
    }
}