Examples with KafkaCruiseControlConfig com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig used on opensource projects

Example 16 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class ExcludedTopicsTest method goal.

private static Goal goal(Class<? extends Goal> goalClass) throws Exception {
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    props.setProperty(KafkaCruiseControlConfig.MAX_REPLICAS_PER_BROKER_CONFIG, Long.toString(1L));
    BalancingConstraint balancingConstraint = new BalancingConstraint(new KafkaCruiseControlConfig(props));
    balancingConstraint.setResourceBalancePercentage(TestConstants.LOW_BALANCE_PERCENTAGE);
    balancingConstraint.setCapacityThreshold(TestConstants.MEDIUM_CAPACITY_THRESHOLD);
    try {
        Constructor<? extends Goal> constructor = goalClass.getDeclaredConstructor(BalancingConstraint.class);
        constructor.setAccessible(true);
        return constructor.newInstance(balancingConstraint);
    } catch (NoSuchMethodException badConstructor) {
        // Try default constructor
        return goalClass.newInstance();
    }
}

Example 17 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class OfflineProposalGenerator method main.

public static void main(String[] argv) throws Exception {
    // TODO: probably need to save this in the original model file
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(props);
    ModelUtils.init(config);
    ModelParameters.init(config);
    BalancingConstraint balancingConstraint = new BalancingConstraint(config);
    long start = System.currentTimeMillis();
    ClusterModel clusterModel = clusterModelFromFile(argv[0]);
    long end = System.currentTimeMillis();
    double duration = (end - start) / 1000.0;
    System.out.println("Model loaded in " + duration + "s.");
    ClusterModelStats origStats = clusterModel.getClusterStats(balancingConstraint);
    String loadBeforeOptimization = clusterModel.brokerStats().toString();
    // Instantiate the components.
    GoalOptimizer goalOptimizer = new GoalOptimizer(config, null, new SystemTime(), new MetricRegistry());
    start = System.currentTimeMillis();
    GoalOptimizer.OptimizerResult optimizerResult = goalOptimizer.optimizations(clusterModel, new OperationProgress());
    end = System.currentTimeMillis();
    duration = (end - start) / 1000.0;
    String loadAfterOptimization = clusterModel.brokerStats().toString();
    System.out.println("Optimize goals in " + duration + "s.");
    System.out.println(optimizerResult.goalProposals().size());
    System.out.println(loadBeforeOptimization);
    System.out.println(loadAfterOptimization);
    ClusterModelStats optimizedStats = clusterModel.getClusterStats(balancingConstraint);
    double[] testStatistics = AnalyzerUtils.testDifference(origStats.utilizationMatrix(), optimizedStats.utilizationMatrix());
    System.out.println(Arrays.stream(RawAndDerivedResource.values()).map(x -> x.toString()).collect(Collectors.joining(", ")));
    System.out.println(Arrays.stream(testStatistics).boxed().map(pValue -> Double.toString(pValue)).collect(Collectors.joining(", ")));
}

Example 18 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class OptimizationVerifier method executeGoalsFor.

/**
 * Execute given goals in the given cluster enforcing the given constraint. Return pass / fail status of a test.
 * A test fails if:
 * 1) Rebalance: During the optimization process, optimization of a goal leads to a worse cluster state (in terms of
 * the requirements of the same goal) than the cluster state just before starting the optimization.
 * 2) Self Healing: There are replicas on dead brokers after self healing.
 * 3) Adding a new broker causes the replicas to move among old brokers.
 *
 * @param constraint         Balancing constraint for the given cluster.
 * @param clusterModel       The state of the cluster.
 * @param goalNameByPriority Name of goals by the order of execution priority.
 * @param excludedTopics     The excluded topics.
 * @param verifications      The verifications to make after the optimization.
 * @return Pass / fail status of a test.
 */
static boolean executeGoalsFor(BalancingConstraint constraint, ClusterModel clusterModel, Map<Integer, String> goalNameByPriority, Collection<String> excludedTopics, List<Verification> verifications) throws Exception {
    // Get the initial stats from the cluster.
    ClusterModelStats preOptimizedStats = clusterModel.getClusterStats(constraint);
    // Set goals by their priority.
    SortedMap<Integer, Goal> goalByPriority = new TreeMap<>();
    for (Map.Entry<Integer, String> goalEntry : goalNameByPriority.entrySet()) {
        Integer priority = goalEntry.getKey();
        String goalClassName = goalEntry.getValue();
        Class<? extends Goal> goalClass = (Class<? extends Goal>) Class.forName(goalClassName);
        try {
            Constructor<? extends Goal> constructor = goalClass.getDeclaredConstructor(BalancingConstraint.class);
            constructor.setAccessible(true);
            goalByPriority.put(priority, constructor.newInstance(constraint));
        } catch (NoSuchMethodException badConstructor) {
            // Try default constructor
            goalByPriority.put(priority, goalClass.newInstance());
        }
    }
    // Generate the goalOptimizer and optimize given goals.
    long startTime = System.currentTimeMillis();
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    StringJoiner stringJoiner = new StringJoiner(",");
    excludedTopics.forEach(stringJoiner::add);
    props.setProperty(KafkaCruiseControlConfig.TOPICS_EXCLUDED_FROM_PARTITION_MOVEMENT_CONFIG, stringJoiner.toString());
    GoalOptimizer goalOptimizer = new GoalOptimizer(new KafkaCruiseControlConfig(constraint.setProps(props)), null, new SystemTime(), new MetricRegistry());
    GoalOptimizer.OptimizerResult optimizerResult = goalOptimizer.optimizations(clusterModel, goalByPriority, new OperationProgress());
    LOG.trace("Took {} ms to execute {} to generate {} proposals.", System.currentTimeMillis() - startTime, goalByPriority, optimizerResult.goalProposals().size());
    for (Verification verification : verifications) {
        switch(verification) {
            case GOAL_VIOLATION:
                if (!verifyGoalViolations(optimizerResult)) {
                    return false;
                }
                break;
            case NEW_BROKERS:
                if (!clusterModel.newBrokers().isEmpty() && !verifyNewBrokers(clusterModel, constraint)) {
                    return false;
                }
                break;
            case DEAD_BROKERS:
                if (!clusterModel.deadBrokers().isEmpty() && !verifyDeadBrokers(clusterModel)) {
                    return false;
                }
                break;
            case REGRESSION:
                if (clusterModel.selfHealingEligibleReplicas().isEmpty() && !verifyRegression(optimizerResult, preOptimizedStats)) {
                    return false;
                }
                break;
            default:
                throw new IllegalStateException("Invalid verification " + verification);
        }
    }
    return true;
}

Example 19 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class KafkaAssignerDiskUsageDistributionGoalTest method testFindReplicaToSwapWith.

@Test
public void testFindReplicaToSwapWith() {
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    props.setProperty(KafkaCruiseControlConfig.MAX_REPLICAS_PER_BROKER_CONFIG, Long.toString(10L));
    props.setProperty(KafkaCruiseControlConfig.DISK_BALANCE_THRESHOLD_CONFIG, "1.05");
    BalancingConstraint balancingConstraint = new BalancingConstraint(new KafkaCruiseControlConfig(props));
    KafkaAssignerDiskUsageDistributionGoal goal = new KafkaAssignerDiskUsageDistributionGoal(balancingConstraint);
    ClusterModel clusterModel = createClusterModel();
    Broker b2 = clusterModel.broker(2);
    Replica r = b2.replica(T0P1);
    assertNull(goal.findReplicaToSwapWith(r, sortedReplicaAscend(clusterModel.broker(1)), 30, 10, 90, clusterModel));
    // The replicas on broker 3 are of the following sizes
    // T0P0: 10
    // T0P2: 20
    // T1P1: 30
    // T2P2: 50
    // T2P1: 60
    // T1P0: 80
    // Only T0P0 and T1P1 are eligible to swap with r.
    findReplicaToSwapWithAndVerify(Arrays.asList(-1.0, 5.0, 10.0, 20.0, 21.0, 60.0, 100.0), Arrays.asList(T0P0, T0P0, T0P0, T0P0, T1P1, T1P1, T1P1), 9, 90, r, 3, clusterModel, goal);
    findReplicaToSwapWithAndVerify(Arrays.asList(-1.0, 5.0, 10.0, 20.0, 21.0, 60.0, 100.0), Arrays.asList(T1P1, T1P1, T1P1, T1P1, T1P1, T1P1, T1P1), 10, 31, r, 3, clusterModel, goal);
    findReplicaToSwapWithAndVerify(Arrays.asList(-1.0, 5.0, 10.0, 20.0, 21.0, 60.0, 100.0), Arrays.asList(T0P0, T0P0, T0P0, T0P0, T0P0, T0P0, T0P0), 9, 30, r, 3, clusterModel, goal);
    findReplicaToSwapWithAndVerify(Arrays.asList(-1.0, 5.0, 10.0, 20.0, 21.0, 60.0, 100.0), Arrays.asList(null, null, null, null, null, null, null), 10, 30, r, 3, clusterModel, goal);
}

Example 20 with KafkaCruiseControlConfig

use of com.linkedin.kafka.cruisecontrol.config.KafkaCruiseControlConfig in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method setupScenario2.

/**
 * Two topics with 2 partitions each.
 * T1P1 misses window 6000 (index=5), 7000 (index=6) and 20000 (index=19)
 * Other partitions has full data.
 */
private TestContext setupScenario2() {
    TopicPartition t0p1 = new TopicPartition(TOPIC, 1);
    TopicPartition t1p0 = new TopicPartition("TOPIC1", 0);
    TopicPartition t1p1 = new TopicPartition("TOPIC1", 1);
    List<TopicPartition> allPartitions = Arrays.asList(TP, t0p1, t1p0, t1p1);
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(allPartitions);
    KafkaMetricSampleAggregator aggregator = new KafkaMetricSampleAggregator(config, metadata);
    for (TopicPartition tp : Arrays.asList(TP, t0p1, t1p0)) {
        populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, aggregator, tp);
    }
    // Let t1p1 miss two consecutive windows and the most recent window.
    populateSampleAggregator(5, MIN_SAMPLES_PER_WINDOW, aggregator, t1p1);
    CruiseControlUnitTestUtils.populateSampleAggregator(NUM_WINDOWS - 8, MIN_SAMPLES_PER_WINDOW, aggregator, new PartitionEntity(t1p1), 7, WINDOW_MS, KafkaCruiseControlMetricDef.metricDef());
    return new TestContext(metadata, aggregator);
}