Examples with Resource com.linkedin.kafka.cruisecontrol.common.Resource used on opensource projects

Example 16 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class KafkaCruiseControlUnitTestUtils method getAggregatedMetricValues.

/**
 * Get the aggregated metric values with the given resource usage.
 */
public static AggregatedMetricValues getAggregatedMetricValues(double cpuUsage, double networkInBoundUsage, double networkOutBoundUsage, double diskUsage) {
    double[] values = new double[Resource.cachedValues().size()];
    values[KafkaCruiseControlMetricDef.resourceToMetricId(CPU)] = cpuUsage;
    values[KafkaCruiseControlMetricDef.resourceToMetricId(NW_IN)] = networkInBoundUsage;
    values[KafkaCruiseControlMetricDef.resourceToMetricId(NW_OUT)] = networkOutBoundUsage;
    values[KafkaCruiseControlMetricDef.resourceToMetricId(DISK)] = diskUsage;
    AggregatedMetricValues aggregateMetricValues = new AggregatedMetricValues();
    for (Resource r : Resource.cachedValues()) {
        int metricId = KafkaCruiseControlMetricDef.resourceToMetricId(r);
        MetricValues metricValues = new MetricValues(1);
        metricValues.set(0, values[metricId]);
        aggregateMetricValues.add(metricId, metricValues);
    }
    return aggregateMetricValues;
}

Example 17 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class DeterministicClusterTest method data.

/**
 * Populate parameters for the {@link OptimizationVerifier}. All brokers are alive.
 *
 * @return Parameters for the {@link OptimizationVerifier}.
 */
@Parameterized.Parameters
public static Collection<Object[]> data() {
    Collection<Object[]> p = new ArrayList<>();
    Map<Integer, String> goalNameByPriority = new HashMap<>();
    goalNameByPriority.put(1, RackAwareGoal.class.getName());
    goalNameByPriority.put(2, ReplicaCapacityGoal.class.getName());
    goalNameByPriority.put(3, DiskCapacityGoal.class.getName());
    goalNameByPriority.put(4, NetworkInboundCapacityGoal.class.getName());
    goalNameByPriority.put(5, NetworkOutboundCapacityGoal.class.getName());
    goalNameByPriority.put(6, CpuCapacityGoal.class.getName());
    goalNameByPriority.put(7, ReplicaDistributionGoal.class.getName());
    goalNameByPriority.put(8, PotentialNwOutGoal.class.getName());
    goalNameByPriority.put(9, DiskUsageDistributionGoal.class.getName());
    goalNameByPriority.put(10, NetworkInboundUsageDistributionGoal.class.getName());
    goalNameByPriority.put(11, NetworkOutboundUsageDistributionGoal.class.getName());
    goalNameByPriority.put(12, CpuUsageDistributionGoal.class.getName());
    goalNameByPriority.put(13, TopicReplicaDistributionGoal.class.getName());
    goalNameByPriority.put(14, PreferredLeaderElectionGoal.class.getName());
    goalNameByPriority.put(15, LeaderBytesInDistributionGoal.class.getName());
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    props.setProperty(KafkaCruiseControlConfig.MAX_REPLICAS_PER_BROKER_CONFIG, Long.toString(6L));
    BalancingConstraint balancingConstraint = new BalancingConstraint(new KafkaCruiseControlConfig(props));
    List<OptimizationVerifier.Verification> verifications = Arrays.asList(NEW_BROKERS, DEAD_BROKERS, REGRESSION);
    // ----------##TEST: BALANCE PERCENTAGES.
    balancingConstraint.setCapacityThreshold(TestConstants.MEDIUM_CAPACITY_THRESHOLD);
    List<Double> balancePercentages = new ArrayList<>();
    balancePercentages.add(TestConstants.HIGH_BALANCE_PERCENTAGE);
    balancePercentages.add(TestConstants.MEDIUM_BALANCE_PERCENTAGE);
    balancePercentages.add(TestConstants.LOW_BALANCE_PERCENTAGE);
    // -- TEST DECK #1: SMALL CLUSTER.
    for (Double balancePercentage : balancePercentages) {
        balancingConstraint.setResourceBalancePercentage(balancePercentage);
        p.add(params(balancingConstraint, DeterministicCluster.smallClusterModel(TestConstants.BROKER_CAPACITY), goalNameByPriority, verifications, null));
    }
    // -- TEST DECK #2: MEDIUM CLUSTER.
    for (Double balancePercentage : balancePercentages) {
        balancingConstraint.setResourceBalancePercentage(balancePercentage);
        p.add(params(balancingConstraint, DeterministicCluster.mediumClusterModel(TestConstants.BROKER_CAPACITY), goalNameByPriority, verifications, null));
    }
    // ----------##TEST: CAPACITY THRESHOLD.
    balancingConstraint.setResourceBalancePercentage(TestConstants.MEDIUM_BALANCE_PERCENTAGE);
    List<Double> capacityThresholds = new ArrayList<>();
    capacityThresholds.add(TestConstants.HIGH_CAPACITY_THRESHOLD);
    capacityThresholds.add(TestConstants.MEDIUM_CAPACITY_THRESHOLD);
    capacityThresholds.add(TestConstants.LOW_CAPACITY_THRESHOLD);
    // -- TEST DECK #3: SMALL CLUSTER.
    for (Double capacityThreshold : capacityThresholds) {
        balancingConstraint.setCapacityThreshold(capacityThreshold);
        p.add(params(balancingConstraint, DeterministicCluster.smallClusterModel(TestConstants.BROKER_CAPACITY), goalNameByPriority, verifications, null));
    }
    // -- TEST DECK #4: MEDIUM CLUSTER.
    for (Double capacityThreshold : capacityThresholds) {
        balancingConstraint.setCapacityThreshold(capacityThreshold);
        p.add(params(balancingConstraint, DeterministicCluster.mediumClusterModel(TestConstants.BROKER_CAPACITY), goalNameByPriority, verifications, null));
    }
    // ----------##TEST: BROKER CAPACITY.
    List<Double> brokerCapacities = new ArrayList<>();
    brokerCapacities.add(TestConstants.LARGE_BROKER_CAPACITY);
    brokerCapacities.add(TestConstants.MEDIUM_BROKER_CAPACITY);
    brokerCapacities.add(TestConstants.SMALL_BROKER_CAPACITY);
    // -- TEST DECK #5: SMALL AND MEDIUM CLUSTERS.
    for (Double capacity : brokerCapacities) {
        Map<Resource, Double> testBrokerCapacity = new HashMap<>();
        testBrokerCapacity.put(Resource.CPU, capacity);
        testBrokerCapacity.put(Resource.DISK, capacity);
        testBrokerCapacity.put(Resource.NW_IN, capacity);
        testBrokerCapacity.put(Resource.NW_OUT, capacity);
        p.add(params(balancingConstraint, DeterministicCluster.smallClusterModel(testBrokerCapacity), goalNameByPriority, verifications, null));
        p.add(params(balancingConstraint, DeterministicCluster.mediumClusterModel(testBrokerCapacity), goalNameByPriority, verifications, null));
    }
    Map<Integer, String> kafkaAssignerGoals = new HashMap<>();
    kafkaAssignerGoals.put(0, KafkaAssignerEvenRackAwareGoal.class.getName());
    kafkaAssignerGoals.put(1, KafkaAssignerDiskUsageDistributionGoal.class.getName());
    List<OptimizationVerifier.Verification> kafkaAssignerVerifications = Arrays.asList(DEAD_BROKERS, REGRESSION);
    // Small cluster.
    p.add(params(balancingConstraint, DeterministicCluster.smallClusterModel(TestConstants.BROKER_CAPACITY), kafkaAssignerGoals, kafkaAssignerVerifications, null));
    // Medium cluster.
    p.add(params(balancingConstraint, DeterministicCluster.mediumClusterModel(TestConstants.BROKER_CAPACITY), kafkaAssignerGoals, kafkaAssignerVerifications, null));
    // Rack-aware satisfiable.
    p.add(params(balancingConstraint, DeterministicCluster.rackAwareSatisfiable(), kafkaAssignerGoals, kafkaAssignerVerifications, null));
    // Rack-aware unsatisfiable.
    p.add(params(balancingConstraint, DeterministicCluster.rackAwareUnsatisfiable(), kafkaAssignerGoals, kafkaAssignerVerifications, OptimizationFailureException.class));
    return p;
}

Example 18 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class RandomClusterTest method testNewBrokers.

/**
 * This test first creates a random cluster, balance it. Then add two new brokers, balance the cluster again.
 */
public void testNewBrokers() throws Exception {
    ClusterModel clusterModel = rebalance();
    ClusterModel clusterWithNewBroker = new ClusterModel(new ModelGeneration(0, 0L), 1.0);
    for (Broker b : clusterModel.brokers()) {
        clusterWithNewBroker.createRack(b.rack().id());
        Map<Resource, Double> brokerCapacity = new HashMap<>();
        for (Resource r : Resource.cachedValues()) {
            brokerCapacity.put(r, b.capacityFor(r));
        }
        clusterWithNewBroker.createBroker(b.rack().id(), Integer.toString(b.id()), b.id(), brokerCapacity);
    }
    for (Map.Entry<String, List<Partition>> entry : clusterModel.getPartitionsByTopic().entrySet()) {
        for (Partition p : entry.getValue()) {
            int index = 0;
            for (Replica r : p.replicas()) {
                clusterWithNewBroker.createReplica(r.broker().rack().id(), r.broker().id(), p.topicPartition(), index++, r.isLeader());
            }
        }
    }
    for (Broker b : clusterModel.brokers()) {
        for (Replica replica : b.replicas()) {
            AggregatedMetricValues aggregatedMetricValues = clusterModel.broker(b.id()).replica(replica.topicPartition()).load().loadByWindows();
            clusterWithNewBroker.setReplicaLoad(b.rack().id(), b.id(), replica.topicPartition(), aggregatedMetricValues, clusterModel.load().windows());
        }
    }
    for (int i = 1; i < 3; i++) {
        clusterWithNewBroker.createBroker(Integer.toString(i), Integer.toString(i + clusterModel.brokers().size() - 1), i + clusterModel.brokers().size() - 1, TestConstants.BROKER_CAPACITY);
        clusterWithNewBroker.setBrokerState(i + clusterModel.brokers().size() - 1, Broker.State.NEW);
    }
    assertTrue("Random Cluster Test failed to improve the existing state with new brokers.", OptimizationVerifier.executeGoalsFor(_balancingConstraint, clusterWithNewBroker, _goalNameByPriority, _verifications));
}

Example 19 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method testExcludeInvalidMetricSample.

@Test
public void testExcludeInvalidMetricSample() throws NotEnoughValidWindowsException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
    MetricDef metricDef = KafkaCruiseControlMetricDef.metricDef();
    populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
    // Set the leader to be node 1, which is different from the leader in the metadata.
    PartitionMetricSample sampleWithDifferentLeader = new PartitionMetricSample(1, TP);
    sampleWithDifferentLeader.record(metricDef.metricInfo(DISK_USAGE.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_IN.name()), 10000);
    sampleWithDifferentLeader.record(metricDef.metricInfo(LEADER_BYTES_OUT.name()), 10000);
    sampleWithDifferentLeader.close(0);
    // Only populate the CPU metric
    PartitionMetricSample incompletePartitionMetricSample = new PartitionMetricSample(0, TP);
    incompletePartitionMetricSample.record(metricDef.metricInfo(CPU_USAGE.name()), 10000);
    incompletePartitionMetricSample.close(0);
    metricSampleAggregator.addSample(sampleWithDifferentLeader);
    metricSampleAggregator.addSample(incompletePartitionMetricSample);
    // Check the snapshot value and make sure the metric samples above are excluded.
    Map<PartitionEntity, ValuesAndExtrapolations> snapshotsForPartition = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), NUM_WINDOWS * WINDOW_MS, new OperationProgress()).valuesAndExtrapolations();
    ValuesAndExtrapolations snapshots = snapshotsForPartition.get(PE);
    for (Resource resource : Resource.values()) {
        int metricId = KafkaCruiseControlMetricDef.resourceToMetricId(resource);
        double expectedValue = resource == Resource.DISK ? MIN_SAMPLES_PER_WINDOW - 1 : (MIN_SAMPLES_PER_WINDOW - 1) / 2.0;
        assertEquals("The utilization for " + resource + " should be " + expectedValue, expectedValue, snapshots.metricValues().valuesFor(metricId).get(NUM_WINDOWS - 1), 0);
    }
}

Example 20 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class Rack method setBrokerState.

/**
 * Set the broker state and update the capacity
 */
void setBrokerState(int brokerId, Broker.State newState) {
    // Broker is dead
    Broker broker = broker(brokerId);
    broker.host().setBrokerState(brokerId, newState);
    for (Resource r : Resource.cachedValues()) {
        double capacity = 0;
        for (Host h : _hosts.values()) {
            capacity += h.capacityFor(r);
        }
        _rackCapacity[r.id()] = capacity;
    }
}