Examples with Resource com.linkedin.kafka.cruisecontrol.common.Resource used on opensource projects

Example 6 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class PartitionMetricSampleTest method testSerde.

@Test
public void testSerde() throws UnknownVersionException {
    MetricDef metricDef = KafkaCruiseControlMetricDef.metricDef();
    PartitionMetricSample sample = new PartitionMetricSample(0, new TopicPartition("topic", 0));
    int i = 0;
    for (Resource r : Resource.values()) {
        sample.record(KafkaCruiseControlMetricDef.resourceToMetricInfo(r), i);
        i++;
    }
    sample.record(metricDef.metricInfo(PRODUCE_RATE.name()), (double) i++);
    sample.record(metricDef.metricInfo(FETCH_RATE.name()), (double) i++);
    sample.record(metricDef.metricInfo(MESSAGE_IN_RATE.name()), (double) i++);
    sample.record(metricDef.metricInfo(REPLICATION_BYTES_IN_RATE.name()), (double) i++);
    sample.record(metricDef.metricInfo(REPLICATION_BYTES_OUT_RATE.name()), (double) i);
    sample.close(10);
    byte[] bytes = sample.toBytes();
    PartitionMetricSample deserializedSample = PartitionMetricSample.fromBytes(bytes);
    assertEquals(sample.brokerId(), deserializedSample.brokerId());
    assertEquals(sample.entity().tp(), deserializedSample.entity().tp());
    assertEquals(sample.metricValue(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.CPU)), deserializedSample.metricValue(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.CPU)));
    assertEquals(sample.metricValue(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.DISK)), deserializedSample.metricValue(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.DISK)));
    assertEquals(sample.metricValue(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_IN)), deserializedSample.metricValue(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_IN)));
    assertEquals(sample.metricValue(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_OUT)), deserializedSample.metricValue(KafkaCruiseControlMetricDef.resourceToMetricId(Resource.NW_OUT)));
    assertEquals(sample.metricValue(metricDef.metricInfo(PRODUCE_RATE.name()).id()), deserializedSample.metricValue(metricDef.metricInfo(PRODUCE_RATE.name()).id()), EPSILON);
    assertEquals(sample.metricValue(metricDef.metricInfo(FETCH_RATE.name()).id()), deserializedSample.metricValue(metricDef.metricInfo(FETCH_RATE.name()).id()), EPSILON);
    assertEquals(sample.metricValue(metricDef.metricInfo(MESSAGE_IN_RATE.name()).id()), deserializedSample.metricValue(metricDef.metricInfo(MESSAGE_IN_RATE.name()).id()), EPSILON);
    assertEquals(sample.metricValue(metricDef.metricInfo(REPLICATION_BYTES_IN_RATE.name()).id()), deserializedSample.metricValue(metricDef.metricInfo(REPLICATION_BYTES_IN_RATE.name()).id()), EPSILON);
    assertEquals(sample.metricValue(metricDef.metricInfo(REPLICATION_BYTES_OUT_RATE.name()).id()), deserializedSample.metricValue(metricDef.metricInfo(REPLICATION_BYTES_OUT_RATE.name()).id()), EPSILON);
    assertEquals(sample.sampleTime(), deserializedSample.sampleTime());
}

Example 7 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class KafkaMetricSampleAggregatorTest method testRecentSnapshot.

@Test
public void testRecentSnapshot() throws NotEnoughValidWindowsException {
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(getLoadMonitorProperties());
    Metadata metadata = getMetadata(Collections.singleton(TP));
    KafkaMetricSampleAggregator metricSampleAggregator = new KafkaMetricSampleAggregator(config, metadata);
    populateSampleAggregator(NUM_WINDOWS + 1, MIN_SAMPLES_PER_WINDOW, metricSampleAggregator);
    MetricSampleAggregationResult<String, PartitionEntity> result = metricSampleAggregator.aggregate(clusterAndGeneration(metadata.fetch()), Long.MAX_VALUE, new OperationProgress());
    Map<PartitionEntity, ValuesAndExtrapolations> snapshotsForPartition = result.valuesAndExtrapolations();
    assertEquals("The snapshots should only have one partition", 1, snapshotsForPartition.size());
    ValuesAndExtrapolations snapshots = snapshotsForPartition.get(PE);
    assertNotNull(snapshots);
    assertEquals(NUM_WINDOWS, snapshots.metricValues().length());
    for (int i = 0; i < NUM_WINDOWS; i++) {
        assertEquals((NUM_WINDOWS - i) * WINDOW_MS, result.valuesAndExtrapolations().get(PE).window(i));
        for (Resource resource : Resource.values()) {
            double expectedValue = resource == Resource.DISK ? (NUM_WINDOWS - 1 - i) * 10 + MIN_SAMPLES_PER_WINDOW - 1 : (NUM_WINDOWS - 1 - i) * 10 + (MIN_SAMPLES_PER_WINDOW - 1) / 2.0;
            assertEquals("The utilization for " + resource + " should be " + expectedValue, expectedValue, snapshots.metricValues().valuesFor(KafkaCruiseControlMetricDef.resourceToMetricId(resource)).get(i), 0);
        }
    }
    // Verify the metric completeness checker state
    MetadataClient.ClusterAndGeneration clusterAndGeneration = new MetadataClient.ClusterAndGeneration(metadata.fetch(), 1);
    assertEquals(NUM_WINDOWS, metricSampleAggregator.validWindows(clusterAndGeneration, 1.0).size());
    Map<Long, Float> monitoredPercentages = metricSampleAggregator.partitionCoverageByWindows(clusterAndGeneration);
    for (double percentage : monitoredPercentages.values()) {
        assertEquals(1.0, percentage, 0.0);
    }
    assertEquals(NUM_WINDOWS, metricSampleAggregator.availableWindows().size());
}

Example 8 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class LoadConsistencyTest method data.

/**
 * Populate parameters for the parametrized test.
 */
@Parameterized.Parameters
public static Collection<Object[]> data() {
    Collection<Object[]> params = new ArrayList<>();
    Map<Resource, Double> brokerCapacity = new HashMap<>();
    brokerCapacity.put(Resource.CPU, TestConstants.LARGE_BROKER_CAPACITY);
    brokerCapacity.put(Resource.DISK, TestConstants.LARGE_BROKER_CAPACITY);
    brokerCapacity.put(Resource.NW_IN, TestConstants.LARGE_BROKER_CAPACITY);
    brokerCapacity.put(Resource.NW_OUT, TestConstants.MEDIUM_BROKER_CAPACITY);
    // --SUCCESS: sum of load on sub-components equals the load of component.
    // Test for success after removing replica.
    ClusterModel smallClusterModel = DeterministicCluster.smallClusterModel(brokerCapacity);
    TopicPartition pInfoT10 = new TopicPartition("T1", 0);
    smallClusterModel.removeReplica(0, pInfoT10);
    Object[] smallClusterModelParams = { smallClusterModel, true };
    params.add(smallClusterModelParams);
    ClusterModel mediumClusterModel = DeterministicCluster.mediumClusterModel(brokerCapacity);
    TopicPartition pInfoB0 = new TopicPartition("B", 0);
    mediumClusterModel.removeReplica(1, pInfoB0);
    Object[] mediumClusterModelParams = { mediumClusterModel, true };
    params.add(mediumClusterModelParams);
    // Test for success after relocating replica.
    ClusterModel smallReplicaMoveClusterModel = DeterministicCluster.smallClusterModel(brokerCapacity);
    smallReplicaMoveClusterModel.relocateReplica(pInfoT10, 0, 1);
    Object[] smallReplicaMoveClusterModelParams = { smallReplicaMoveClusterModel, true };
    params.add(smallReplicaMoveClusterModelParams);
    ClusterModel mediumReplicaMoveClusterModel = DeterministicCluster.mediumClusterModel(brokerCapacity);
    mediumReplicaMoveClusterModel.relocateReplica(pInfoB0, 1, 0);
    Object[] mediumReplicaMoveClusterModelParams = { mediumReplicaMoveClusterModel, true };
    params.add(mediumReplicaMoveClusterModelParams);
    // --FAILURE:  sum of load on sub-components not equal to the load of component.
    // Test for failure after removing replica in low level of abstraction only.
    ClusterModel smallFaultyClusterModel = DeterministicCluster.smallClusterModel(brokerCapacity);
    smallFaultyClusterModel.broker(2).removeReplica(pInfoT10);
    Object[] smallFaultyClusterModelParams = { smallFaultyClusterModel, false };
    params.add(smallFaultyClusterModelParams);
    ClusterModel mediumFaultyClusterModel = DeterministicCluster.mediumClusterModel(brokerCapacity);
    mediumFaultyClusterModel.broker(1).removeReplica(pInfoB0);
    Object[] mediumFaultyClusterModelParams = { mediumFaultyClusterModel, false };
    params.add(mediumFaultyClusterModelParams);
    // Test for failure after adding replica in low level of abstraction only.
    ClusterModel smallFaultyReplicaAddClusterModel = DeterministicCluster.smallClusterModel(brokerCapacity);
    smallFaultyReplicaAddClusterModel.broker(0).addReplica(new Replica(pInfoT10, smallFaultyReplicaAddClusterModel.broker(1), false));
    Object[] smallFaultyReplicaAddClusterModelParams = { smallFaultyReplicaAddClusterModel, false };
    params.add(smallFaultyReplicaAddClusterModelParams);
    ClusterModel mediumFaultyReplicaAddClusterModel = DeterministicCluster.mediumClusterModel(brokerCapacity);
    mediumFaultyReplicaAddClusterModel.broker(0).addReplica(new Replica(pInfoB0, mediumFaultyReplicaAddClusterModel.broker(1), false));
    Object[] mediumFaultyReplicaAddClusterModelParams = { mediumFaultyReplicaAddClusterModel, false };
    params.add(mediumFaultyReplicaAddClusterModelParams);
    return params;
}

Example 9 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class OptimizationVerifier method verifyNewBrokers.

private static boolean verifyNewBrokers(ClusterModel clusterModel, BalancingConstraint constraint) {
    for (Broker broker : clusterModel.healthyBrokers()) {
        if (!broker.isNew()) {
            for (Replica replica : broker.replicas()) {
                if (replica.originalBroker() != broker) {
                    LOG.error("Broker {} is not a new broker but has received new replicas", broker.id());
                    return false;
                }
            }
        }
    }
    for (Broker broker : clusterModel.newBrokers()) {
        // We can only check the first resource.
        Resource r = constraint.resources().get(0);
        double utilizationLowerThreshold = clusterModel.load().expectedUtilizationFor(r) / clusterModel.capacityFor(r) * (2 - constraint.resourceBalancePercentage(r));
        double brokerUtilization = broker.load().expectedUtilizationFor(r) / broker.capacityFor(r);
        if (brokerUtilization < utilizationLowerThreshold) {
            LOG.error("Broker {} is still underutilized for resource {}. Broker utilization is {}, the " + "lower threshold is {}", broker, r, brokerUtilization, utilizationLowerThreshold);
            return false;
        }
    }
    return true;
}

Example 10 with Resource

use of com.linkedin.kafka.cruisecontrol.common.Resource in project cruise-control by linkedin.

the class ClusterModel method relocateLeadership.

/**
 * (1) Removes leadership from source replica.
 * (2) Adds this leadership to the destination replica.
 * (3) Transfers the whole outbound network and a fraction of CPU load of source replica to the destination replica.
 * (4) Updates the leader and list of followers of the partition.
 *
 * @param tp      Topic partition of this replica.
 * @param sourceBrokerId      Source broker id.
 * @param destinationBrokerId Destination broker id.
 * @return True if relocation is successful, false otherwise.
 */
public boolean relocateLeadership(TopicPartition tp, int sourceBrokerId, int destinationBrokerId) {
    // Sanity check to see if the source replica is the leader.
    Replica sourceReplica = _partitionsByTopicPartition.get(tp).replica(sourceBrokerId);
    if (!sourceReplica.isLeader()) {
        return false;
    }
    // Sanity check to see if the destination replica is a follower.
    Replica destinationReplica = _partitionsByTopicPartition.get(tp).replica(destinationBrokerId);
    if (destinationReplica.isLeader()) {
        throw new IllegalArgumentException("Cannot relocate leadership of partition " + tp + "from broker " + sourceBrokerId + " to broker " + destinationBrokerId + " because the destination replica is a leader.");
    }
    // Transfer the leadership load (whole outbound network and a fraction of CPU load) of source replica to the
    // destination replica.
    // (1) Remove and get the outbound network load and a fraction of CPU load associated with leadership from the
    // given replica.
    // (2) Add the outbound network load and CPU load associated with leadership to the given replica.
    // 
    // Remove the load from the source rack.
    Rack rack = broker(sourceBrokerId).rack();
    Map<Resource, double[]> resourceToLeadershipLoadByWindowTime = rack.makeFollower(sourceBrokerId, tp);
    // Add the load to the destination rack.
    rack = broker(destinationBrokerId).rack();
    rack.makeLeader(destinationBrokerId, tp, resourceToLeadershipLoadByWindowTime);
    // Update the leader and list of followers of the partition.
    Partition partition = _partitionsByTopicPartition.get(tp);
    partition.relocateLeadership(destinationReplica);
    return true;
}