Examples with KafkaMetricSampleAggregator com.linkedin.kafka.cruisecontrol.monitor.sampling.aggregator.KafkaMetricSampleAggregator used on opensource projects

Example 6 with KafkaMetricSampleAggregator

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.aggregator.KafkaMetricSampleAggregator in project cruise-control by linkedin.

the class LoadMonitorTest method testStateWithoutEnoughSnapshotWindows.

@Test
public void testStateWithoutEnoughSnapshotWindows() {
    TestContext context = prepareContext();
    LoadMonitor loadMonitor = context.loadmonitor();
    KafkaMetricSampleAggregator aggregator = context.aggregator();
    // populate the metrics aggregator.
    // four samples for each partition except T1P1. T1P1 has no sample in the first window and one in the second window.
    CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T0P0, 0, WINDOW_MS, METRIC_DEF);
    CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T0P1, 0, WINDOW_MS, METRIC_DEF);
    CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T1P0, 0, WINDOW_MS, METRIC_DEF);
    LoadMonitorState state = loadMonitor.state(new OperationProgress());
    // The load monitor has 1 stable window with 0.5 of valid partitions ratio.
    assertEquals(0, state.numValidPartitions());
    assertEquals(0, state.numValidWindows());
    assertEquals(1, state.monitoredWindows().size());
    assertEquals(0.5, state.monitoredWindows().get(WINDOW_MS), 0.0);
    // Back fill for T1P1
    CruiseControlUnitTestUtils.populateSampleAggregator(1, 1, aggregator, PE_T1P1, 0, WINDOW_MS, METRIC_DEF);
    state = loadMonitor.state(new OperationProgress());
    // The load monitor now has one stable window with 1.0 of valid partitions ratio.
    assertEquals(0, state.numValidPartitions());
    assertEquals(1, state.numValidWindows());
    assertEquals(1, state.monitoredWindows().size());
    assertEquals(1.0, state.monitoredWindows().get(WINDOW_MS), 0.0);
}

Example 7 with KafkaMetricSampleAggregator

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.aggregator.KafkaMetricSampleAggregator in project cruise-control by linkedin.

the class LoadMonitorTest method testStateWithOnlyActiveSnapshotWindow.

@Test
public void testStateWithOnlyActiveSnapshotWindow() {
    TestContext context = prepareContext();
    LoadMonitor loadMonitor = context.loadmonitor();
    KafkaMetricSampleAggregator aggregator = context.aggregator();
    // populate the metrics aggregator.
    // four samples for each partition
    CruiseControlUnitTestUtils.populateSampleAggregator(1, 4, aggregator, PE_T0P0, 0, WINDOW_MS, METRIC_DEF);
    CruiseControlUnitTestUtils.populateSampleAggregator(1, 4, aggregator, PE_T0P1, 0, WINDOW_MS, METRIC_DEF);
    CruiseControlUnitTestUtils.populateSampleAggregator(1, 4, aggregator, PE_T1P0, 0, WINDOW_MS, METRIC_DEF);
    CruiseControlUnitTestUtils.populateSampleAggregator(1, 4, aggregator, PE_T1P1, 0, WINDOW_MS, METRIC_DEF);
    LoadMonitorState state = loadMonitor.state(new OperationProgress());
    // The load monitor only has an active window. There is no stable window.
    assertEquals(0, state.numValidPartitions());
    assertEquals(0, state.numValidWindows());
    assertTrue(state.monitoredWindows().isEmpty());
}

Example 8 with KafkaMetricSampleAggregator

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.aggregator.KafkaMetricSampleAggregator in project cruise-control by linkedin.

the class LoadMonitorTest method testClusterModelWithInvalidPartitionAndInsufficientSnapshotWindows.

// Not enough snapshot windows and some partitions are missing from all snapshot windows.
@Test
public void testClusterModelWithInvalidPartitionAndInsufficientSnapshotWindows() throws NotEnoughValidWindowsException {
    TestContext context = prepareContext();
    LoadMonitor loadMonitor = context.loadmonitor();
    KafkaMetricSampleAggregator aggregator = context.aggregator();
    ModelCompletenessRequirements requirements1 = new ModelCompletenessRequirements(1, 1.0, false);
    ModelCompletenessRequirements requirements2 = new ModelCompletenessRequirements(1, 0.5, false);
    ModelCompletenessRequirements requirements3 = new ModelCompletenessRequirements(2, 1.0, false);
    ModelCompletenessRequirements requirements4 = new ModelCompletenessRequirements(2, 0.5, false);
    // populate the metrics aggregator.
    // two samples for each partition except T1P1
    CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T0P0, 0, WINDOW_MS, METRIC_DEF);
    CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T0P1, 0, WINDOW_MS, METRIC_DEF);
    CruiseControlUnitTestUtils.populateSampleAggregator(2, 4, aggregator, PE_T1P0, 0, WINDOW_MS, METRIC_DEF);
    try {
        loadMonitor.clusterModel(-1, Long.MAX_VALUE, requirements1, new OperationProgress());
        fail("Should have thrown NotEnoughValidWindowsException.");
    } catch (NotEnoughValidWindowsException nevwe) {
    // let it go
    }
    ClusterModel clusterModel = loadMonitor.clusterModel(-1L, Long.MAX_VALUE, requirements2, new OperationProgress());
    assertNull(clusterModel.partition(T1P0));
    assertNull(clusterModel.partition(T1P1));
    assertEquals(1, clusterModel.partition(T0P0).leader().load().numWindows());
    assertEquals(3, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.DISK), 0.0);
    assertEquals(1.5, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.CPU), 0.0);
    assertEquals(1.5, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.NW_IN), 0.0);
    assertEquals(1.5, clusterModel.partition(T0P0).leader().load().expectedUtilizationFor(Resource.NW_OUT), 0.0);
    try {
        loadMonitor.clusterModel(-1L, Long.MAX_VALUE, requirements3, new OperationProgress());
        fail("Should have thrown NotEnoughValidWindowsException.");
    } catch (NotEnoughValidWindowsException nevwe) {
    // let it go
    }
    try {
        loadMonitor.clusterModel(-1L, Long.MAX_VALUE, requirements4, new OperationProgress());
        fail("Should have thrown NotEnoughValidWindowsException.");
    } catch (NotEnoughValidWindowsException nevwe) {
    // let it go
    }
}

Example 9 with KafkaMetricSampleAggregator

use of com.linkedin.kafka.cruisecontrol.monitor.sampling.aggregator.KafkaMetricSampleAggregator in project cruise-control by linkedin.

the class LoadMonitorTest method prepareContext.

private TestContext prepareContext() {
    // Create mock metadata client.
    Metadata metadata = getMetadata(Arrays.asList(T0P0, T0P1, T1P0, T1P1));
    MetadataClient mockMetadataClient = EasyMock.mock(MetadataClient.class);
    EasyMock.expect(mockMetadataClient.cluster()).andReturn(metadata.fetch()).anyTimes();
    EasyMock.expect(mockMetadataClient.clusterAndGeneration()).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
    EasyMock.expect(mockMetadataClient.metadata()).andReturn(metadata).anyTimes();
    EasyMock.expect(mockMetadataClient.refreshMetadata()).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
    EasyMock.expect(mockMetadataClient.refreshMetadata(anyLong())).andReturn(new MetadataClient.ClusterAndGeneration(metadata.fetch(), 0)).anyTimes();
    EasyMock.replay(mockMetadataClient);
    // create load monitor.
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    props.put(KafkaCruiseControlConfig.NUM_METRICS_WINDOWS_CONFIG, Integer.toString(NUM_WINDOWS));
    props.put(KafkaCruiseControlConfig.MIN_SAMPLES_PER_METRICS_WINDOW_CONFIG, Integer.toString(MIN_SAMPLES_PER_WINDOW));
    props.put(KafkaCruiseControlConfig.METRICS_WINDOW_MS_CONFIG, Long.toString(WINDOW_MS));
    props.put("cleanup.policy", DEFAULT_CLEANUP_POLICY);
    props.put(KafkaCruiseControlConfig.SAMPLE_STORE_CLASS_CONFIG, NoopSampleStore.class.getName());
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(props);
    LoadMonitor loadMonitor = new LoadMonitor(config, mockMetadataClient, _time, new MetricRegistry(), METRIC_DEF);
    KafkaMetricSampleAggregator aggregator = loadMonitor.aggregator();
    ModelParameters.init(config);
    loadMonitor.startUp();
    while (loadMonitor.state(new OperationProgress()).state() != LoadMonitorTaskRunner.LoadMonitorTaskRunnerState.RUNNING) {
        try {
            Thread.sleep(1);
        } catch (InterruptedException e) {
        // let it go.
        }
    }
    return new TestContext(loadMonitor, aggregator, config, metadata);
}