Examples with AutoScalerConfig org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig used on opensource projects

Example 1 with AutoScalerConfig

use of org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig in project druid by druid-io.

the class KinesisSupervisorTest method testNoInitialStateWithAutoScaleIn.

@Test
public void testNoInitialStateWithAutoScaleIn() throws Exception {
    HashMap<String, Object> autoScalerConfigMap = new HashMap<>();
    autoScalerConfigMap.put("enableTaskAutoScaler", true);
    autoScalerConfigMap.put("lagCollectionIntervalMillis", 500);
    autoScalerConfigMap.put("lagCollectionRangeMillis", 500);
    autoScalerConfigMap.put("scaleOutThreshold", 1000000);
    autoScalerConfigMap.put("triggerScaleOutFractionThreshold", 0.8);
    autoScalerConfigMap.put("scaleInThreshold", 0);
    autoScalerConfigMap.put("triggerScaleInFractionThreshold", 0.0);
    autoScalerConfigMap.put("scaleActionStartDelayMillis", 0);
    autoScalerConfigMap.put("scaleActionPeriodMillis", 100);
    autoScalerConfigMap.put("taskCountMax", 2);
    autoScalerConfigMap.put("taskCountMin", 1);
    autoScalerConfigMap.put("scaleInStep", 1);
    autoScalerConfigMap.put("scaleOutStep", 2);
    autoScalerConfigMap.put("minTriggerScaleActionFrequencyMillis", 1200000);
    AutoScalerConfig autoScalerConfig = OBJECT_MAPPER.convertValue(autoScalerConfigMap, AutoScalerConfig.class);
    supervisor = getTestableSupervisor(1, 2, true, "PT1H", null, null, false, null, null, autoScalerConfig);
    KinesisSupervisorSpec kinesisSupervisorSpec = supervisor.getKinesisSupervisorSpec();
    SupervisorTaskAutoScaler autoscaler = kinesisSupervisorSpec.createAutoscaler(supervisor);
    supervisorRecordSupplier.assign(EasyMock.anyObject());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getPartitionIds(STREAM)).andReturn(ImmutableSet.of(SHARD_ID1, SHARD_ID0)).anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getAssignment()).andReturn(ImmutableSet.of(SHARD1_PARTITION, SHARD0_PARTITION)).anyTimes();
    supervisorRecordSupplier.seekToLatest(EasyMock.anyObject());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getEarliestSequenceNumber(EasyMock.anyObject())).andReturn("0").anyTimes();
    supervisorRecordSupplier.seek(EasyMock.anyObject(), EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    Capture<KinesisIndexTask> captured = Capture.newInstance(CaptureType.ALL);
    EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes();
    EasyMock.expect(taskStorage.getActiveTasksByDatasource(DATASOURCE)).andReturn(ImmutableList.of()).anyTimes();
    EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(new KinesisDataSourceMetadata(null)).anyTimes();
    EasyMock.expect(taskQueue.add(EasyMock.capture(captured))).andReturn(true).times(2);
    replayAll();
    int taskCountInit = supervisor.getIoConfig().getTaskCount();
    // when enable autoScaler the init taskCount will be equal to taskCountMin
    Assert.assertEquals(1, taskCountInit);
    supervisor.getIoConfig().setTaskCount(2);
    supervisor.start();
    int taskCountBeforeScale = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(2, taskCountBeforeScale);
    autoscaler.start();
    supervisor.runInternal();
    verifyAll();
    Thread.sleep(1 * 1000);
    int taskCountAfterScale = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(1, taskCountAfterScale);
}

Example 2 with AutoScalerConfig

use of org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig in project druid by druid-io.

the class KinesisSupervisorTest method testKinesisIOConfigInitAndAutoscalerConfigCreation.

@Test
public void testKinesisIOConfigInitAndAutoscalerConfigCreation() {
    // create KinesisSupervisorIOConfig with autoScalerConfig null
    KinesisSupervisorIOConfig kinesisSupervisorIOConfigWithNullAutoScalerConfig = new KinesisSupervisorIOConfig(STREAM, INPUT_FORMAT, "awsEndpoint", null, 1, 1, new Period("PT30M"), new Period("P1D"), new Period("PT30S"), false, new Period("PT30M"), null, null, null, 100, 1000, null, null, null, false);
    AutoScalerConfig autoscalerConfigNull = kinesisSupervisorIOConfigWithNullAutoScalerConfig.getAutoscalerConfig();
    Assert.assertNull(autoscalerConfigNull);
    // create KinesisSupervisorIOConfig with autoScalerConfig Empty
    KinesisSupervisorIOConfig kinesisSupervisorIOConfigWithEmptyAutoScalerConfig = new KinesisSupervisorIOConfig(STREAM, INPUT_FORMAT, "awsEndpoint", null, 1, 1, new Period("PT30M"), new Period("P1D"), new Period("PT30S"), false, new Period("PT30M"), null, null, null, 100, 1000, null, null, OBJECT_MAPPER.convertValue(new HashMap<>(), AutoScalerConfig.class), false);
    AutoScalerConfig autoscalerConfig = kinesisSupervisorIOConfigWithEmptyAutoScalerConfig.getAutoscalerConfig();
    Assert.assertNotNull(autoscalerConfig);
    Assert.assertTrue(autoscalerConfig instanceof LagBasedAutoScalerConfig);
    Assert.assertFalse(autoscalerConfig.getEnableTaskAutoScaler());
    Assert.assertTrue(autoscalerConfig.toString().contains("autoScalerConfig"));
}

Example 3 with AutoScalerConfig

use of org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig in project druid by druid-io.

the class KinesisSupervisorTest method testNoInitialStateWithAutoScaleOut.

@Test
public void testNoInitialStateWithAutoScaleOut() throws Exception {
    HashMap<String, Object> autoScalerConfigMap = new HashMap<>();
    autoScalerConfigMap.put("enableTaskAutoScaler", true);
    autoScalerConfigMap.put("lagCollectionIntervalMillis", 500);
    autoScalerConfigMap.put("lagCollectionRangeMillis", 500);
    autoScalerConfigMap.put("scaleOutThreshold", 0);
    autoScalerConfigMap.put("triggerScaleOutFractionThreshold", 0.0);
    autoScalerConfigMap.put("scaleInThreshold", 1000000);
    autoScalerConfigMap.put("triggerScaleInFractionThreshold", 0.8);
    autoScalerConfigMap.put("scaleActionStartDelayMillis", 0);
    autoScalerConfigMap.put("scaleActionPeriodMillis", 100);
    autoScalerConfigMap.put("taskCountMax", 2);
    autoScalerConfigMap.put("taskCountMin", 1);
    autoScalerConfigMap.put("scaleInStep", 1);
    autoScalerConfigMap.put("scaleOutStep", 2);
    autoScalerConfigMap.put("minTriggerScaleActionFrequencyMillis", 1200000);
    AutoScalerConfig autoScalerConfig = OBJECT_MAPPER.convertValue(autoScalerConfigMap, AutoScalerConfig.class);
    supervisor = getTestableSupervisor(1, 1, true, "PT1H", null, null, false, null, null, autoScalerConfig);
    KinesisSupervisorSpec kinesisSupervisorSpec = supervisor.getKinesisSupervisorSpec();
    SupervisorTaskAutoScaler autoscaler = kinesisSupervisorSpec.createAutoscaler(supervisor);
    supervisorRecordSupplier.assign(EasyMock.anyObject());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getPartitionIds(STREAM)).andReturn(ImmutableSet.of(SHARD_ID1, SHARD_ID0)).anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getAssignment()).andReturn(ImmutableSet.of(SHARD1_PARTITION, SHARD0_PARTITION)).anyTimes();
    supervisorRecordSupplier.seekToLatest(EasyMock.anyObject());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getEarliestSequenceNumber(EasyMock.anyObject())).andReturn("0").anyTimes();
    supervisorRecordSupplier.seek(EasyMock.anyObject(), EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    Capture<KinesisIndexTask> captured = Capture.newInstance();
    EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    EasyMock.expect(taskStorage.getActiveTasksByDatasource(DATASOURCE)).andReturn(ImmutableList.of()).anyTimes();
    EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(new KinesisDataSourceMetadata(null)).anyTimes();
    EasyMock.expect(taskQueue.add(EasyMock.capture(captured))).andReturn(true);
    taskRunner.registerListener(EasyMock.anyObject(TaskRunnerListener.class), EasyMock.anyObject(Executor.class));
    replayAll();
    supervisor.start();
    int taskCountBeforeScale = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(1, taskCountBeforeScale);
    autoscaler.start();
    supervisor.runInternal();
    verifyAll();
    Thread.sleep(1 * 1000);
    int taskCountAfterScale = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(2, taskCountAfterScale);
}

Example 4 with AutoScalerConfig

use of org.apache.druid.indexing.seekablestream.supervisor.autoscaler.AutoScalerConfig in project druid by druid-io.

the class SeekableStreamSupervisorSpecTest method testAutoScalerConfig.

@Test
public void testAutoScalerConfig() {
    AutoScalerConfig autoScalerConfigEmpty = mapper.convertValue(new HashMap<>(), AutoScalerConfig.class);
    Assert.assertTrue(autoScalerConfigEmpty instanceof LagBasedAutoScalerConfig);
    Assert.assertFalse(autoScalerConfigEmpty.getEnableTaskAutoScaler());
    AutoScalerConfig autoScalerConfigNull = mapper.convertValue(null, AutoScalerConfig.class);
    Assert.assertNull(autoScalerConfigNull);
    AutoScalerConfig autoScalerConfigDefault = mapper.convertValue(ImmutableMap.of("autoScalerStrategy", "lagBased"), AutoScalerConfig.class);
    Assert.assertTrue(autoScalerConfigDefault instanceof LagBasedAutoScalerConfig);
    AutoScalerConfig autoScalerConfigValue = mapper.convertValue(ImmutableMap.of("lagCollectionIntervalMillis", "1"), AutoScalerConfig.class);
    Assert.assertTrue(autoScalerConfigValue instanceof LagBasedAutoScalerConfig);
    LagBasedAutoScalerConfig lagBasedAutoScalerConfig = (LagBasedAutoScalerConfig) autoScalerConfigValue;
    Assert.assertEquals(lagBasedAutoScalerConfig.getLagCollectionIntervalMillis(), 1);
    Exception e = null;
    try {
        AutoScalerConfig autoScalerError = mapper.convertValue(ImmutableMap.of("enableTaskAutoScaler", "true", "taskCountMax", "1", "taskCountMin", "4"), AutoScalerConfig.class);
    } catch (RuntimeException ex) {
        e = ex;
    }
    Assert.assertNotNull(e);
    e = null;
    try {
        // taskCountMax and taskCountMin couldn't be ignored.
        AutoScalerConfig autoScalerError2 = mapper.convertValue(ImmutableMap.of("enableTaskAutoScaler", "true"), AutoScalerConfig.class);
    } catch (RuntimeException ex) {
        e = ex;
    }
    Assert.assertNotNull(e);
}