Examples with SupervisorTaskAutoScaler org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler used on opensource projects

Example 6 with SupervisorTaskAutoScaler

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler in project druid by druid-io.

the class KafkaSupervisorTest method testNoInitialStateWithAutoscaler.

@Test
public void testNoInitialStateWithAutoscaler() throws Exception {
    KafkaIndexTaskClientFactory taskClientFactory = new KafkaIndexTaskClientFactory(null, null) {

        @Override
        public KafkaIndexTaskClient build(TaskInfoProvider taskInfoProvider, String dataSource, int numThreads, Duration httpTimeout, long numRetries) {
            Assert.assertEquals(TEST_CHAT_THREADS, numThreads);
            Assert.assertEquals(TEST_HTTP_TIMEOUT.toStandardDuration(), httpTimeout);
            Assert.assertEquals(TEST_CHAT_RETRIES, numRetries);
            return taskClient;
        }
    };
    HashMap<String, Object> autoScalerConfig = new HashMap<>();
    autoScalerConfig.put("enableTaskAutoScaler", true);
    autoScalerConfig.put("lagCollectionIntervalMillis", 500);
    autoScalerConfig.put("lagCollectionRangeMillis", 500);
    autoScalerConfig.put("scaleOutThreshold", 0);
    autoScalerConfig.put("triggerScaleOutFractionThreshold", 0.0);
    autoScalerConfig.put("scaleInThreshold", 1000000);
    autoScalerConfig.put("triggerScaleInFractionThreshold", 0.8);
    autoScalerConfig.put("scaleActionStartDelayMillis", 0);
    autoScalerConfig.put("scaleActionPeriodMillis", 100);
    autoScalerConfig.put("taskCountMax", 2);
    autoScalerConfig.put("taskCountMin", 1);
    autoScalerConfig.put("scaleInStep", 1);
    autoScalerConfig.put("scaleOutStep", 2);
    autoScalerConfig.put("minTriggerScaleActionFrequencyMillis", 1200000);
    final Map<String, Object> consumerProperties = KafkaConsumerConfigs.getConsumerProperties();
    consumerProperties.put("myCustomKey", "myCustomValue");
    consumerProperties.put("bootstrap.servers", kafkaHost);
    KafkaSupervisorIOConfig kafkaSupervisorIOConfig = new KafkaSupervisorIOConfig(topic, INPUT_FORMAT, 1, 1, new Period("PT1H"), consumerProperties, OBJECT_MAPPER.convertValue(autoScalerConfig, LagBasedAutoScalerConfig.class), KafkaSupervisorIOConfig.DEFAULT_POLL_TIMEOUT_MILLIS, new Period("P1D"), new Period("PT30S"), true, new Period("PT30M"), null, null, null);
    final KafkaSupervisorTuningConfig tuningConfigOri = new KafkaSupervisorTuningConfig(null, 1000, null, null, 50000, null, new Period("P1Y"), new File("/test"), null, null, null, false, null, false, null, numThreads, TEST_CHAT_THREADS, TEST_CHAT_RETRIES, TEST_HTTP_TIMEOUT, TEST_SHUTDOWN_TIMEOUT, null, null, null, null, null);
    EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(kafkaSupervisorIOConfig).anyTimes();
    EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes();
    EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(tuningConfigOri).anyTimes();
    EasyMock.replay(ingestionSchema);
    SeekableStreamSupervisorSpec testableSupervisorSpec = new KafkaSupervisorSpec(ingestionSchema, dataSchema, tuningConfigOri, kafkaSupervisorIOConfig, null, false, taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, new NoopServiceEmitter(), new DruidMonitorSchedulerConfig(), rowIngestionMetersFactory, new SupervisorStateManagerConfig());
    supervisor = new TestableKafkaSupervisor(taskStorage, taskMaster, indexerMetadataStorageCoordinator, taskClientFactory, OBJECT_MAPPER, (KafkaSupervisorSpec) testableSupervisorSpec, rowIngestionMetersFactory);
    SupervisorTaskAutoScaler autoscaler = testableSupervisorSpec.createAutoscaler(supervisor);
    final KafkaSupervisorTuningConfig tuningConfig = supervisor.getTuningConfig();
    addSomeEvents(1);
    Capture<KafkaIndexTask> captured = Capture.newInstance();
    EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes();
    EasyMock.expect(taskStorage.getActiveTasksByDatasource(DATASOURCE)).andReturn(ImmutableList.of()).anyTimes();
    EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(new KafkaDataSourceMetadata(null)).anyTimes();
    EasyMock.expect(taskQueue.add(EasyMock.capture(captured))).andReturn(true);
    taskRunner.registerListener(EasyMock.anyObject(TaskRunnerListener.class), EasyMock.anyObject(Executor.class));
    replayAll();
    supervisor.start();
    int taskCountBeforeScale = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(1, taskCountBeforeScale);
    autoscaler.start();
    supervisor.runInternal();
    Thread.sleep(1 * 1000);
    verifyAll();
    int taskCountAfterScale = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(2, taskCountAfterScale);
    KafkaIndexTask task = captured.getValue();
    Assert.assertEquals(KafkaSupervisorTest.dataSchema, task.getDataSchema());
    Assert.assertEquals(tuningConfig.convertToTaskTuningConfig(), task.getTuningConfig());
    KafkaIndexTaskIOConfig taskConfig = task.getIOConfig();
    Assert.assertEquals(kafkaHost, taskConfig.getConsumerProperties().get("bootstrap.servers"));
    Assert.assertEquals("myCustomValue", taskConfig.getConsumerProperties().get("myCustomKey"));
    Assert.assertEquals("sequenceName-0", taskConfig.getBaseSequenceName());
    Assert.assertTrue("isUseTransaction", taskConfig.isUseTransaction());
    Assert.assertFalse("minimumMessageTime", taskConfig.getMinimumMessageTime().isPresent());
    Assert.assertFalse("maximumMessageTime", taskConfig.getMaximumMessageTime().isPresent());
    Assert.assertEquals(topic, taskConfig.getStartSequenceNumbers().getStream());
    Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(0));
    Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(1));
    Assert.assertEquals(0L, (long) taskConfig.getStartSequenceNumbers().getPartitionSequenceNumberMap().get(2));
    Assert.assertEquals(topic, taskConfig.getEndSequenceNumbers().getStream());
    Assert.assertEquals(Long.MAX_VALUE, (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(0));
    Assert.assertEquals(Long.MAX_VALUE, (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(1));
    Assert.assertEquals(Long.MAX_VALUE, (long) taskConfig.getEndSequenceNumbers().getPartitionSequenceNumberMap().get(2));
    autoscaler.reset();
    autoscaler.stop();
}

Example 7 with SupervisorTaskAutoScaler

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler in project druid by druid-io.

the class KinesisSupervisorTest method testNoInitialStateWithAutoScaleIn.

@Test
public void testNoInitialStateWithAutoScaleIn() throws Exception {
    HashMap<String, Object> autoScalerConfigMap = new HashMap<>();
    autoScalerConfigMap.put("enableTaskAutoScaler", true);
    autoScalerConfigMap.put("lagCollectionIntervalMillis", 500);
    autoScalerConfigMap.put("lagCollectionRangeMillis", 500);
    autoScalerConfigMap.put("scaleOutThreshold", 1000000);
    autoScalerConfigMap.put("triggerScaleOutFractionThreshold", 0.8);
    autoScalerConfigMap.put("scaleInThreshold", 0);
    autoScalerConfigMap.put("triggerScaleInFractionThreshold", 0.0);
    autoScalerConfigMap.put("scaleActionStartDelayMillis", 0);
    autoScalerConfigMap.put("scaleActionPeriodMillis", 100);
    autoScalerConfigMap.put("taskCountMax", 2);
    autoScalerConfigMap.put("taskCountMin", 1);
    autoScalerConfigMap.put("scaleInStep", 1);
    autoScalerConfigMap.put("scaleOutStep", 2);
    autoScalerConfigMap.put("minTriggerScaleActionFrequencyMillis", 1200000);
    AutoScalerConfig autoScalerConfig = OBJECT_MAPPER.convertValue(autoScalerConfigMap, AutoScalerConfig.class);
    supervisor = getTestableSupervisor(1, 2, true, "PT1H", null, null, false, null, null, autoScalerConfig);
    KinesisSupervisorSpec kinesisSupervisorSpec = supervisor.getKinesisSupervisorSpec();
    SupervisorTaskAutoScaler autoscaler = kinesisSupervisorSpec.createAutoscaler(supervisor);
    supervisorRecordSupplier.assign(EasyMock.anyObject());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getPartitionIds(STREAM)).andReturn(ImmutableSet.of(SHARD_ID1, SHARD_ID0)).anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getAssignment()).andReturn(ImmutableSet.of(SHARD1_PARTITION, SHARD0_PARTITION)).anyTimes();
    supervisorRecordSupplier.seekToLatest(EasyMock.anyObject());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getEarliestSequenceNumber(EasyMock.anyObject())).andReturn("0").anyTimes();
    supervisorRecordSupplier.seek(EasyMock.anyObject(), EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    Capture<KinesisIndexTask> captured = Capture.newInstance(CaptureType.ALL);
    EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes();
    EasyMock.expect(taskStorage.getActiveTasksByDatasource(DATASOURCE)).andReturn(ImmutableList.of()).anyTimes();
    EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(new KinesisDataSourceMetadata(null)).anyTimes();
    EasyMock.expect(taskQueue.add(EasyMock.capture(captured))).andReturn(true).times(2);
    replayAll();
    int taskCountInit = supervisor.getIoConfig().getTaskCount();
    // when enable autoScaler the init taskCount will be equal to taskCountMin
    Assert.assertEquals(1, taskCountInit);
    supervisor.getIoConfig().setTaskCount(2);
    supervisor.start();
    int taskCountBeforeScale = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(2, taskCountBeforeScale);
    autoscaler.start();
    supervisor.runInternal();
    verifyAll();
    Thread.sleep(1 * 1000);
    int taskCountAfterScale = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(1, taskCountAfterScale);
}

Example 8 with SupervisorTaskAutoScaler

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler in project druid by druid-io.

the class MaterializedViewSupervisorSpecTest method testMaterializedViewSupervisorSpecCreated.

@Test
public void testMaterializedViewSupervisorSpecCreated() {
    Exception ex = null;
    try {
        MaterializedViewSupervisorSpec spec = new MaterializedViewSupervisorSpec("wikiticker", new DimensionsSpec(Lists.newArrayList(new StringDimensionSchema("isUnpatrolled"), new StringDimensionSchema("metroCode"), new StringDimensionSchema("namespace"), new StringDimensionSchema("page"), new StringDimensionSchema("regionIsoCode"), new StringDimensionSchema("regionName"), new StringDimensionSchema("user"))), new AggregatorFactory[] { new CountAggregatorFactory("count"), new LongSumAggregatorFactory("added", "added") }, HadoopTuningConfig.makeDefaultTuningConfig(), null, null, null, null, null, false, objectMapper, null, null, null, null, null, new MaterializedViewTaskConfig(), EasyMock.createMock(AuthorizerMapper.class), new NoopChatHandlerProvider(), new SupervisorStateManagerConfig());
        Supervisor supervisor = spec.createSupervisor();
        Assert.assertTrue(supervisor instanceof MaterializedViewSupervisor);
        SupervisorTaskAutoScaler autoscaler = spec.createAutoscaler(supervisor);
        Assert.assertNull(autoscaler);
        try {
            supervisor.computeLagStats();
        } catch (Exception e) {
            Assert.assertTrue(e instanceof UnsupportedOperationException);
        }
        try {
            int count = supervisor.getActiveTaskGroupsCount();
        } catch (Exception e) {
            Assert.assertTrue(e instanceof UnsupportedOperationException);
        }
        Callable<Integer> noop = new Callable<Integer>() {

            @Override
            public Integer call() {
                return -1;
            }
        };
    } catch (Exception e) {
        ex = e;
    }
    Assert.assertNull(ex);
}

Example 9 with SupervisorTaskAutoScaler

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler in project druid by druid-io.

the class KinesisSupervisorTest method testNoInitialStateWithAutoScaleOut.

@Test
public void testNoInitialStateWithAutoScaleOut() throws Exception {
    HashMap<String, Object> autoScalerConfigMap = new HashMap<>();
    autoScalerConfigMap.put("enableTaskAutoScaler", true);
    autoScalerConfigMap.put("lagCollectionIntervalMillis", 500);
    autoScalerConfigMap.put("lagCollectionRangeMillis", 500);
    autoScalerConfigMap.put("scaleOutThreshold", 0);
    autoScalerConfigMap.put("triggerScaleOutFractionThreshold", 0.0);
    autoScalerConfigMap.put("scaleInThreshold", 1000000);
    autoScalerConfigMap.put("triggerScaleInFractionThreshold", 0.8);
    autoScalerConfigMap.put("scaleActionStartDelayMillis", 0);
    autoScalerConfigMap.put("scaleActionPeriodMillis", 100);
    autoScalerConfigMap.put("taskCountMax", 2);
    autoScalerConfigMap.put("taskCountMin", 1);
    autoScalerConfigMap.put("scaleInStep", 1);
    autoScalerConfigMap.put("scaleOutStep", 2);
    autoScalerConfigMap.put("minTriggerScaleActionFrequencyMillis", 1200000);
    AutoScalerConfig autoScalerConfig = OBJECT_MAPPER.convertValue(autoScalerConfigMap, AutoScalerConfig.class);
    supervisor = getTestableSupervisor(1, 1, true, "PT1H", null, null, false, null, null, autoScalerConfig);
    KinesisSupervisorSpec kinesisSupervisorSpec = supervisor.getKinesisSupervisorSpec();
    SupervisorTaskAutoScaler autoscaler = kinesisSupervisorSpec.createAutoscaler(supervisor);
    supervisorRecordSupplier.assign(EasyMock.anyObject());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getPartitionIds(STREAM)).andReturn(ImmutableSet.of(SHARD_ID1, SHARD_ID0)).anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getAssignment()).andReturn(ImmutableSet.of(SHARD1_PARTITION, SHARD0_PARTITION)).anyTimes();
    supervisorRecordSupplier.seekToLatest(EasyMock.anyObject());
    EasyMock.expectLastCall().anyTimes();
    EasyMock.expect(supervisorRecordSupplier.getEarliestSequenceNumber(EasyMock.anyObject())).andReturn("0").anyTimes();
    supervisorRecordSupplier.seek(EasyMock.anyObject(), EasyMock.anyString());
    EasyMock.expectLastCall().anyTimes();
    Capture<KinesisIndexTask> captured = Capture.newInstance();
    EasyMock.expect(taskMaster.getTaskQueue()).andReturn(Optional.of(taskQueue)).anyTimes();
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(taskRunner)).anyTimes();
    EasyMock.expect(taskStorage.getActiveTasksByDatasource(DATASOURCE)).andReturn(ImmutableList.of()).anyTimes();
    EasyMock.expect(indexerMetadataStorageCoordinator.retrieveDataSourceMetadata(DATASOURCE)).andReturn(new KinesisDataSourceMetadata(null)).anyTimes();
    EasyMock.expect(taskQueue.add(EasyMock.capture(captured))).andReturn(true);
    taskRunner.registerListener(EasyMock.anyObject(TaskRunnerListener.class), EasyMock.anyObject(Executor.class));
    replayAll();
    supervisor.start();
    int taskCountBeforeScale = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(1, taskCountBeforeScale);
    autoscaler.start();
    supervisor.runInternal();
    verifyAll();
    Thread.sleep(1 * 1000);
    int taskCountAfterScale = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(2, taskCountAfterScale);
}

Example 10 with SupervisorTaskAutoScaler

use of org.apache.druid.indexing.overlord.supervisor.autoscaler.SupervisorTaskAutoScaler in project druid by druid-io.

the class SupervisorManager method resetSupervisor.

public boolean resetSupervisor(String id, @Nullable DataSourceMetadata dataSourceMetadata) {
    Preconditions.checkState(started, "SupervisorManager not started");
    Preconditions.checkNotNull(id, "id");
    Pair<Supervisor, SupervisorSpec> supervisor = supervisors.get(id);
    if (supervisor == null) {
        return false;
    }
    supervisor.lhs.reset(dataSourceMetadata);
    SupervisorTaskAutoScaler autoscaler = autoscalers.get(id);
    if (autoscaler != null) {
        autoscaler.reset();
    }
    return true;
}