Examples with NoopTaskAutoScaler org.apache.druid.indexing.seekablestream.supervisor.autoscaler.NoopTaskAutoScaler used on opensource projects

Example 1 with NoopTaskAutoScaler

use of org.apache.druid.indexing.seekablestream.supervisor.autoscaler.NoopTaskAutoScaler in project druid by druid-io.

the class SeekableStreamSupervisorSpecTest method testSeekableStreamSupervisorSpecWithScaleDisable.

@Test
public void testSeekableStreamSupervisorSpecWithScaleDisable() throws InterruptedException {
    SeekableStreamSupervisorIOConfig seekableStreamSupervisorIOConfig = new SeekableStreamSupervisorIOConfig("stream", new JsonInputFormat(new JSONPathSpec(true, ImmutableList.of()), ImmutableMap.of(), false), 1, 1, new Period("PT1H"), new Period("P1D"), new Period("PT30S"), false, new Period("PT30M"), null, null, null, null) {
    };
    EasyMock.expect(spec.getSupervisorStateManagerConfig()).andReturn(supervisorConfig).anyTimes();
    EasyMock.expect(spec.getDataSchema()).andReturn(getDataSchema()).anyTimes();
    EasyMock.expect(spec.getIoConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes();
    EasyMock.expect(spec.getTuningConfig()).andReturn(getTuningConfig()).anyTimes();
    EasyMock.expect(spec.getEmitter()).andReturn(emitter).anyTimes();
    EasyMock.expect(spec.isSuspended()).andReturn(false).anyTimes();
    EasyMock.replay(spec);
    EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(this.seekableStreamSupervisorIOConfig).anyTimes();
    EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes();
    EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes();
    EasyMock.replay(ingestionSchema);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.absent()).anyTimes();
    EasyMock.expect(taskMaster.getSupervisorManager()).andReturn(Optional.absent()).anyTimes();
    EasyMock.replay(taskMaster);
    TestSeekableStreamSupervisor supervisor = new TestSeekableStreamSupervisor(3);
    NoopTaskAutoScaler autoScaler = new NoopTaskAutoScaler();
    supervisor.start();
    autoScaler.start();
    supervisor.runInternal();
    int taskCountBeforeScaleOut = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(1, taskCountBeforeScaleOut);
    Thread.sleep(1 * 1000);
    int taskCountAfterScaleOut = supervisor.getIoConfig().getTaskCount();
    Assert.assertEquals(1, taskCountAfterScaleOut);
    autoScaler.reset();
    autoScaler.stop();
}

Example 2 with NoopTaskAutoScaler

use of org.apache.druid.indexing.seekablestream.supervisor.autoscaler.NoopTaskAutoScaler in project druid by druid-io.

the class SeekableStreamSupervisorSpecTest method testAutoScalerCreated.

@Test
public void testAutoScalerCreated() {
    HashMap<String, Object> autoScalerConfig = new HashMap<>();
    autoScalerConfig.put("enableTaskAutoScaler", true);
    autoScalerConfig.put("lagCollectionIntervalMillis", 500);
    autoScalerConfig.put("lagCollectionRangeMillis", 500);
    autoScalerConfig.put("scaleOutThreshold", 5000000);
    autoScalerConfig.put("triggerScaleOutFractionThreshold", 0.3);
    autoScalerConfig.put("scaleInThreshold", 1000000);
    autoScalerConfig.put("triggerScaleInFractionThreshold", 0.8);
    autoScalerConfig.put("scaleActionStartDelayMillis", 0);
    autoScalerConfig.put("scaleActionPeriodMillis", 100);
    autoScalerConfig.put("taskCountMax", 8);
    autoScalerConfig.put("taskCountMin", 1);
    autoScalerConfig.put("scaleInStep", 1);
    autoScalerConfig.put("scaleOutStep", 2);
    autoScalerConfig.put("minTriggerScaleActionFrequencyMillis", 1200000);
    EasyMock.expect(ingestionSchema.getIOConfig()).andReturn(seekableStreamSupervisorIOConfig).anyTimes();
    EasyMock.expect(ingestionSchema.getDataSchema()).andReturn(dataSchema).anyTimes();
    EasyMock.expect(ingestionSchema.getTuningConfig()).andReturn(seekableStreamSupervisorTuningConfig).anyTimes();
    EasyMock.replay(ingestionSchema);
    EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes();
    EasyMock.replay(seekableStreamSupervisorIOConfig);
    EasyMock.expect(supervisor4.getActiveTaskGroupsCount()).andReturn(0).anyTimes();
    EasyMock.replay(supervisor4);
    TestSeekableStreamSupervisorSpec spec = new TestSeekableStreamSupervisorSpec(ingestionSchema, null, false, taskStorage, taskMaster, indexerMetadataStorageCoordinator, indexTaskClientFactory, mapper, emitter, monitorSchedulerConfig, rowIngestionMetersFactory, supervisorStateManagerConfig, supervisor4, "id1");
    SupervisorTaskAutoScaler autoscaler = spec.createAutoscaler(supervisor4);
    Assert.assertTrue(autoscaler instanceof LagBasedAutoScaler);
    EasyMock.reset(seekableStreamSupervisorIOConfig);
    autoScalerConfig.put("enableTaskAutoScaler", false);
    EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes();
    EasyMock.replay(seekableStreamSupervisorIOConfig);
    SupervisorTaskAutoScaler autoscaler2 = spec.createAutoscaler(supervisor4);
    Assert.assertTrue(autoscaler2 instanceof NoopTaskAutoScaler);
    EasyMock.reset(seekableStreamSupervisorIOConfig);
    autoScalerConfig.remove("enableTaskAutoScaler");
    EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes();
    EasyMock.replay(seekableStreamSupervisorIOConfig);
    SupervisorTaskAutoScaler autoscaler3 = spec.createAutoscaler(supervisor4);
    Assert.assertTrue(autoscaler3 instanceof NoopTaskAutoScaler);
    EasyMock.reset(seekableStreamSupervisorIOConfig);
    autoScalerConfig.clear();
    EasyMock.expect(seekableStreamSupervisorIOConfig.getAutoscalerConfig()).andReturn(mapper.convertValue(autoScalerConfig, AutoScalerConfig.class)).anyTimes();
    EasyMock.replay(seekableStreamSupervisorIOConfig);
    Assert.assertTrue(autoScalerConfig.isEmpty());
    SupervisorTaskAutoScaler autoscaler4 = spec.createAutoscaler(supervisor4);
    Assert.assertTrue(autoscaler4 instanceof NoopTaskAutoScaler);
}

Example 3 with NoopTaskAutoScaler

use of org.apache.druid.indexing.seekablestream.supervisor.autoscaler.NoopTaskAutoScaler in project druid by druid-io.

the class OverlordSecurityResourceFilterTest method setUp.

@Before
public void setUp() {
    if (resourceFilter instanceof TaskResourceFilter && !mockedOnceTsqa) {
        // Since we are creating the mocked tsqa object only once and getting that object from Guice here therefore
        // if the mockedOnce check is not done then we will call EasyMock.expect and EasyMock.replay on the mocked object
        // multiple times and it will throw exceptions
        tsqa = injector.getInstance(TaskStorageQueryAdapter.class);
        EasyMock.expect(tsqa.getTask(EasyMock.anyString())).andReturn(Optional.of(noopTask)).anyTimes();
        EasyMock.replay(tsqa);
        mockedOnceTsqa = true;
    }
    if (resourceFilter instanceof SupervisorResourceFilter && !mockedOnceSM) {
        supervisorManager = injector.getInstance(SupervisorManager.class);
        SupervisorSpec supervisorSpec = new SupervisorSpec() {

            @Override
            public String getId() {
                return "id";
            }

            @Override
            public Supervisor createSupervisor() {
                return null;
            }

            @Override
            public SupervisorTaskAutoScaler createAutoscaler(Supervisor supervisor) {
                return new NoopTaskAutoScaler();
            }

            @Override
            public List<String> getDataSources() {
                return ImmutableList.of("test");
            }

            @Override
            public SupervisorSpec createSuspendedSpec() {
                return null;
            }

            @Override
            public SupervisorSpec createRunningSpec() {
                return null;
            }

            @Override
            public boolean isSuspended() {
                return false;
            }

            @Override
            public String getType() {
                return null;
            }

            @Override
            public String getSource() {
                return null;
            }
        };
        EasyMock.expect(supervisorManager.getSupervisorSpec(EasyMock.anyString())).andReturn(Optional.of(supervisorSpec)).anyTimes();
        EasyMock.replay(supervisorManager);
        mockedOnceSM = true;
    }
    setUp(resourceFilter);
}