Examples with AutoScaler org.apache.druid.indexing.overlord.autoscaling.AutoScaler used on opensource projects

Example 1 with AutoScaler

use of org.apache.druid.indexing.overlord.autoscaling.AutoScaler in project druid by druid-io.

the class OverlordResourceTest method testGetTotalWorkerCapacityWithAutoScaleConfiguredAndProvisioningStrategySupportExpectedWorkerCapacity.

@Test
public void testGetTotalWorkerCapacityWithAutoScaleConfiguredAndProvisioningStrategySupportExpectedWorkerCapacity() {
    int expectedWorkerCapacity = 3;
    int maxNumWorkers = 2;
    WorkerTaskRunner workerTaskRunner = EasyMock.createMock(WorkerTaskRunner.class);
    Collection<ImmutableWorkerInfo> workerInfos = ImmutableList.of(new ImmutableWorkerInfo(new Worker("http", "testWorker", "192.0.0.1", expectedWorkerCapacity, "v1", WorkerConfig.DEFAULT_CATEGORY), 2, ImmutableSet.of("grp1", "grp2"), ImmutableSet.of("task1", "task2"), DateTimes.of("2015-01-01T01:01:01Z")));
    EasyMock.expect(workerTaskRunner.getWorkers()).andReturn(workerInfos);
    EasyMock.reset(taskMaster);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(workerTaskRunner)).anyTimes();
    EasyMock.expect(provisioningStrategy.getExpectedWorkerCapacity(workerInfos)).andReturn(expectedWorkerCapacity).anyTimes();
    AutoScaler autoScaler = EasyMock.createMock(AutoScaler.class);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(maxNumWorkers);
    DefaultWorkerBehaviorConfig workerBehaviorConfig = new DefaultWorkerBehaviorConfig(null, autoScaler);
    AtomicReference<WorkerBehaviorConfig> workerBehaviorConfigAtomicReference = new AtomicReference<>(workerBehaviorConfig);
    EasyMock.expect(configManager.watch(WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.class)).andReturn(workerBehaviorConfigAtomicReference);
    EasyMock.replay(workerTaskRunner, autoScaler, taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req, workerTaskRunnerQueryAdapter, configManager, provisioningStrategy);
    final Response response = overlordResource.getTotalWorkerCapacity();
    Assert.assertEquals(HttpResponseStatus.OK.getCode(), response.getStatus());
    Assert.assertEquals(expectedWorkerCapacity, ((TotalWorkerCapacityResponse) response.getEntity()).getCurrentClusterCapacity());
    Assert.assertEquals(expectedWorkerCapacity * maxNumWorkers, ((TotalWorkerCapacityResponse) response.getEntity()).getMaximumCapacityWithAutoScale());
}

Example 2 with AutoScaler

use of org.apache.druid.indexing.overlord.autoscaling.AutoScaler in project druid by druid-io.

the class GceAutoScalerTest method testConfig.

@Test
public void testConfig() {
    final String json = "{\n" + "   \"envConfig\" : {\n" + "      \"numInstances\" : 1,\n" + "      \"projectId\" : \"super-project\",\n" + "      \"zoneName\" : \"winkie-country\",\n" + "      \"managedInstanceGroupName\" : \"druid-mig\"\n" + "   },\n" + "   \"maxNumWorkers\" : 4,\n" + "   \"minNumWorkers\" : 2,\n" + "   \"type\" : \"gce\"\n" + "}";
    final ObjectMapper objectMapper = new DefaultObjectMapper().registerModules((Iterable<Module>) new GceModule().getJacksonModules());
    objectMapper.setInjectableValues(new InjectableValues() {

        @Override
        public Object findInjectableValue(Object o, DeserializationContext deserializationContext, BeanProperty beanProperty, Object o1) {
            return null;
        }
    });
    try {
        final GceAutoScaler autoScaler = (GceAutoScaler) objectMapper.readValue(json, AutoScaler.class);
        verifyAutoScaler(autoScaler);
        final GceAutoScaler roundTripAutoScaler = (GceAutoScaler) objectMapper.readValue(objectMapper.writeValueAsBytes(autoScaler), AutoScaler.class);
        verifyAutoScaler(roundTripAutoScaler);
        Assert.assertEquals("Round trip equals", autoScaler, roundTripAutoScaler);
    } catch (Exception e) {
        Assert.fail(StringUtils.format("Got exception in test %s", e.getMessage()));
    }
}

Example 3 with AutoScaler

use of org.apache.druid.indexing.overlord.autoscaling.AutoScaler in project druid by druid-io.

the class EC2AutoScalerSerdeTest method testSerde.

@Test
public void testSerde() throws Exception {
    final ObjectMapper objectMapper = new DefaultObjectMapper().registerModules((Iterable<Module>) new EC2Module().getJacksonModules());
    objectMapper.setInjectableValues(new InjectableValues() {

        @Override
        public Object findInjectableValue(Object o, DeserializationContext deserializationContext, BeanProperty beanProperty, Object o1) {
            return null;
        }
    });
    final EC2AutoScaler autoScaler = (EC2AutoScaler) objectMapper.readValue(json, AutoScaler.class);
    verifyAutoScaler(autoScaler);
    final EC2AutoScaler roundTripAutoScaler = (EC2AutoScaler) objectMapper.readValue(objectMapper.writeValueAsBytes(autoScaler), AutoScaler.class);
    verifyAutoScaler(roundTripAutoScaler);
    Assert.assertEquals("Round trip equals", autoScaler, roundTripAutoScaler);
}

Example 4 with AutoScaler

use of org.apache.druid.indexing.overlord.autoscaling.AutoScaler in project druid by druid-io.

the class OverlordResourceTest method testGetTotalWorkerCapacityWithAutoScaleConfiguredAndProvisioningStrategyNotSupportExpectedWorkerCapacity.

@Test
public void testGetTotalWorkerCapacityWithAutoScaleConfiguredAndProvisioningStrategyNotSupportExpectedWorkerCapacity() {
    int invalidExpectedCapacity = -1;
    int maxNumWorkers = 2;
    WorkerTaskRunner workerTaskRunner = EasyMock.createMock(WorkerTaskRunner.class);
    Collection<ImmutableWorkerInfo> workerInfos = ImmutableList.of(new ImmutableWorkerInfo(new Worker("http", "testWorker", "192.0.0.1", 3, "v1", WorkerConfig.DEFAULT_CATEGORY), 2, ImmutableSet.of("grp1", "grp2"), ImmutableSet.of("task1", "task2"), DateTimes.of("2015-01-01T01:01:01Z")));
    EasyMock.expect(workerTaskRunner.getWorkers()).andReturn(workerInfos);
    EasyMock.reset(taskMaster);
    EasyMock.expect(taskMaster.getTaskRunner()).andReturn(Optional.of(workerTaskRunner)).anyTimes();
    EasyMock.expect(provisioningStrategy.getExpectedWorkerCapacity(workerInfos)).andReturn(invalidExpectedCapacity).anyTimes();
    AutoScaler autoScaler = EasyMock.createMock(AutoScaler.class);
    EasyMock.expect(autoScaler.getMinNumWorkers()).andReturn(0);
    EasyMock.expect(autoScaler.getMaxNumWorkers()).andReturn(maxNumWorkers);
    DefaultWorkerBehaviorConfig workerBehaviorConfig = new DefaultWorkerBehaviorConfig(null, autoScaler);
    AtomicReference<WorkerBehaviorConfig> workerBehaviorConfigAtomicReference = new AtomicReference<>(workerBehaviorConfig);
    EasyMock.expect(configManager.watch(WorkerBehaviorConfig.CONFIG_KEY, WorkerBehaviorConfig.class)).andReturn(workerBehaviorConfigAtomicReference);
    EasyMock.replay(workerTaskRunner, autoScaler, taskRunner, taskMaster, taskStorageQueryAdapter, indexerMetadataStorageAdapter, req, workerTaskRunnerQueryAdapter, configManager, provisioningStrategy);
    final Response response = overlordResource.getTotalWorkerCapacity();
    Assert.assertEquals(HttpResponseStatus.OK.getCode(), response.getStatus());
    Assert.assertEquals(workerInfos.stream().findFirst().get().getWorker().getCapacity(), ((TotalWorkerCapacityResponse) response.getEntity()).getCurrentClusterCapacity());
    Assert.assertEquals(invalidExpectedCapacity, ((TotalWorkerCapacityResponse) response.getEntity()).getMaximumCapacityWithAutoScale());
}