Examples with AutoScalingData org.apache.druid.indexing.overlord.autoscaling.AutoScalingData used on opensource projects

Example 6 with AutoScalingData

use of org.apache.druid.indexing.overlord.autoscaling.AutoScalingData in project druid by druid-io.

the class GceAutoScalerTest method testProvision.

@Test
public void testProvision() throws IOException, GeneralSecurityException, GceServiceException {
    GceAutoScaler autoScaler = EasyMock.createMockBuilder(GceAutoScaler.class).withConstructor(int.class, int.class, GceEnvironmentConfig.class).withArgs(2, 4, new GceEnvironmentConfig(1, "proj-x", "us-central-1", "druid-mig")).addMockedMethod("createComputeServiceImpl").createMock();
    EasyMock.expect(autoScaler.createComputeServiceImpl()).andReturn(null);
    EasyMock.expect(autoScaler.createComputeServiceImpl()).andReturn(mockCompute);
    EasyMock.replay(autoScaler);
    // set up getRunningInstances results
    InstanceGroupManagersListManagedInstancesResponse beforeRunningInstance = createRunningInstances(Arrays.asList("http://xyz/foo", "http://xyz/bar"));
    InstanceGroupManagersListManagedInstancesResponse afterRunningInstance = createRunningInstances(Arrays.asList("http://xyz/foo", "http://xyz/bar", "http://xyz/baz"));
    // 1st call
    EasyMock.expect(mockInstancesRequest.execute()).andReturn(beforeRunningInstance);
    EasyMock.expect(mockInstancesRequest.setMaxResults(500L)).andReturn(mockInstancesRequest);
    // 2nd call
    EasyMock.expect(mockInstancesRequest.execute()).andReturn(afterRunningInstance);
    EasyMock.expect(mockInstancesRequest.setMaxResults(500L)).andReturn(mockInstancesRequest);
    EasyMock.replay(mockInstancesRequest);
    EasyMock.expect(mockInstanceGroupManagers.listManagedInstances("proj-x", "us-central-1", "druid-mig")).andReturn(mockInstancesRequest).times(2);
    // set up the resize operation
    Operation mockResponse = new Operation();
    mockResponse.setStatus("DONE");
    mockResponse.setError(new Operation.Error());
    EasyMock.expect(mockResizeRequest.execute()).andReturn(mockResponse);
    EasyMock.replay(mockResizeRequest);
    EasyMock.expect(mockInstanceGroupManagers.resize("proj-x", "us-central-1", "druid-mig", 3)).andReturn(mockResizeRequest);
    EasyMock.replay(mockInstanceGroupManagers);
    // called twice in getRunningInstances...
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    // ...and once in provision
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    // and that's all folks!
    EasyMock.replay(mockCompute);
    AutoScalingData autoScalingData = autoScaler.provision();
    Assert.assertEquals(1, autoScalingData.getNodeIds().size());
    Assert.assertEquals("baz", autoScalingData.getNodeIds().get(0));
    EasyMock.verify(mockCompute);
    EasyMock.verify(mockInstanceGroupManagers);
    EasyMock.verify(mockResizeRequest);
    EasyMock.verify(mockInstancesRequest);
}

Example 7 with AutoScalingData

use of org.apache.druid.indexing.overlord.autoscaling.AutoScalingData in project druid by druid-io.

the class GceAutoScalerTest method testProvisionSkipped.

@Test
public void testProvisionSkipped() throws IOException, GeneralSecurityException, GceServiceException {
    GceAutoScaler autoScaler = EasyMock.createMockBuilder(GceAutoScaler.class).withConstructor(int.class, int.class, GceEnvironmentConfig.class).withArgs(2, 4, new GceEnvironmentConfig(1, "proj-x", "us-central-1", "druid-mig")).addMockedMethod("createComputeServiceImpl").createMock();
    EasyMock.expect(autoScaler.createComputeServiceImpl()).andReturn(null);
    EasyMock.expect(autoScaler.createComputeServiceImpl()).andReturn(mockCompute);
    EasyMock.replay(autoScaler);
    // set up getRunningInstances results
    InstanceGroupManagersListManagedInstancesResponse beforeRunningInstance = createRunningInstances(Arrays.asList("http://xyz/foo", "http://xyz/bar", "http://xyz/baz", // already max instances, will not scale
    "http://xyz/zab"));
    EasyMock.expect(mockInstancesRequest.execute()).andReturn(beforeRunningInstance);
    EasyMock.expect(mockInstancesRequest.setMaxResults(500L)).andReturn(mockInstancesRequest);
    EasyMock.replay(mockInstancesRequest);
    EasyMock.expect(mockInstanceGroupManagers.listManagedInstances("proj-x", "us-central-1", "druid-mig")).andReturn(mockInstancesRequest);
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    EasyMock.replay(mockInstanceGroupManagers);
    // and that's all folks!
    EasyMock.replay(mockCompute);
    AutoScalingData autoScalingData = autoScaler.provision();
    Assert.assertEquals(0, autoScalingData.getNodeIds().size());
    EasyMock.verify(mockCompute);
    EasyMock.verify(mockInstancesRequest);
    EasyMock.verify(mockInstanceGroupManagers);
}

Example 8 with AutoScalingData

use of org.apache.druid.indexing.overlord.autoscaling.AutoScalingData in project druid by druid-io.

the class GceAutoScalerTest method testTerminateWithIdsWithMissingRemoval.

@Test
public void testTerminateWithIdsWithMissingRemoval() throws IOException, GeneralSecurityException, GceServiceException {
    GceAutoScaler autoScaler = EasyMock.createMockBuilder(GceAutoScaler.class).withConstructor(int.class, int.class, GceEnvironmentConfig.class).withArgs(2, 4, new GceEnvironmentConfig(1, "proj-x", "us-central-1", "druid-mig")).addMockedMethod("createComputeServiceImpl").createMock();
    EasyMock.expect(autoScaler.createComputeServiceImpl()).andReturn(null);
    EasyMock.expect(autoScaler.createComputeServiceImpl()).andReturn(mockCompute);
    EasyMock.replay(autoScaler);
    // set up getRunningInstances results
    InstanceGroupManagersListManagedInstancesResponse beforeRunningInstance = createRunningInstances(Arrays.asList("http://xyz/foo", "http://xyz/bar", "http://xyz/baz"));
    InstanceGroupManagersListManagedInstancesResponse after1RunningInstance = createRunningInstances(Arrays.asList("http://xyz/foo", "http://xyz/bar", "http://xyz/baz"));
    // not changing anything, will trigger the loop around getRunningInstances
    InstanceGroupManagersListManagedInstancesResponse after2RunningInstance = createRunningInstances(Arrays.asList("http://xyz/foo", "http://xyz/bar"));
    // now the machine got dropped!
    // 1st call
    EasyMock.expect(mockInstancesRequest.execute()).andReturn(beforeRunningInstance);
    EasyMock.expect(mockInstancesRequest.setMaxResults(500L)).andReturn(mockInstancesRequest);
    // 2nd call, the next is needed
    EasyMock.expect(mockInstancesRequest.execute()).andReturn(after1RunningInstance);
    EasyMock.expect(mockInstancesRequest.setMaxResults(500L)).andReturn(mockInstancesRequest);
    // 3rd call, this unblocks
    EasyMock.expect(mockInstancesRequest.execute()).andReturn(after2RunningInstance);
    EasyMock.expect(mockInstancesRequest.setMaxResults(500L)).andReturn(mockInstancesRequest);
    EasyMock.replay(mockInstancesRequest);
    EasyMock.expect(mockInstanceGroupManagers.listManagedInstances("proj-x", "us-central-1", "druid-mig")).andReturn(mockInstancesRequest).times(3);
    // set up the delete operation
    Operation mockResponse = new Operation();
    mockResponse.setStatus("DONE");
    mockResponse.setError(new Operation.Error());
    EasyMock.expect(mockDeleteRequest.execute()).andReturn(mockResponse);
    EasyMock.replay(mockDeleteRequest);
    InstanceGroupManagersDeleteInstancesRequest requestBody = new InstanceGroupManagersDeleteInstancesRequest();
    requestBody.setInstances(Collections.singletonList("zones/us-central-1/instances/baz"));
    EasyMock.expect(mockInstanceGroupManagers.deleteInstances("proj-x", "us-central-1", "druid-mig", requestBody)).andReturn(mockDeleteRequest);
    EasyMock.replay(mockInstanceGroupManagers);
    // called three times in getRunningInstances...
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    // ...and once in terminateWithIds
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    // and that's all folks!
    EasyMock.replay(mockCompute);
    AutoScalingData autoScalingData = autoScaler.terminateWithIds(Collections.singletonList("baz"));
    Assert.assertEquals(1, autoScalingData.getNodeIds().size());
    Assert.assertEquals("baz", autoScalingData.getNodeIds().get(0));
    EasyMock.verify(mockCompute);
    EasyMock.verify(mockInstanceGroupManagers);
    EasyMock.verify(mockDeleteRequest);
    EasyMock.verify(mockInstancesRequest);
}

Example 9 with AutoScalingData

use of org.apache.druid.indexing.overlord.autoscaling.AutoScalingData in project druid by druid-io.

the class GceAutoScalerTest method testProvisionWithMissingNewInstances.

@Test
public void testProvisionWithMissingNewInstances() throws IOException, GeneralSecurityException, GceServiceException {
    GceAutoScaler autoScaler = EasyMock.createMockBuilder(GceAutoScaler.class).withConstructor(int.class, int.class, GceEnvironmentConfig.class).withArgs(2, 4, new GceEnvironmentConfig(1, "proj-x", "us-central-1", "druid-mig")).addMockedMethod("createComputeServiceImpl").createMock();
    EasyMock.expect(autoScaler.createComputeServiceImpl()).andReturn(null);
    EasyMock.expect(autoScaler.createComputeServiceImpl()).andReturn(mockCompute);
    EasyMock.replay(autoScaler);
    // set up getRunningInstances results
    InstanceGroupManagersListManagedInstancesResponse beforeRunningInstance = createRunningInstances(Arrays.asList("http://xyz/foo", "http://xyz/bar"));
    InstanceGroupManagersListManagedInstancesResponse after1RunningInstance = createRunningInstances(Arrays.asList("http://xyz/foo", "http://xyz/bar"));
    // not changing anything, will trigger the loop around getRunningInstances
    InstanceGroupManagersListManagedInstancesResponse after2RunningInstance = createRunningInstances(Arrays.asList("http://xyz/foo", "http://xyz/bar", "http://xyz/baz"));
    // now the new machine is here!
    // 1st call
    EasyMock.expect(mockInstancesRequest.execute()).andReturn(beforeRunningInstance);
    EasyMock.expect(mockInstancesRequest.setMaxResults(500L)).andReturn(mockInstancesRequest);
    // 2nd call, the next is needed
    EasyMock.expect(mockInstancesRequest.execute()).andReturn(after1RunningInstance);
    EasyMock.expect(mockInstancesRequest.setMaxResults(500L)).andReturn(mockInstancesRequest);
    // 3rd call, this unblocks
    EasyMock.expect(mockInstancesRequest.execute()).andReturn(after2RunningInstance);
    EasyMock.expect(mockInstancesRequest.setMaxResults(500L)).andReturn(mockInstancesRequest);
    EasyMock.replay(mockInstancesRequest);
    EasyMock.expect(mockInstanceGroupManagers.listManagedInstances("proj-x", "us-central-1", "druid-mig")).andReturn(mockInstancesRequest).times(3);
    // set up the resize operation
    Operation mockResponse = new Operation();
    mockResponse.setStatus("DONE");
    mockResponse.setError(new Operation.Error());
    EasyMock.expect(mockResizeRequest.execute()).andReturn(mockResponse);
    EasyMock.replay(mockResizeRequest);
    EasyMock.expect(mockInstanceGroupManagers.resize("proj-x", "us-central-1", "druid-mig", 3)).andReturn(mockResizeRequest);
    EasyMock.replay(mockInstanceGroupManagers);
    // called three times in getRunningInstances...
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    // ...and once in provision
    EasyMock.expect(mockCompute.instanceGroupManagers()).andReturn(mockInstanceGroupManagers);
    // and that's all folks!
    EasyMock.replay(mockCompute);
    AutoScalingData autoScalingData = autoScaler.provision();
    Assert.assertEquals(1, autoScalingData.getNodeIds().size());
    Assert.assertEquals("baz", autoScalingData.getNodeIds().get(0));
    EasyMock.verify(mockCompute);
    EasyMock.verify(mockInstanceGroupManagers);
    EasyMock.verify(mockResizeRequest);
    EasyMock.verify(mockInstancesRequest);
}

Example 10 with AutoScalingData

use of org.apache.druid.indexing.overlord.autoscaling.AutoScalingData in project druid by druid-io.

the class EC2AutoScaler method terminate.

@Override
public AutoScalingData terminate(List<String> ips) {
    if (ips.isEmpty()) {
        return new AutoScalingData(new ArrayList<>());
    }
    DescribeInstancesResult result = amazonEC2Client.describeInstances(new DescribeInstancesRequest().withFilters(new Filter("private-ip-address", ips)));
    List<Instance> instances = new ArrayList<>();
    for (Reservation reservation : result.getReservations()) {
        instances.addAll(reservation.getInstances());
    }
    try {
        return terminateWithIds(Lists.transform(instances, new Function<Instance, String>() {

            @Override
            public String apply(Instance input) {
                return input.getInstanceId();
            }
        }));
    } catch (Exception e) {
        log.error(e, "Unable to terminate any instances.");
    }
    return null;
}