Examples with AmazonAutoScalingClient com.sequenceiq.cloudbreak.cloud.aws.client.AmazonAutoScalingClient used on opensource projects

Example 11 with AmazonAutoScalingClient

use of com.sequenceiq.cloudbreak.cloud.aws.client.AmazonAutoScalingClient in project cloudbreak by hortonworks.

the class AwsCloudFormationSetup method scalingPrerequisites.

@Override
public void scalingPrerequisites(AuthenticatedContext ac, CloudStack stack, boolean upscale) {
    if (!upscale) {
        return;
    }
    String regionName = ac.getCloudContext().getLocation().getRegion().value();
    AwsCredentialView awsCredential = new AwsCredentialView(ac.getCloudCredential());
    AmazonCloudFormationClient cloudFormationClient = awsClient.createCloudFormationClient(awsCredential, regionName);
    AmazonAutoScalingClient amazonASClient = awsClient.createAutoScalingClient(awsCredential, regionName);
    List<Group> groups = stack.getGroups().stream().filter(g -> g.getInstances().stream().anyMatch(inst -> InstanceStatus.CREATE_REQUESTED == inst.getTemplate().getStatus())).collect(Collectors.toList());
    Map<String, Group> groupMap = groups.stream().collect(Collectors.toMap(g -> cfStackUtil.getAutoscalingGroupName(ac, cloudFormationClient, g.getName()), g -> g));
    DescribeAutoScalingGroupsResult result = amazonASClient.describeAutoScalingGroups(new DescribeAutoScalingGroupsRequest().withAutoScalingGroupNames(groupMap.keySet()));
    for (AutoScalingGroup asg : result.getAutoScalingGroups()) {
        Group group = groupMap.get(asg.getAutoScalingGroupName());
        List<CloudInstance> groupInstances = group.getInstances().stream().filter(inst -> InstanceStatus.CREATED.equals(inst.getTemplate().getStatus())).collect(Collectors.toList());
        List<CloudVmInstanceStatus> instanceStatuses = instanceConnector.check(ac, groupInstances);
        if (checkInstanceStatuses(instanceStatuses, ac.getCloudCredential().getAccountId())) {
            String errorMessage = "Not all the existing instances are in allowed state, upscale is not possible!";
            LOGGER.info(errorMessage);
            throw new CloudConnectorException(errorMessage);
        }
        List<Instance> asgOnlyInstances = asg.getInstances().stream().filter(inst -> groupInstances.stream().noneMatch(gi -> gi.getInstanceId().equals(inst.getInstanceId()))).collect(Collectors.toList());
        List<CloudInstance> cbOnlyInstances = groupInstances.stream().filter(gi -> asg.getInstances().stream().noneMatch(inst -> inst.getInstanceId().equals(gi.getInstanceId()))).collect(Collectors.toList());
        if (!asgOnlyInstances.isEmpty() || !cbOnlyInstances.isEmpty()) {
            String errorMessage = "The instances in the autoscaling group are not in sync with the instances in cloudbreak! Cloudbreak only instances: [" + cbOnlyInstances.stream().map(CloudInstance::getInstanceId).collect(Collectors.joining(",")) + "], AWS only instances: [" + asgOnlyInstances.stream().map(Instance::getInstanceId).collect(Collectors.joining(",")) + "]. Upscale is not possible!";
            LOGGER.info(errorMessage);
            throw new CloudConnectorException(errorMessage);
        }
        if (groupInstances.size() != asg.getDesiredCapacity()) {
            String errorMessage = String.format("The autoscale group's desired instance count is not match with the instance count in the cloudbreak." + " Desired count: %d <> cb instance count: %d. Upscale is not possible!", asg.getDesiredCapacity(), groupInstances.size());
            LOGGER.info(errorMessage);
            throw new CloudConnectorException(errorMessage);
        }
    }
}

Example 12 with AmazonAutoScalingClient

use of com.sequenceiq.cloudbreak.cloud.aws.client.AmazonAutoScalingClient in project cloudbreak by hortonworks.

the class AwsUpscaleServiceTest method upscaleTest.

@Test
void upscaleTest() throws AmazonAutoscalingFailed {
    AmazonAutoScalingClient amazonAutoScalingClient = mock(AmazonAutoScalingClient.class);
    AmazonCloudFormationClient amazonCloudFormationClient = mock(AmazonCloudFormationClient.class);
    DescribeAutoScalingGroupsResult describeAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult().withAutoScalingGroups(newAutoScalingGroup("masterASG", List.of("i-master1", "i-master2")), newAutoScalingGroup("workerASG", List.of("i-worker1", "i-worker2", "i-worker3")));
    when(amazonAutoScalingClient.describeAutoScalingGroups(any(DescribeAutoScalingGroupsRequest.class))).thenReturn(describeAutoScalingGroupsResult);
    when(awsClient.createAutoScalingClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonAutoScalingClient);
    when(awsClient.createCloudFormationClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonCloudFormationClient);
    when(awsClient.createEc2Client(any(), any())).thenReturn(mock(AmazonEc2Client.class));
    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationClient.class), eq("worker"))).thenReturn("workerASG");
    CloudContext cloudContext = CloudContext.Builder.builder().withId(1L).withName("teststack").withCrn("crn").withPlatform("AWS").withVariant("AWS").withLocation(Location.location(Region.region("eu-west-1"), AvailabilityZone.availabilityZone("eu-west-1a"))).withAccountId("1").build();
    AuthenticatedContext authenticatedContext = new AuthenticatedContext(cloudContext, new CloudCredential());
    List<CloudResource> allInstances = new ArrayList<>();
    allInstances.add(newInstanceResource("worker1", "worker", "i-worker1"));
    allInstances.add(newInstanceResource("worker2", "worker", "i-worker2"));
    allInstances.add(newInstanceResource("worker3", "worker", "i-worker3"));
    CloudResource workerInstance4 = newInstanceResource("worker4", "worker", "i-worker4");
    allInstances.add(workerInstance4);
    CloudResource workerInstance5 = newInstanceResource("worker5", "worker", "i-worker5");
    allInstances.add(workerInstance5);
    when(cfStackUtil.getInstanceCloudResources(eq(authenticatedContext), eq(amazonCloudFormationClient), eq(amazonAutoScalingClient), anyList())).thenReturn(allInstances);
    InstanceAuthentication instanceAuthentication = new InstanceAuthentication("sshkey", "", "cloudbreak");
    List<Group> groups = new ArrayList<>();
    groups.add(getMasterGroup(instanceAuthentication));
    Group worker = getWorkerGroup(instanceAuthentication);
    groups.add(worker);
    Map<String, String> tags = new HashMap<>();
    tags.put("owner", "cbuser");
    tags.put("created", "yesterday");
    CloudStack cloudStack = new CloudStack(groups, getNetwork(), null, emptyMap(), tags, null, instanceAuthentication, instanceAuthentication.getLoginUserName(), instanceAuthentication.getPublicKey(), null);
    List<CloudResource> cloudResourceList = Collections.emptyList();
    AdjustmentTypeWithThreshold adjustmentTypeWithThreshold = new AdjustmentTypeWithThreshold(AdjustmentType.EXACT, 0L);
    awsUpscaleService.upscale(authenticatedContext, cloudStack, cloudResourceList, adjustmentTypeWithThreshold);
    verify(awsAutoScalingService, times(1)).updateAutoscalingGroup(any(AmazonAutoScalingClient.class), eq("workerASG"), eq(5));
    verify(awsAutoScalingService, times(1)).scheduleStatusChecks(eq(List.of(worker)), eq(authenticatedContext), eq(amazonCloudFormationClient), any(), any());
    verify(awsAutoScalingService, times(1)).suspendAutoScaling(eq(authenticatedContext), eq(cloudStack));
    ArgumentCaptor<List<CloudResource>> captor = ArgumentCaptor.forClass(List.class);
    verify(awsComputeResourceService, times(1)).buildComputeResourcesForUpscale(eq(authenticatedContext), eq(cloudStack), anyList(), captor.capture(), any(), any(), eq(adjustmentTypeWithThreshold));
    verify(awsTaggingService, times(1)).tagRootVolumes(eq(authenticatedContext), any(AmazonEc2Client.class), eq(allInstances), eq(tags));
    verify(awsCloudWatchService, times(1)).addCloudWatchAlarmsForSystemFailures(any(), eq("eu-west-1"), any(AwsCredentialView.class));
    List<CloudResource> newInstances = captor.getValue();
    assertEquals("Two new instances should be created", 2, newInstances.size());
    assertThat(newInstances, hasItem(workerInstance4));
    assertThat(newInstances, hasItem(workerInstance5));
    verify(cfStackUtil, times(0)).addLoadBalancerTargets(any(), any(), any());
}

Example 13 with AmazonAutoScalingClient

use of com.sequenceiq.cloudbreak.cloud.aws.client.AmazonAutoScalingClient in project cloudbreak by hortonworks.

the class AwsUpscaleServiceTest method upscaleWithLoadBalancers.

@Test
void upscaleWithLoadBalancers() {
    AmazonAutoScalingClient amazonAutoScalingClient = mock(AmazonAutoScalingClient.class);
    AmazonCloudFormationClient amazonCloudFormationClient = mock(AmazonCloudFormationClient.class);
    when(amazonAutoScalingClient.describeAutoScalingGroups(any(DescribeAutoScalingGroupsRequest.class))).thenReturn(new DescribeAutoScalingGroupsResult().withAutoScalingGroups(newAutoScalingGroup("masterASG", List.of("i-master1", "i-master2")), newAutoScalingGroup("workerASG", List.of("i-worker1", "i-worker2", "i-worker3"))));
    when(awsClient.createAutoScalingClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonAutoScalingClient);
    when(awsClient.createCloudFormationClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonCloudFormationClient);
    when(awsClient.createEc2Client(any(), any())).thenReturn(mock(AmazonEc2Client.class));
    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationClient.class), eq("worker"))).thenReturn("workerASG");
    CloudContext cloudContext = CloudContext.Builder.builder().withId(1L).withName("teststack").withCrn("crn").withPlatform("AWS").withVariant("AWS").withLocation(Location.location(Region.region("eu-west-1"), AvailabilityZone.availabilityZone("eu-west-1a"))).withAccountId("1").build();
    AuthenticatedContext authenticatedContext = new AuthenticatedContext(cloudContext, new CloudCredential());
    List<CloudResource> allInstances = List.of(newInstanceResource("worker1", "worker", "i-worker1"), newInstanceResource("worker2", "worker", "i-worker2"), newInstanceResource("worker3", "worker", "i-worker3"), newInstanceResource("worker4", "worker", "i-worker4"), newInstanceResource("worker5", "worker", "i-worker5"));
    when(cfStackUtil.getInstanceCloudResources(eq(authenticatedContext), eq(amazonCloudFormationClient), eq(amazonAutoScalingClient), anyList())).thenReturn(allInstances);
    doNothing().when(cfStackUtil).addLoadBalancerTargets(any(), any(), any());
    InstanceAuthentication instanceAuthentication = new InstanceAuthentication("sshkey", "", "cloudbreak");
    List<Group> groups = new ArrayList<>();
    List<CloudLoadBalancer> loadBalancers = List.of(new CloudLoadBalancer(LoadBalancerType.PRIVATE), new CloudLoadBalancer(LoadBalancerType.PUBLIC));
    Group master = getMasterGroup(instanceAuthentication);
    groups.add(master);
    Group worker = getWorkerGroup(instanceAuthentication);
    groups.add(worker);
    Map<String, String> tags = new HashMap<>();
    tags.put("owner", "cbuser");
    tags.put("created", "yesterday");
    CloudStack cloudStack = new CloudStack(groups, getNetwork(), null, emptyMap(), tags, null, instanceAuthentication, instanceAuthentication.getLoginUserName(), instanceAuthentication.getPublicKey(), null, loadBalancers);
    List<CloudResource> cloudResourceList = Collections.emptyList();
    AdjustmentTypeWithThreshold adjustmentTypeWithThreshold = new AdjustmentTypeWithThreshold(AdjustmentType.EXACT, 0L);
    awsUpscaleService.upscale(authenticatedContext, cloudStack, cloudResourceList, adjustmentTypeWithThreshold);
    verify(cfStackUtil, times(2)).addLoadBalancerTargets(any(), any(), any());
}

Example 14 with AmazonAutoScalingClient

use of com.sequenceiq.cloudbreak.cloud.aws.client.AmazonAutoScalingClient in project cloudbreak by hortonworks.

the class AwsUpscaleServiceTest method upscaleAwsVolumeFail.

@Test
void upscaleAwsVolumeFail() throws AmazonAutoscalingFailed {
    AmazonAutoScalingClient amazonAutoScalingClient = mock(AmazonAutoScalingClient.class);
    AmazonCloudFormationClient amazonCloudFormationClient = mock(AmazonCloudFormationClient.class);
    DescribeAutoScalingGroupsRequest request = new DescribeAutoScalingGroupsRequest();
    request.setAutoScalingGroupNames(new ArrayList<>(Collections.singletonList("workerASG")));
    DescribeAutoScalingGroupsResult describeScaledAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult().withAutoScalingGroups(newAutoScalingGroup("workerASG", List.of("i-worker1", "i-worker2", "i-worker3")));
    when(amazonAutoScalingClient.describeAutoScalingGroups(eq(request))).thenReturn(describeScaledAutoScalingGroupsResult);
    when(awsClient.createAutoScalingClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonAutoScalingClient);
    when(awsClient.createCloudFormationClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonCloudFormationClient);
    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationClient.class), eq("worker"))).thenReturn("workerASG");
    CloudContext cloudContext = CloudContext.Builder.builder().withId(1L).withName("teststack").withCrn("crn").withPlatform("AWS").withVariant("AWS").withLocation(Location.location(Region.region("eu-west-1"), AvailabilityZone.availabilityZone("eu-west-1a"))).withAccountId("1").build();
    AuthenticatedContext authenticatedContext = new AuthenticatedContext(cloudContext, new CloudCredential());
    List<CloudResource> allInstances = new ArrayList<>();
    allInstances.add(newInstanceResource("worker1", "worker", "i-worker1"));
    allInstances.add(newInstanceResource("worker2", "worker", "i-worker2"));
    allInstances.add(newInstanceResource("worker3", "worker", "i-worker3"));
    CloudResource workerInstance4 = newInstanceResource("worker4", "worker", "i-worker4");
    allInstances.add(workerInstance4);
    CloudResource workerInstance5 = newInstanceResource("worker5", "worker", "i-worker5");
    allInstances.add(workerInstance5);
    when(cfStackUtil.getInstanceCloudResources(eq(authenticatedContext), eq(amazonCloudFormationClient), eq(amazonAutoScalingClient), anyList())).thenReturn(allInstances);
    InstanceAuthentication instanceAuthentication = new InstanceAuthentication("sshkey", "", "cloudbreak");
    List<Group> groups = new ArrayList<>();
    groups.add(getMasterGroup(instanceAuthentication));
    Group worker = getWorkerGroup(instanceAuthentication);
    groups.add(worker);
    CloudStack cloudStack = new CloudStack(groups, getNetwork(), null, emptyMap(), emptyMap(), null, instanceAuthentication, instanceAuthentication.getLoginUserName(), instanceAuthentication.getPublicKey(), null);
    List<CloudResource> cloudResourceList = Collections.emptyList();
    AutoScalingGroup newWorkerASGroup = newAutoScalingGroup("workerASG", List.of("i-worker1", "i-worker2", "i-worker3", "i-worker4", "i-worker5"));
    when(awsAutoScalingService.getAutoscalingGroups(eq(amazonAutoScalingClient), any())).thenReturn(List.of(newWorkerASGroup));
    AdjustmentTypeWithThreshold adjustmentTypeWithThreshold = new AdjustmentTypeWithThreshold(AdjustmentType.EXACT, 0L);
    when(awsComputeResourceService.buildComputeResourcesForUpscale(any(), any(), anyList(), anyList(), anyList(), anyList(), eq(adjustmentTypeWithThreshold))).thenThrow(new CloudConnectorException("volume create error"));
    CloudConnectorException exception = assertThrows(CloudConnectorException.class, () -> awsUpscaleService.upscale(authenticatedContext, cloudStack, cloudResourceList, adjustmentTypeWithThreshold));
    Assertions.assertEquals("Failed to create some resource on AWS for upscaled nodes, please check your quotas on AWS. " + "Original autoscaling group state has been recovered. Exception: volume create error", exception.getMessage());
    verify(awsAutoScalingService, times(1)).updateAutoscalingGroup(any(AmazonAutoScalingClient.class), eq("workerASG"), eq(5));
    verify(awsAutoScalingService, times(1)).scheduleStatusChecks(eq(List.of(worker)), eq(authenticatedContext), eq(amazonCloudFormationClient), any(), any());
    verify(awsComputeResourceService, times(1)).buildComputeResourcesForUpscale(eq(authenticatedContext), eq(cloudStack), anyList(), anyList(), any(), any(), eq(adjustmentTypeWithThreshold));
    verify(awsAutoScalingService, times(2)).suspendAutoScaling(eq(authenticatedContext), eq(cloudStack));
    verify(awsAutoScalingService, times(1)).terminateInstance(eq(amazonAutoScalingClient), eq("i-worker4"));
    verify(awsAutoScalingService, times(1)).terminateInstance(eq(amazonAutoScalingClient), eq("i-worker5"));
    verify(cfStackUtil, times(0)).addLoadBalancerTargets(any(), any(), any());
}

Example 15 with AmazonAutoScalingClient

use of com.sequenceiq.cloudbreak.cloud.aws.client.AmazonAutoScalingClient in project cloudbreak by hortonworks.

the class AwsUpscaleServiceTest method upscaleAwsASGroupFail.

@Test
void upscaleAwsASGroupFail() throws AmazonAutoscalingFailed {
    AmazonAutoScalingClient amazonAutoScalingClient = mock(AmazonAutoScalingClient.class);
    AmazonCloudFormationClient amazonCloudFormationClient = mock(AmazonCloudFormationClient.class);
    DescribeAutoScalingGroupsRequest request = new DescribeAutoScalingGroupsRequest();
    request.setAutoScalingGroupNames(List.of("workerASG"));
    DescribeAutoScalingGroupsResult describeScaledAutoScalingGroupsResult = new DescribeAutoScalingGroupsResult().withAutoScalingGroups(newAutoScalingGroup("workerASG", List.of("i-worker1", "i-worker2", "i-worker3")));
    when(amazonAutoScalingClient.describeAutoScalingGroups(eq(request))).thenReturn(describeScaledAutoScalingGroupsResult);
    when(awsClient.createAutoScalingClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonAutoScalingClient);
    when(awsClient.createCloudFormationClient(any(AwsCredentialView.class), anyString())).thenReturn(amazonCloudFormationClient);
    when(cfStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationClient.class), eq("worker"))).thenReturn("workerASG");
    CloudContext cloudContext = CloudContext.Builder.builder().withId(1L).withName("teststack").withCrn("crn").withPlatform("AWS").withVariant("AWS").withLocation(Location.location(Region.region("eu-west-1"), AvailabilityZone.availabilityZone("eu-west-1a"))).withAccountId("1").build();
    AuthenticatedContext authenticatedContext = new AuthenticatedContext(cloudContext, new CloudCredential());
    InstanceAuthentication instanceAuthentication = new InstanceAuthentication("sshkey", "", "cloudbreak");
    List<Group> groups = new ArrayList<>();
    groups.add(getMasterGroup(instanceAuthentication));
    Group worker = getWorkerGroup(instanceAuthentication);
    groups.add(worker);
    CloudStack cloudStack = new CloudStack(groups, getNetwork(), null, emptyMap(), emptyMap(), null, instanceAuthentication, instanceAuthentication.getLoginUserName(), instanceAuthentication.getPublicKey(), null);
    List<CloudResource> cloudResourceList = Collections.emptyList();
    AutoScalingGroup newWorkerASGroup = newAutoScalingGroup("workerASG", List.of("i-worker1", "i-worker2", "i-worker3", "i-worker4", "i-worker5"));
    when(awsAutoScalingService.getAutoscalingGroups(eq(amazonAutoScalingClient), any())).thenReturn(Collections.singletonList(newWorkerASGroup));
    doThrow(new AmazonAutoscalingFailed("autoscaling failed")).when(awsAutoScalingService).scheduleStatusChecks(eq(List.of(worker)), eq(authenticatedContext), eq(amazonCloudFormationClient), any(Date.class), any());
    AdjustmentTypeWithThreshold adjustmentTypeWithThreshold = new AdjustmentTypeWithThreshold(AdjustmentType.EXACT, 0L);
    CloudConnectorException exception = assertThrows(CloudConnectorException.class, () -> awsUpscaleService.upscale(authenticatedContext, cloudStack, cloudResourceList, adjustmentTypeWithThreshold));
    Assertions.assertEquals("Autoscaling group update failed: Amazon Autoscaling Group was not able to reach the desired state (3 instances instead of 5). " + "Original autoscaling group state has been recovered. Failure reason: autoscaling failed", exception.getMessage());
    verify(awsAutoScalingService, times(1)).updateAutoscalingGroup(any(AmazonAutoScalingClient.class), eq("workerASG"), eq(5));
    verify(awsAutoScalingService, times(1)).scheduleStatusChecks(eq(List.of(worker)), eq(authenticatedContext), eq(amazonCloudFormationClient), any(), any());
    verify(awsComputeResourceService, times(0)).buildComputeResourcesForUpscale(eq(authenticatedContext), eq(cloudStack), anyList(), anyList(), any(), any(), eq(adjustmentTypeWithThreshold));
    verify(awsAutoScalingService, times(1)).suspendAutoScaling(eq(authenticatedContext), eq(cloudStack));
    verify(awsAutoScalingService, times(1)).terminateInstance(eq(amazonAutoScalingClient), eq("i-worker4"));
    verify(awsAutoScalingService, times(1)).terminateInstance(eq(amazonAutoScalingClient), eq("i-worker5"));
}