Search in sources :

Example 46 with CloudLoadBalancer

use of com.sequenceiq.cloudbreak.cloud.model.CloudLoadBalancer in project cloudbreak by hortonworks.

the class AwsUpscaleService method upscale.

public List<CloudResourceStatus> upscale(AuthenticatedContext ac, CloudStack stack, List<CloudResource> resources, AdjustmentTypeWithThreshold adjustmentTypeWithThreshold) {
    LOGGER.info("Upscale AWS cluster with adjustment and threshold: {}. Resources: {}", adjustmentTypeWithThreshold, resources);
    String regionName = ac.getCloudContext().getLocation().getRegion().value();
    AwsCredentialView credentialView = new AwsCredentialView(ac.getCloudCredential());
    AmazonCloudFormationClient cloudFormationClient = awsClient.createCloudFormationClient(credentialView, regionName);
    AmazonAutoScalingClient amazonASClient = awsClient.createAutoScalingClient(credentialView, regionName);
    AmazonEc2Client amazonEC2Client = awsClient.createEc2Client(credentialView, regionName);
    List<Group> scaledGroups = cloudResourceHelper.getScaledGroups(stack);
    Map<String, Group> desiredAutoscalingGroupsByName = getAutoScaleGroupsByNameFromCloudFormationTemplate(ac, cloudFormationClient, scaledGroups);
    LOGGER.info("Desired autoscaling groups: {}", desiredAutoscalingGroupsByName);
    awsAutoScalingService.resumeAutoScaling(amazonASClient, desiredAutoscalingGroupsByName.keySet(), UPSCALE_PROCESSES);
    Map<String, Integer> originalAutoScalingGroupsBySize = getAutoScalingGroupsBySize(desiredAutoscalingGroupsByName.keySet(), amazonASClient);
    LOGGER.info("Update autoscaling groups for stack: {}", ac.getCloudContext().getName());
    Date timeBeforeASUpdate = new Date();
    updateAutoscalingGroups(amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize);
    List<String> knownInstances = getKnownInstancesByCloudbreak(stack);
    try {
        awsAutoScalingService.scheduleStatusChecks(scaledGroups, ac, cloudFormationClient, timeBeforeASUpdate, knownInstances);
    } catch (AmazonAutoscalingFailed amazonAutoscalingFailed) {
        LOGGER.info("Amazon autoscaling group update failed", amazonAutoscalingFailed);
        recoverOriginalState(ac, stack, amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize, amazonAutoscalingFailed);
        sendASGUpdateFailedMessage(amazonASClient, desiredAutoscalingGroupsByName, amazonAutoscalingFailed);
    }
    try {
        awsAutoScalingService.suspendAutoScaling(ac, stack);
        validateInstanceStatusesInScaledGroups(ac, amazonASClient, amazonEC2Client, cloudFormationClient, scaledGroups);
        List<CloudResource> instances = cfStackUtil.getInstanceCloudResources(ac, cloudFormationClient, amazonASClient, scaledGroups);
        associateElasticIpWithNewInstances(stack, resources, cloudFormationClient, amazonEC2Client, scaledGroups, instances);
        List<Group> groupsWithNewInstances = getGroupsWithNewInstances(scaledGroups);
        List<CloudResource> newInstances = getNewInstances(scaledGroups, instances);
        List<CloudResource> reattachableVolumeSets = getReattachableVolumeSets(scaledGroups, resources);
        List<CloudResource> networkResources = resources.stream().filter(cloudResource -> ResourceType.AWS_SUBNET.equals(cloudResource.getType())).collect(Collectors.toList());
        List<CloudResourceStatus> cloudResourceStatuses = awsComputeResourceService.buildComputeResourcesForUpscale(ac, stack, groupsWithNewInstances, newInstances, reattachableVolumeSets, networkResources, adjustmentTypeWithThreshold);
        List<String> failedResources = cloudResourceStatuses.stream().map(CloudResourceStatus::getCloudResource).filter(cloudResource -> CommonStatus.FAILED == cloudResource.getStatus()).map(cloudResource -> cloudResource.getType() + " - " + cloudResource.getName()).collect(Collectors.toList());
        if (!failedResources.isEmpty()) {
            throw new RuntimeException("Additional resource creation failed: " + failedResources);
        }
        awsTaggingService.tagRootVolumes(ac, amazonEC2Client, instances, stack.getTags());
        awsCloudWatchService.addCloudWatchAlarmsForSystemFailures(instances, regionName, credentialView);
        for (CloudLoadBalancer loadBalancer : stack.getLoadBalancers()) {
            cfStackUtil.addLoadBalancerTargets(ac, loadBalancer, newInstances);
        }
        List<CloudResourceStatus> successfulInstances = newInstances.stream().map(cloudResource -> new CloudResourceStatus(cloudResource, ResourceStatus.CREATED, cloudResource.getParameter(CloudResource.PRIVATE_ID, Long.class))).collect(Collectors.toList());
        return ListUtils.union(singletonList(new CloudResourceStatus(cfStackUtil.getCloudFormationStackResource(resources), ResourceStatus.UPDATED)), successfulInstances);
    } catch (RuntimeException runtimeException) {
        recoverOriginalState(ac, stack, amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize, runtimeException);
        throw new CloudConnectorException(String.format("Failed to create some resource on AWS for upscaled nodes, please check your quotas on AWS. " + "Original autoscaling group state has been recovered. Exception: %s", runtimeException.getMessage()), runtimeException);
    }
}
Also used : EntitlementService(com.sequenceiq.cloudbreak.auth.altus.EntitlementService) Date(java.util.Date) LoggerFactory(org.slf4j.LoggerFactory) CloudInstance(com.sequenceiq.cloudbreak.cloud.model.CloudInstance) AwsMetadataCollector(com.sequenceiq.cloudbreak.cloud.aws.AwsMetadataCollector) Collections.singletonList(java.util.Collections.singletonList) ArrayList(java.util.ArrayList) AwsNetworkService(com.sequenceiq.cloudbreak.cloud.aws.common.connector.resource.AwsNetworkService) Inject(javax.inject.Inject) CloudConnectorException(com.sequenceiq.cloudbreak.cloud.exception.CloudConnectorException) AmazonCloudFormationClient(com.sequenceiq.cloudbreak.cloud.aws.client.AmazonCloudFormationClient) AwsNetworkView(com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsNetworkView) AmazonEc2Client(com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEc2Client) AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView) AuthenticatedContext(com.sequenceiq.cloudbreak.cloud.context.AuthenticatedContext) Service(org.springframework.stereotype.Service) Map(java.util.Map) ListUtils(org.apache.commons.collections4.ListUtils) CloudLoadBalancer(com.sequenceiq.cloudbreak.cloud.model.CloudLoadBalancer) AwsElasticIpService(com.sequenceiq.cloudbreak.cloud.aws.common.connector.resource.AwsElasticIpService) CommonStatus(com.sequenceiq.common.api.type.CommonStatus) ResourceStatus(com.sequenceiq.cloudbreak.cloud.model.ResourceStatus) CloudResourceHelper(com.sequenceiq.cloudbreak.cloud.transform.CloudResourceHelper) ResourceType(com.sequenceiq.common.api.type.ResourceType) Logger(org.slf4j.Logger) AmazonServiceException(com.amazonaws.AmazonServiceException) AutoScalingGroup(com.amazonaws.services.autoscaling.model.AutoScalingGroup) AwsCloudFormationClient(com.sequenceiq.cloudbreak.cloud.aws.AwsCloudFormationClient) CloudResource(com.sequenceiq.cloudbreak.cloud.model.CloudResource) AdjustmentTypeWithThreshold(com.sequenceiq.common.api.adjustment.AdjustmentTypeWithThreshold) CloudFormationStackUtil(com.sequenceiq.cloudbreak.cloud.aws.CloudFormationStackUtil) CloudResourceStatus(com.sequenceiq.cloudbreak.cloud.model.CloudResourceStatus) Set(java.util.Set) Collectors(java.util.stream.Collectors) DescribeAutoScalingGroupsRequest(com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsRequest) Objects(java.util.Objects) CloudStack(com.sequenceiq.cloudbreak.cloud.model.CloudStack) List(java.util.List) AmazonAutoScalingClient(com.sequenceiq.cloudbreak.cloud.aws.client.AmazonAutoScalingClient) Stream(java.util.stream.Stream) Group(com.sequenceiq.cloudbreak.cloud.model.Group) Entry(java.util.Map.Entry) Instance(com.amazonaws.services.autoscaling.model.Instance) AwsTaggingService(com.sequenceiq.cloudbreak.cloud.aws.common.AwsTaggingService) AutoScalingGroup(com.amazonaws.services.autoscaling.model.AutoScalingGroup) Group(com.sequenceiq.cloudbreak.cloud.model.Group) CloudConnectorException(com.sequenceiq.cloudbreak.cloud.exception.CloudConnectorException) Date(java.util.Date) AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView) AmazonAutoScalingClient(com.sequenceiq.cloudbreak.cloud.aws.client.AmazonAutoScalingClient) CloudLoadBalancer(com.sequenceiq.cloudbreak.cloud.model.CloudLoadBalancer) CloudResourceStatus(com.sequenceiq.cloudbreak.cloud.model.CloudResourceStatus) AmazonEc2Client(com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEc2Client) CloudResource(com.sequenceiq.cloudbreak.cloud.model.CloudResource) AmazonCloudFormationClient(com.sequenceiq.cloudbreak.cloud.aws.client.AmazonCloudFormationClient)

Example 47 with CloudLoadBalancer

use of com.sequenceiq.cloudbreak.cloud.model.CloudLoadBalancer in project cloudbreak by hortonworks.

the class CloudFormationStackUtil method removeLoadBalancerTargets.

public void removeLoadBalancerTargets(AuthenticatedContext ac, CloudLoadBalancer loadBalancer, List<CloudResource> resourcesToRemove) {
    String region = ac.getCloudContext().getLocation().getRegion().value();
    AmazonElasticLoadBalancingClient amazonElbClient = awsClient.createElasticLoadBalancingClient(new AwsCredentialView(ac.getCloudCredential()), region);
    for (Map.Entry<TargetGroupPortPair, Set<Group>> entry : loadBalancer.getPortToTargetGroupMapping().entrySet()) {
        // Get a list of the instance ids to remove
        Set<String> instancesToRemove = getInstanceIdsForGroups(resourcesToRemove, entry.getValue());
        // Find target group ARN
        AwsLoadBalancerScheme scheme = loadBalancerTypeConverter.convert(loadBalancer.getType());
        String targetGroupArn = getResourceArnByLogicalId(ac, AwsTargetGroup.getTargetGroupName(entry.getKey().getTrafficPort(), scheme), region);
        // Deregister any instances that no longer exist
        if (!instancesToRemove.isEmpty()) {
            try {
                List<TargetDescription> targetsToRemove = instancesToRemove.stream().map(instanceId -> new TargetDescription().withId(instanceId)).collect(Collectors.toList());
                DeregisterTargetsResult deregisterTargetsResult = amazonElbClient.deregisterTargets(new DeregisterTargetsRequest().withTargetGroupArn(targetGroupArn).withTargets(targetsToRemove));
            } catch (InvalidTargetException ignored) {
            // no-op - we tried to remove a target that wasn't in the target group, which is fine
            }
        }
    }
}
Also used : DeregisterTargetsRequest(com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest) RegisterTargetsRequest(com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsRequest) DescribeLoadBalancersRequest(com.amazonaws.services.elasticloadbalancingv2.model.DescribeLoadBalancersRequest) CloudInstance(com.sequenceiq.cloudbreak.cloud.model.CloudInstance) TargetHealthDescription(com.amazonaws.services.elasticloadbalancingv2.model.TargetHealthDescription) AmazonCloudFormationClient(com.sequenceiq.cloudbreak.cloud.aws.client.AmazonCloudFormationClient) AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView) AuthenticatedContext(com.sequenceiq.cloudbreak.cloud.context.AuthenticatedContext) DescribeInstancesRequest(com.amazonaws.services.ec2.model.DescribeInstancesRequest) Map(java.util.Map) CloudLoadBalancer(com.sequenceiq.cloudbreak.cloud.model.CloudLoadBalancer) AwsTargetGroup(com.sequenceiq.cloudbreak.cloud.aws.common.loadbalancer.AwsTargetGroup) Splitter(com.google.common.base.Splitter) CommonStatus(com.sequenceiq.common.api.type.CommonStatus) DescribeFileSystemsResult(com.amazonaws.services.elasticfilesystem.model.DescribeFileSystemsResult) AmazonServiceException(com.amazonaws.AmazonServiceException) DescribeTargetHealthResult(com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetHealthResult) Collection(java.util.Collection) Output(com.amazonaws.services.cloudformation.model.Output) Set(java.util.Set) Retry(com.sequenceiq.cloudbreak.service.Retry) Collectors(java.util.stream.Collectors) Backoff(org.springframework.retry.annotation.Backoff) Objects(java.util.Objects) List(java.util.List) AmazonAutoScalingClient(com.sequenceiq.cloudbreak.cloud.aws.client.AmazonAutoScalingClient) Group(com.sequenceiq.cloudbreak.cloud.model.Group) InvalidTargetException(com.amazonaws.services.elasticloadbalancingv2.model.InvalidTargetException) PRIVATE_ID(com.sequenceiq.cloudbreak.cloud.model.CloudResource.PRIVATE_ID) LoadBalancerTypeConverter(com.sequenceiq.cloudbreak.cloud.aws.common.loadbalancer.LoadBalancerTypeConverter) DescribeTargetHealthRequest(com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetHealthRequest) LoadBalancer(com.amazonaws.services.elasticloadbalancingv2.model.LoadBalancer) TargetGroupPortPair(com.sequenceiq.cloudbreak.cloud.model.TargetGroupPortPair) INSTANCE_TYPE(com.sequenceiq.cloudbreak.cloud.model.CloudResource.INSTANCE_TYPE) DescribeStacksRequest(com.amazonaws.services.cloudformation.model.DescribeStacksRequest) AmazonElasticLoadBalancingClient(com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonElasticLoadBalancingClient) AmazonEfsClient(com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEfsClient) Supplier(java.util.function.Supplier) DescribeLoadBalancersResult(com.amazonaws.services.elasticloadbalancingv2.model.DescribeLoadBalancersResult) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) Value(org.springframework.beans.factory.annotation.Value) CloudConnectorException(com.sequenceiq.cloudbreak.cloud.exception.CloudConnectorException) DeregisterTargetsResult(com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsResult) Service(org.springframework.stereotype.Service) DescribeAutoScalingGroupsResult(com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsResult) AwsLoadBalancerScheme(com.sequenceiq.cloudbreak.cloud.aws.common.loadbalancer.AwsLoadBalancerScheme) Retryable(org.springframework.retry.annotation.Retryable) DescribeStackResourceRequest(com.amazonaws.services.cloudformation.model.DescribeStackResourceRequest) TargetDescription(com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription) ResourceType(com.sequenceiq.common.api.type.ResourceType) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) FileSystemDescription(com.amazonaws.services.elasticfilesystem.model.FileSystemDescription) CloudResource(com.sequenceiq.cloudbreak.cloud.model.CloudResource) DescribeFileSystemsRequest(com.amazonaws.services.elasticfilesystem.model.DescribeFileSystemsRequest) RegisterTargetsResult(com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsResult) DescribeAutoScalingGroupsRequest(com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsRequest) SdkClientException(com.amazonaws.SdkClientException) LoggerFactory.getLogger(org.slf4j.LoggerFactory.getLogger) Instance(com.amazonaws.services.autoscaling.model.Instance) DescribeStackResourceResult(com.amazonaws.services.cloudformation.model.DescribeStackResourceResult) Collections(java.util.Collections) Set(java.util.Set) AmazonElasticLoadBalancingClient(com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonElasticLoadBalancingClient) DeregisterTargetsResult(com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsResult) TargetDescription(com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription) DeregisterTargetsRequest(com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest) AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView) InvalidTargetException(com.amazonaws.services.elasticloadbalancingv2.model.InvalidTargetException) Map(java.util.Map) TargetGroupPortPair(com.sequenceiq.cloudbreak.cloud.model.TargetGroupPortPair) AwsLoadBalancerScheme(com.sequenceiq.cloudbreak.cloud.aws.common.loadbalancer.AwsLoadBalancerScheme)

Aggregations

CloudLoadBalancer (com.sequenceiq.cloudbreak.cloud.model.CloudLoadBalancer)47 TargetGroupPortPair (com.sequenceiq.cloudbreak.cloud.model.TargetGroupPortPair)27 Group (com.sequenceiq.cloudbreak.cloud.model.Group)24 CloudStack (com.sequenceiq.cloudbreak.cloud.model.CloudStack)21 ArrayList (java.util.ArrayList)21 Test (org.junit.jupiter.api.Test)20 List (java.util.List)15 AwsNetworkView (com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsNetworkView)13 LoadBalancerType (com.sequenceiq.common.api.type.LoadBalancerType)13 HashMap (java.util.HashMap)13 ArgumentMatchers.anyString (org.mockito.ArgumentMatchers.anyString)13 CloudContext (com.sequenceiq.cloudbreak.cloud.context.CloudContext)12 Set (java.util.Set)12 InstanceAuthentication (com.sequenceiq.cloudbreak.cloud.model.InstanceAuthentication)11 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)11 CloudResource (com.sequenceiq.cloudbreak.cloud.model.CloudResource)10 GroupNetwork (com.sequenceiq.cloudbreak.cloud.model.GroupNetwork)10 Network (com.sequenceiq.cloudbreak.cloud.model.Network)10 Map (java.util.Map)10 CloudInstance (com.sequenceiq.cloudbreak.cloud.model.CloudInstance)9