Search in sources :

Example 91 with AwsCredentialView

use of com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView in project cloudbreak by hortonworks.

the class AwsNativeCloudWatchResourceBuilder method build.

@Override
public List<CloudResource> build(AwsContext context, CloudInstance instance, long privateId, AuthenticatedContext auth, Group group, List<CloudResource> buildableResource, CloudStack cloudStack) throws Exception {
    String region = context.getLocation().getRegion().getRegionName();
    AwsCredentialView credential = new AwsCredentialView(auth.getCloudCredential());
    nativeCloudWatchService.addCloudWatchAlarmsForSystemFailures(buildableResource.get(0), region, credential);
    CloudResource resource = CloudResource.builder().cloudResource(buildableResource.get(0)).build();
    return List.of(resource);
}
Also used : AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView) CloudResource(com.sequenceiq.cloudbreak.cloud.model.CloudResource)

Example 92 with AwsCredentialView

use of com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView in project cloudbreak by hortonworks.

the class AwsNativeCloudWatchService method deleteCloudWatchAlarmsForSystemFailures.

public void deleteCloudWatchAlarmsForSystemFailures(String regionName, AwsCredentialView credentialView, List<String> instanceIds) {
    // The list of alarms received may be longer than 100 items, but alarms can only be deleted in batches of max
    // size 100. To work around this, break the instanceIds list into chunks no greater than 100, and process each
    // chunk. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_DeleteAlarms.html
    final AtomicInteger counter = new AtomicInteger(0);
    Map<Integer, List<String>> grouping = instanceIds.stream().map(instanceId -> instanceId + alarmSuffix).collect(Collectors.groupingBy(s -> counter.getAndIncrement() / maxBatchsize));
    LOGGER.debug("Batched cloudwatch alarm delete requests: {}", grouping);
    grouping.values().stream().flatMap(alarmNames -> getExistingCloudWatchAlarms(regionName, credentialView, alarmNames)).filter(alarmNames -> !alarmNames.isEmpty()).forEach(alarmNames -> deleteCloudWatchAlarms(regionName, credentialView, alarmNames));
}
Also used : AtomicInteger(java.util.concurrent.atomic.AtomicInteger) LoggerFactory(org.slf4j.LoggerFactory) AlarmType(com.amazonaws.services.cloudwatch.model.AlarmType) CloudInstance(com.sequenceiq.cloudbreak.cloud.model.CloudInstance) CollectionUtils.isEmpty(org.apache.commons.collections4.CollectionUtils.isEmpty) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) Dimension(com.amazonaws.services.cloudwatch.model.Dimension) CloudConnectorException(com.sequenceiq.cloudbreak.cloud.exception.CloudConnectorException) AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Service(org.springframework.stereotype.Service) Map(java.util.Map) DeleteAlarmsRequest(com.amazonaws.services.cloudwatch.model.DeleteAlarmsRequest) Logger(org.slf4j.Logger) DescribeAlarmsRequest(com.amazonaws.services.cloudwatch.model.DescribeAlarmsRequest) MetricAlarm(com.amazonaws.services.cloudwatch.model.MetricAlarm) CloudResource(com.sequenceiq.cloudbreak.cloud.model.CloudResource) Set(java.util.Set) Collectors(java.util.stream.Collectors) Maximum(com.amazonaws.services.cloudwatch.model.Statistic.Maximum) CloudStack(com.sequenceiq.cloudbreak.cloud.model.CloudStack) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List) PutMetricAlarmRequest(com.amazonaws.services.cloudwatch.model.PutMetricAlarmRequest) Stream(java.util.stream.Stream) CommonAwsClient(com.sequenceiq.cloudbreak.cloud.aws.common.CommonAwsClient) AmazonCloudWatchException(com.amazonaws.services.cloudwatch.model.AmazonCloudWatchException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ArrayList(java.util.ArrayList) Collectors.toList(java.util.stream.Collectors.toList) List(java.util.List)

Example 93 with AwsCredentialView

use of com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView in project cloudbreak by hortonworks.

the class AwsNativeMetadataCollector method collectLoadBalancer.

@Override
public List<CloudLoadBalancerMetadata> collectLoadBalancer(AuthenticatedContext ac, List<LoadBalancerType> lbTypes, List<CloudResource> resources) {
    List<CloudLoadBalancerMetadata> result = new ArrayList<>();
    String region = ac.getCloudContext().getLocation().getRegion().value();
    AwsCredentialView awsCredential = new AwsCredentialView(ac.getCloudCredential());
    LOGGER.debug("Collect AWS load balancer metadata, for cluster {}", ac.getCloudContext().getName());
    Set<CloudResource> loadBalancers = resources.stream().filter(resource -> ResourceType.ELASTIC_LOAD_BALANCER.equals(resource.getType())).collect(Collectors.toSet());
    Set<String> loadBalancerArns = loadBalancers.stream().map(CloudResource::getReference).collect(Collectors.toSet());
    AmazonElasticLoadBalancingClient loadBalancingClient = awsClient.createElasticLoadBalancingClient(awsCredential, region);
    LOGGER.info("Collect AWS load balancer metadata, in region '{}' with ARNs: '{}'", region, String.join(", ", loadBalancerArns));
    for (String loadBalancerArn : loadBalancerArns) {
        Optional<CloudLoadBalancerMetadata> collectedLoadBalancer = collectLoadBalancerMetadata(loadBalancingClient, loadBalancerArn, resources);
        collectedLoadBalancer.ifPresent(result::add);
    }
    return result;
}
Also used : AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView) EntitlementService(com.sequenceiq.cloudbreak.auth.altus.EntitlementService) AmazonElasticLoadBalancingClient(com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonElasticLoadBalancingClient) LoggerFactory(org.slf4j.LoggerFactory) AwsLifeCycleMapper(com.sequenceiq.cloudbreak.cloud.aws.common.util.AwsLifeCycleMapper) DescribeLoadBalancersRequest(com.amazonaws.services.elasticloadbalancingv2.model.DescribeLoadBalancersRequest) CloudInstance(com.sequenceiq.cloudbreak.cloud.model.CloudInstance) Function(java.util.function.Function) StringUtils(org.apache.commons.lang3.StringUtils) DescribeLoadBalancersResult(com.amazonaws.services.elasticloadbalancingv2.model.DescribeLoadBalancersResult) ArrayList(java.util.ArrayList) LoadBalancerType(com.sequenceiq.common.api.type.LoadBalancerType) LinkedHashMap(java.util.LinkedHashMap) Inject(javax.inject.Inject) Value(org.springframework.beans.factory.annotation.Value) CloudConnectorException(com.sequenceiq.cloudbreak.cloud.exception.CloudConnectorException) AmazonEc2Client(com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEc2Client) INSTANCE_NOT_FOUND_ERROR_CODE(com.sequenceiq.cloudbreak.cloud.aws.common.AwsInstanceConnector.INSTANCE_NOT_FOUND_ERROR_CODE) AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView) AuthenticatedContext(com.sequenceiq.cloudbreak.cloud.context.AuthenticatedContext) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Service(org.springframework.stereotype.Service) DescribeInstancesRequest(com.amazonaws.services.ec2.model.DescribeInstancesRequest) Map(java.util.Map) Instance(com.amazonaws.services.ec2.model.Instance) CloudVmInstanceStatus(com.sequenceiq.cloudbreak.cloud.model.CloudVmInstanceStatus) NotFoundException(com.sequenceiq.cloudbreak.common.exception.NotFoundException) LinkedHashSet(java.util.LinkedHashSet) InstanceStoreMetadata(com.sequenceiq.cloudbreak.cloud.model.InstanceStoreMetadata) ResourceType(com.sequenceiq.common.api.type.ResourceType) Logger(org.slf4j.Logger) AmazonServiceException(com.amazonaws.AmazonServiceException) CloudInstanceMetaData(com.sequenceiq.cloudbreak.cloud.model.CloudInstanceMetaData) CloudResource(com.sequenceiq.cloudbreak.cloud.model.CloudResource) Set(java.util.Set) DescribeInstancesResult(com.amazonaws.services.ec2.model.DescribeInstancesResult) MetadataCollector(com.sequenceiq.cloudbreak.cloud.MetadataCollector) Collectors(java.util.stream.Collectors) List(java.util.List) AwsPlatformResources(com.sequenceiq.cloudbreak.cloud.aws.common.AwsPlatformResources) InstanceStatus(com.sequenceiq.cloudbreak.cloud.model.InstanceStatus) Optional(java.util.Optional) CloudLoadBalancerMetadata(com.sequenceiq.cloudbreak.cloud.model.CloudLoadBalancerMetadata) CommonAwsClient(com.sequenceiq.cloudbreak.cloud.aws.common.CommonAwsClient) CloudVmMetaDataStatus(com.sequenceiq.cloudbreak.cloud.model.CloudVmMetaDataStatus) LoadBalancerTypeConverter(com.sequenceiq.cloudbreak.cloud.aws.common.loadbalancer.LoadBalancerTypeConverter) AmazonElasticLoadBalancingClient(com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonElasticLoadBalancingClient) ArrayList(java.util.ArrayList) CloudResource(com.sequenceiq.cloudbreak.cloud.model.CloudResource) CloudLoadBalancerMetadata(com.sequenceiq.cloudbreak.cloud.model.CloudLoadBalancerMetadata)

Example 94 with AwsCredentialView

use of com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView in project cloudbreak by hortonworks.

the class CloudFormationStackUtil method removeLoadBalancerTargets.

public void removeLoadBalancerTargets(AuthenticatedContext ac, CloudLoadBalancer loadBalancer, List<CloudResource> resourcesToRemove) {
    String region = ac.getCloudContext().getLocation().getRegion().value();
    AmazonElasticLoadBalancingClient amazonElbClient = awsClient.createElasticLoadBalancingClient(new AwsCredentialView(ac.getCloudCredential()), region);
    for (Map.Entry<TargetGroupPortPair, Set<Group>> entry : loadBalancer.getPortToTargetGroupMapping().entrySet()) {
        // Get a list of the instance ids to remove
        Set<String> instancesToRemove = getInstanceIdsForGroups(resourcesToRemove, entry.getValue());
        // Find target group ARN
        AwsLoadBalancerScheme scheme = loadBalancerTypeConverter.convert(loadBalancer.getType());
        String targetGroupArn = getResourceArnByLogicalId(ac, AwsTargetGroup.getTargetGroupName(entry.getKey().getTrafficPort(), scheme), region);
        // Deregister any instances that no longer exist
        if (!instancesToRemove.isEmpty()) {
            try {
                List<TargetDescription> targetsToRemove = instancesToRemove.stream().map(instanceId -> new TargetDescription().withId(instanceId)).collect(Collectors.toList());
                DeregisterTargetsResult deregisterTargetsResult = amazonElbClient.deregisterTargets(new DeregisterTargetsRequest().withTargetGroupArn(targetGroupArn).withTargets(targetsToRemove));
            } catch (InvalidTargetException ignored) {
            // no-op - we tried to remove a target that wasn't in the target group, which is fine
            }
        }
    }
}
Also used : DeregisterTargetsRequest(com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest) RegisterTargetsRequest(com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsRequest) DescribeLoadBalancersRequest(com.amazonaws.services.elasticloadbalancingv2.model.DescribeLoadBalancersRequest) CloudInstance(com.sequenceiq.cloudbreak.cloud.model.CloudInstance) TargetHealthDescription(com.amazonaws.services.elasticloadbalancingv2.model.TargetHealthDescription) AmazonCloudFormationClient(com.sequenceiq.cloudbreak.cloud.aws.client.AmazonCloudFormationClient) AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView) AuthenticatedContext(com.sequenceiq.cloudbreak.cloud.context.AuthenticatedContext) DescribeInstancesRequest(com.amazonaws.services.ec2.model.DescribeInstancesRequest) Map(java.util.Map) CloudLoadBalancer(com.sequenceiq.cloudbreak.cloud.model.CloudLoadBalancer) AwsTargetGroup(com.sequenceiq.cloudbreak.cloud.aws.common.loadbalancer.AwsTargetGroup) Splitter(com.google.common.base.Splitter) CommonStatus(com.sequenceiq.common.api.type.CommonStatus) DescribeFileSystemsResult(com.amazonaws.services.elasticfilesystem.model.DescribeFileSystemsResult) AmazonServiceException(com.amazonaws.AmazonServiceException) DescribeTargetHealthResult(com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetHealthResult) Collection(java.util.Collection) Output(com.amazonaws.services.cloudformation.model.Output) Set(java.util.Set) Retry(com.sequenceiq.cloudbreak.service.Retry) Collectors(java.util.stream.Collectors) Backoff(org.springframework.retry.annotation.Backoff) Objects(java.util.Objects) List(java.util.List) AmazonAutoScalingClient(com.sequenceiq.cloudbreak.cloud.aws.client.AmazonAutoScalingClient) Group(com.sequenceiq.cloudbreak.cloud.model.Group) InvalidTargetException(com.amazonaws.services.elasticloadbalancingv2.model.InvalidTargetException) PRIVATE_ID(com.sequenceiq.cloudbreak.cloud.model.CloudResource.PRIVATE_ID) LoadBalancerTypeConverter(com.sequenceiq.cloudbreak.cloud.aws.common.loadbalancer.LoadBalancerTypeConverter) DescribeTargetHealthRequest(com.amazonaws.services.elasticloadbalancingv2.model.DescribeTargetHealthRequest) LoadBalancer(com.amazonaws.services.elasticloadbalancingv2.model.LoadBalancer) TargetGroupPortPair(com.sequenceiq.cloudbreak.cloud.model.TargetGroupPortPair) INSTANCE_TYPE(com.sequenceiq.cloudbreak.cloud.model.CloudResource.INSTANCE_TYPE) DescribeStacksRequest(com.amazonaws.services.cloudformation.model.DescribeStacksRequest) AmazonElasticLoadBalancingClient(com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonElasticLoadBalancingClient) AmazonEfsClient(com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEfsClient) Supplier(java.util.function.Supplier) DescribeLoadBalancersResult(com.amazonaws.services.elasticloadbalancingv2.model.DescribeLoadBalancersResult) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) Value(org.springframework.beans.factory.annotation.Value) CloudConnectorException(com.sequenceiq.cloudbreak.cloud.exception.CloudConnectorException) DeregisterTargetsResult(com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsResult) Service(org.springframework.stereotype.Service) DescribeAutoScalingGroupsResult(com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsResult) AwsLoadBalancerScheme(com.sequenceiq.cloudbreak.cloud.aws.common.loadbalancer.AwsLoadBalancerScheme) Retryable(org.springframework.retry.annotation.Retryable) DescribeStackResourceRequest(com.amazonaws.services.cloudformation.model.DescribeStackResourceRequest) TargetDescription(com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription) ResourceType(com.sequenceiq.common.api.type.ResourceType) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) FileSystemDescription(com.amazonaws.services.elasticfilesystem.model.FileSystemDescription) CloudResource(com.sequenceiq.cloudbreak.cloud.model.CloudResource) DescribeFileSystemsRequest(com.amazonaws.services.elasticfilesystem.model.DescribeFileSystemsRequest) RegisterTargetsResult(com.amazonaws.services.elasticloadbalancingv2.model.RegisterTargetsResult) DescribeAutoScalingGroupsRequest(com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsRequest) SdkClientException(com.amazonaws.SdkClientException) LoggerFactory.getLogger(org.slf4j.LoggerFactory.getLogger) Instance(com.amazonaws.services.autoscaling.model.Instance) DescribeStackResourceResult(com.amazonaws.services.cloudformation.model.DescribeStackResourceResult) Collections(java.util.Collections) Set(java.util.Set) AmazonElasticLoadBalancingClient(com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonElasticLoadBalancingClient) DeregisterTargetsResult(com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsResult) TargetDescription(com.amazonaws.services.elasticloadbalancingv2.model.TargetDescription) DeregisterTargetsRequest(com.amazonaws.services.elasticloadbalancingv2.model.DeregisterTargetsRequest) AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView) InvalidTargetException(com.amazonaws.services.elasticloadbalancingv2.model.InvalidTargetException) Map(java.util.Map) TargetGroupPortPair(com.sequenceiq.cloudbreak.cloud.model.TargetGroupPortPair) AwsLoadBalancerScheme(com.sequenceiq.cloudbreak.cloud.aws.common.loadbalancer.AwsLoadBalancerScheme)

Aggregations

AwsCredentialView (com.sequenceiq.cloudbreak.cloud.aws.common.view.AwsCredentialView)94 AmazonEc2Client (com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEc2Client)32 CloudConnectorException (com.sequenceiq.cloudbreak.cloud.exception.CloudConnectorException)32 List (java.util.List)25 AmazonServiceException (com.amazonaws.AmazonServiceException)22 AmazonCloudFormationClient (com.sequenceiq.cloudbreak.cloud.aws.client.AmazonCloudFormationClient)21 Logger (org.slf4j.Logger)21 Inject (javax.inject.Inject)20 ArrayList (java.util.ArrayList)19 Collectors (java.util.stream.Collectors)19 CloudInstance (com.sequenceiq.cloudbreak.cloud.model.CloudInstance)18 Group (com.sequenceiq.cloudbreak.cloud.model.Group)18 Set (java.util.Set)18 CloudResource (com.sequenceiq.cloudbreak.cloud.model.CloudResource)17 CloudStack (com.sequenceiq.cloudbreak.cloud.model.CloudStack)17 Map (java.util.Map)16 LoggerFactory (org.slf4j.LoggerFactory)16 AuthenticatedContext (com.sequenceiq.cloudbreak.cloud.context.AuthenticatedContext)15 Service (org.springframework.stereotype.Service)15 AmazonAutoScalingClient (com.sequenceiq.cloudbreak.cloud.aws.client.AmazonAutoScalingClient)14