Examples with AmazonEc2Client com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEc2Client used on opensource projects

Example 91 with AmazonEc2Client

use of com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEc2Client in project cloudbreak by hortonworks.

the class AwsNetworkCfTemplateProvider method describeVpcServiceDetails.

private List<ServiceDetail> describeVpcServiceDetails(NetworkCreationRequest networkCreationRequest, Map<String, String> endpointNameMappings) {
    AwsCredentialView awsCredential = new AwsCredentialView(networkCreationRequest.getCloudCredential());
    AmazonEc2Client awsClientAccess = awsClient.createEc2Client(awsCredential, networkCreationRequest.getRegion().value());
    return awsClientAccess.describeVpcEndpointServices().getServiceDetails().stream().filter(sd -> endpointNameMappings.containsKey(sd.getServiceName())).collect(Collectors.toList());
}

Example 92 with AmazonEc2Client

use of com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEc2Client in project cloudbreak by hortonworks.

the class AwsNetworkConnector method getNetworkCidr.

@Override
public NetworkCidr getNetworkCidr(Network network, CloudCredential credential) {
    AwsCredentialView awsCredentialView = new AwsCredentialView(credential);
    AmazonEc2Client awsClientAccess = awsClient.createEc2Client(awsCredentialView, network.getStringParameter(AwsNetworkView.REGION));
    AwsNetworkView awsNetworkView = new AwsNetworkView(network);
    String existingVpc = awsNetworkView.getExistingVpc();
    DescribeVpcsResult describeVpcsResult = awsClientAccess.describeVpcs(new DescribeVpcsRequest().withVpcIds(existingVpc));
    List<String> vpcCidrs = new ArrayList<>();
    for (Vpc vpc : describeVpcsResult.getVpcs()) {
        if (vpc.getCidrBlockAssociationSet() != null) {
            LOGGER.info("The VPC {} has associated CIDR block so using the CIDR blocks in the VPC.", vpc.getVpcId());
            List<String> cidrs = vpc.getCidrBlockAssociationSet().stream().map(VpcCidrBlockAssociation::getCidrBlock).distinct().filter(e -> !vpcCidrs.contains(e)).collect(Collectors.toList());
            LOGGER.info("The VPC {} CIDRs block are {}.", vpc.getVpcId(), cidrs);
            vpcCidrs.addAll(cidrs);
        } else {
            LOGGER.info("The VPC {} has no associated CIDR block so using the CIDR block in the VPC.", vpc.getVpcId());
            vpcCidrs.add(vpc.getCidrBlock());
        }
    }
    if (vpcCidrs.isEmpty()) {
        throw new BadRequestException("VPC cidr could not fetch from AWS: " + existingVpc);
    }
    if (vpcCidrs.size() > 1) {
        LOGGER.info("More than one vpc cidrs for VPC {}. We will use the first one: {}", existingVpc, vpcCidrs.get(0));
    }
    return new NetworkCidr(vpcCidrs.get(0), vpcCidrs);
}

Example 93 with AmazonEc2Client

use of com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEc2Client in project cloudbreak by hortonworks.

the class AwsUpscaleService method upscale.

public List<CloudResourceStatus> upscale(AuthenticatedContext ac, CloudStack stack, List<CloudResource> resources, AdjustmentTypeWithThreshold adjustmentTypeWithThreshold) {
    LOGGER.info("Upscale AWS cluster with adjustment and threshold: {}. Resources: {}", adjustmentTypeWithThreshold, resources);
    String regionName = ac.getCloudContext().getLocation().getRegion().value();
    AwsCredentialView credentialView = new AwsCredentialView(ac.getCloudCredential());
    AmazonCloudFormationClient cloudFormationClient = awsClient.createCloudFormationClient(credentialView, regionName);
    AmazonAutoScalingClient amazonASClient = awsClient.createAutoScalingClient(credentialView, regionName);
    AmazonEc2Client amazonEC2Client = awsClient.createEc2Client(credentialView, regionName);
    List<Group> scaledGroups = cloudResourceHelper.getScaledGroups(stack);
    Map<String, Group> desiredAutoscalingGroupsByName = getAutoScaleGroupsByNameFromCloudFormationTemplate(ac, cloudFormationClient, scaledGroups);
    LOGGER.info("Desired autoscaling groups: {}", desiredAutoscalingGroupsByName);
    awsAutoScalingService.resumeAutoScaling(amazonASClient, desiredAutoscalingGroupsByName.keySet(), UPSCALE_PROCESSES);
    Map<String, Integer> originalAutoScalingGroupsBySize = getAutoScalingGroupsBySize(desiredAutoscalingGroupsByName.keySet(), amazonASClient);
    LOGGER.info("Update autoscaling groups for stack: {}", ac.getCloudContext().getName());
    Date timeBeforeASUpdate = new Date();
    updateAutoscalingGroups(amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize);
    List<String> knownInstances = getKnownInstancesByCloudbreak(stack);
    try {
        awsAutoScalingService.scheduleStatusChecks(scaledGroups, ac, cloudFormationClient, timeBeforeASUpdate, knownInstances);
    } catch (AmazonAutoscalingFailed amazonAutoscalingFailed) {
        LOGGER.info("Amazon autoscaling group update failed", amazonAutoscalingFailed);
        recoverOriginalState(ac, stack, amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize, amazonAutoscalingFailed);
        sendASGUpdateFailedMessage(amazonASClient, desiredAutoscalingGroupsByName, amazonAutoscalingFailed);
    }
    try {
        awsAutoScalingService.suspendAutoScaling(ac, stack);
        validateInstanceStatusesInScaledGroups(ac, amazonASClient, amazonEC2Client, cloudFormationClient, scaledGroups);
        List<CloudResource> instances = cfStackUtil.getInstanceCloudResources(ac, cloudFormationClient, amazonASClient, scaledGroups);
        associateElasticIpWithNewInstances(stack, resources, cloudFormationClient, amazonEC2Client, scaledGroups, instances);
        List<Group> groupsWithNewInstances = getGroupsWithNewInstances(scaledGroups);
        List<CloudResource> newInstances = getNewInstances(scaledGroups, instances);
        List<CloudResource> reattachableVolumeSets = getReattachableVolumeSets(scaledGroups, resources);
        List<CloudResource> networkResources = resources.stream().filter(cloudResource -> ResourceType.AWS_SUBNET.equals(cloudResource.getType())).collect(Collectors.toList());
        List<CloudResourceStatus> cloudResourceStatuses = awsComputeResourceService.buildComputeResourcesForUpscale(ac, stack, groupsWithNewInstances, newInstances, reattachableVolumeSets, networkResources, adjustmentTypeWithThreshold);
        List<String> failedResources = cloudResourceStatuses.stream().map(CloudResourceStatus::getCloudResource).filter(cloudResource -> CommonStatus.FAILED == cloudResource.getStatus()).map(cloudResource -> cloudResource.getType() + " - " + cloudResource.getName()).collect(Collectors.toList());
        if (!failedResources.isEmpty()) {
            throw new RuntimeException("Additional resource creation failed: " + failedResources);
        }
        awsTaggingService.tagRootVolumes(ac, amazonEC2Client, instances, stack.getTags());
        awsCloudWatchService.addCloudWatchAlarmsForSystemFailures(instances, regionName, credentialView);
        for (CloudLoadBalancer loadBalancer : stack.getLoadBalancers()) {
            cfStackUtil.addLoadBalancerTargets(ac, loadBalancer, newInstances);
        }
        List<CloudResourceStatus> successfulInstances = newInstances.stream().map(cloudResource -> new CloudResourceStatus(cloudResource, ResourceStatus.CREATED, cloudResource.getParameter(CloudResource.PRIVATE_ID, Long.class))).collect(Collectors.toList());
        return ListUtils.union(singletonList(new CloudResourceStatus(cfStackUtil.getCloudFormationStackResource(resources), ResourceStatus.UPDATED)), successfulInstances);
    } catch (RuntimeException runtimeException) {
        recoverOriginalState(ac, stack, amazonASClient, desiredAutoscalingGroupsByName, originalAutoScalingGroupsBySize, runtimeException);
        throw new CloudConnectorException(String.format("Failed to create some resource on AWS for upscaled nodes, please check your quotas on AWS. " + "Original autoscaling group state has been recovered. Exception: %s", runtimeException.getMessage()), runtimeException);
    }
}

Example 94 with AmazonEc2Client

use of com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEc2Client in project cloudbreak by hortonworks.

the class AwsGatewaySubnetMultiAzValidator method fetchingSubnetsAndGroupingByAvailabilityZone.

private Map<String, Set<String>> fetchingSubnetsAndGroupingByAvailabilityZone(AwsCredentialView awsCredential, Set<String> subnetIds, String region) {
    String subnetIdsJoined = String.join(",", subnetIds);
    LOGGER.info("Fetching subnets('{}') for validating availability zones for gateway group", subnetIdsJoined);
    DescribeSubnetsRequest describeSubnetsRequest = new DescribeSubnetsRequest().withSubnetIds(subnetIds);
    try {
        AmazonEc2Client ec2Client = awsClient.createEc2Client(awsCredential, region);
        DescribeSubnetsResult describeSubnetsResult = ec2Client.describeSubnets(describeSubnetsRequest);
        return describeSubnetsResult.getSubnets().stream().collect(groupingBy(Subnet::getAvailabilityZone, mapping(Subnet::getSubnetId, toSet())));
    } catch (AmazonServiceException ex) {
        String msg = String.format("Failed to fetch subnets with id: '%s' during the validation of subnets per availability zone for gateway group.", subnetIdsJoined);
        LOGGER.warn(msg, ex);
        throw new CloudConnectorException(msg, ex);
    }
}

Example 95 with AmazonEc2Client

use of com.sequenceiq.cloudbreak.cloud.aws.common.client.AmazonEc2Client in project cloudbreak by hortonworks.

the class AwsNativeMetadataCollector method collectInstances.

private List<CloudVmMetaDataStatus> collectInstances(List<CloudInstance> vms, List<CloudResource> resources, AmazonEc2Client ec2Client) {
    List<CloudVmMetaDataStatus> result = new ArrayList<>();
    Set<String> preferredInstanceIds = resources.stream().filter(resource -> ResourceType.AWS_INSTANCE.equals(resource.getType())).filter(resource -> vms.stream().anyMatch(vm -> String.valueOf(vm.getTemplate().getPrivateId()).equals(resource.getReference()))).map(CloudResource::getInstanceId).collect(Collectors.toCollection(LinkedHashSet::new));
    Map<String, CloudResource> resourcesByInstanceId = resources.stream().filter(resource -> ResourceType.AWS_INSTANCE.equals(resource.getType())).filter(cloudResource -> cloudResource.getInstanceId() != null).collect(Collectors.toMap(CloudResource::getInstanceId, Function.identity()));
    final AtomicInteger counter = new AtomicInteger(0);
    Map<Integer, List<String>> instanceIdBatches = preferredInstanceIds.stream().collect(Collectors.groupingBy(s -> counter.getAndIncrement() / instanceFetchMaxBatchSize, LinkedHashMap::new, Collectors.toList()));
    for (Map.Entry<Integer, List<String>> instanceIdBatchEntry : instanceIdBatches.entrySet()) {
        List<String> instanceIdBatch = instanceIdBatchEntry.getValue();
        LOGGER.info("Collecting metadata for instances iteration {}/{}", instanceIdBatchEntry.getKey() + 1, instanceIdBatches.size());
        try {
            result.addAll(describeInstances(vms, instanceIdBatch, resourcesByInstanceId, ec2Client));
        } catch (AmazonServiceException serviceException) {
            if (StringUtils.isNotEmpty(serviceException.getErrorCode()) && INSTANCE_NOT_FOUND_ERROR_CODE.equals(serviceException.getErrorCode())) {
                result.addAll(handleNotFoundInstancesAndDescribeOthers(vms, ec2Client, instanceIdBatch, serviceException, resourcesByInstanceId));
            } else {
                LOGGER.warn("Collection of instance metadata failed", serviceException);
                throw serviceException;
            }
        }
    }
    return result;
}