Search in sources :

Example 1 with AmazonCloudFormationClient

use of com.amazonaws.services.cloudformation.AmazonCloudFormationClient in project cloudbreak by hortonworks.

the class AwsResourceConnector method upscale.

@Override
public List<CloudResourceStatus> upscale(AuthenticatedContext ac, CloudStack stack, List<CloudResource> resources) {
    resumeAutoScaling(ac, stack);
    AmazonAutoScalingClient amazonASClient = awsClient.createAutoScalingClient(new AwsCredentialView(ac.getCloudCredential()), ac.getCloudContext().getLocation().getRegion().value());
    AmazonCloudFormationClient cloudFormationClient = awsClient.createCloudFormationClient(new AwsCredentialView(ac.getCloudCredential()), ac.getCloudContext().getLocation().getRegion().value());
    AmazonEC2Client amazonEC2Client = awsClient.createAccess(new AwsCredentialView(ac.getCloudCredential()), ac.getCloudContext().getLocation().getRegion().value());
    List<Group> scaledGroups = getScaledGroups(stack);
    for (Group group : scaledGroups) {
        String asGroupName = cfStackUtil.getAutoscalingGroupName(ac, cloudFormationClient, group.getName());
        amazonASClient.updateAutoScalingGroup(new UpdateAutoScalingGroupRequest().withAutoScalingGroupName(asGroupName).withMaxSize(group.getInstancesSize()).withDesiredCapacity(group.getInstancesSize()));
        LOGGER.info("Updated Auto Scaling group's desiredCapacity: [stack: '{}', to: '{}']", ac.getCloudContext().getId(), resources.size());
    }
    scheduleStatusChecks(stack, ac, cloudFormationClient);
    suspendAutoScaling(ac, stack);
    boolean mapPublicIpOnLaunch = isMapPublicOnLaunch(new AwsNetworkView(stack.getNetwork()), amazonEC2Client);
    List<Group> gateways = getGatewayGroups(scaledGroups);
    if (mapPublicIpOnLaunch && !gateways.isEmpty()) {
        String cFStackName = getCloudFormationStackResource(resources).getName();
        Map<String, String> eipAllocationIds = getElasticIpAllocationIds(cFStackName, cloudFormationClient);
        for (Group gateway : gateways) {
            List<String> eips = getEipsForGatewayGroup(eipAllocationIds, gateway);
            List<String> freeEips = getFreeIps(eips, amazonEC2Client);
            List<String> instanceIds = getInstancesForGroup(ac, amazonASClient, cloudFormationClient, gateway);
            List<String> newInstances = instanceIds.stream().filter(iid -> gateway.getInstances().stream().noneMatch(inst -> iid.equals(inst.getInstanceId()))).collect(Collectors.toList());
            associateElasticIpsToInstances(amazonEC2Client, freeEips, newInstances);
        }
    }
    return singletonList(new CloudResourceStatus(getCloudFormationStackResource(resources), ResourceStatus.UPDATED));
}
Also used : AmazonEC2Client(com.amazonaws.services.ec2.AmazonEC2Client) CreateStackRequest(com.amazonaws.services.cloudformation.model.CreateStackRequest) DescribeSubnetsRequest(com.amazonaws.services.ec2.model.DescribeSubnetsRequest) ReleaseAddressRequest(com.amazonaws.services.ec2.model.ReleaseAddressRequest) ROLLBACK_IN_PROGRESS(com.amazonaws.services.cloudformation.model.StackStatus.ROLLBACK_IN_PROGRESS) SuspendProcessesRequest(com.amazonaws.services.autoscaling.model.SuspendProcessesRequest) StringUtils(org.apache.commons.lang3.StringUtils) Collections.singletonList(java.util.Collections.singletonList) DescribeSubnetsResult(com.amazonaws.services.ec2.model.DescribeSubnetsResult) AmazonCloudFormationClient(com.amazonaws.services.cloudformation.AmazonCloudFormationClient) Arrays.asList(java.util.Arrays.asList) Map(java.util.Map) ROLLBACK_FAILED(com.amazonaws.services.cloudformation.model.StackStatus.ROLLBACK_FAILED) AdjustmentType(com.sequenceiq.cloudbreak.api.model.AdjustmentType) SubnetInfo(org.apache.commons.net.util.SubnetUtils.SubnetInfo) AmazonServiceException(com.amazonaws.AmazonServiceException) AutoScalingGroup(com.amazonaws.services.autoscaling.model.AutoScalingGroup) CloudResourceStatus(com.sequenceiq.cloudbreak.cloud.model.CloudResourceStatus) Retry(com.sequenceiq.cloudbreak.service.Retry) SyncPollingScheduler(com.sequenceiq.cloudbreak.cloud.scheduler.SyncPollingScheduler) AwsInstanceProfileView(com.sequenceiq.cloudbreak.cloud.aws.view.AwsInstanceProfileView) AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.view.AwsCredentialView) DELETE_FAILED(com.amazonaws.services.cloudformation.model.StackStatus.DELETE_FAILED) AwsPollTaskFactory(com.sequenceiq.cloudbreak.cloud.aws.task.AwsPollTaskFactory) DescribeStacksRequest(com.amazonaws.services.cloudformation.model.DescribeStacksRequest) StackStatus(com.amazonaws.services.cloudformation.model.StackStatus) DescribeKeyPairsRequest(com.amazonaws.services.ec2.model.DescribeKeyPairsRequest) InstanceGroupType(com.sequenceiq.cloudbreak.api.model.InstanceGroupType) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Value(org.springframework.beans.factory.annotation.Value) CloudConnectorException(com.sequenceiq.cloudbreak.cloud.exception.CloudConnectorException) Lists(com.google.common.collect.Lists) UpdateAutoScalingGroupRequest(com.amazonaws.services.autoscaling.model.UpdateAutoScalingGroupRequest) Builder(com.sequenceiq.cloudbreak.cloud.model.CloudResource.Builder) Service(org.springframework.stereotype.Service) Image(com.amazonaws.services.ec2.model.Image) Filter(com.amazonaws.services.ec2.model.Filter) Network(com.sequenceiq.cloudbreak.cloud.model.Network) DeleteStackRequest(com.amazonaws.services.cloudformation.model.DeleteStackRequest) AmazonEC2Client(com.amazonaws.services.ec2.AmazonEC2Client) DELETE_COMPLETE(com.amazonaws.services.cloudformation.model.StackStatus.DELETE_COMPLETE) CREATE_FAILED(com.amazonaws.services.cloudformation.model.StackStatus.CREATE_FAILED) DetachInstancesRequest(com.amazonaws.services.autoscaling.model.DetachInstancesRequest) Parameter(com.amazonaws.services.cloudformation.model.Parameter) ROLLBACK_COMPLETE(com.amazonaws.services.cloudformation.model.StackStatus.ROLLBACK_COMPLETE) IOException(java.io.IOException) DescribeAddressesRequest(com.amazonaws.services.ec2.model.DescribeAddressesRequest) TerminateInstancesRequest(com.amazonaws.services.ec2.model.TerminateInstancesRequest) PersistenceNotifier(com.sequenceiq.cloudbreak.cloud.notification.PersistenceNotifier) InstanceStatus(com.sequenceiq.cloudbreak.cloud.model.InstanceStatus) ImportKeyPairRequest(com.amazonaws.services.ec2.model.ImportKeyPairRequest) Configuration(freemarker.template.Configuration) InetAddresses(com.google.common.net.InetAddresses) SubnetUtils(org.apache.commons.net.util.SubnetUtils) DescribeVpcsRequest(com.amazonaws.services.ec2.model.DescribeVpcsRequest) LoggerFactory(org.slf4j.LoggerFactory) CloudInstance(com.sequenceiq.cloudbreak.cloud.model.CloudInstance) OnFailure(com.amazonaws.services.cloudformation.model.OnFailure) TlsInfo(com.sequenceiq.cloudbreak.cloud.model.TlsInfo) AmazonCloudFormation(com.amazonaws.services.cloudformation.AmazonCloudFormation) AuthenticatedContext(com.sequenceiq.cloudbreak.cloud.context.AuthenticatedContext) DescribeImagesRequest(com.amazonaws.services.ec2.model.DescribeImagesRequest) ResourceStatus(com.sequenceiq.cloudbreak.cloud.model.ResourceStatus) CREATE_COMPLETE(com.amazonaws.services.cloudformation.model.StackStatus.CREATE_COMPLETE) Collection(java.util.Collection) Output(com.amazonaws.services.cloudformation.model.Output) AmazonAutoScalingClient(com.amazonaws.services.autoscaling.AmazonAutoScalingClient) Collectors(java.util.stream.Collectors) AwsNetworkView(com.sequenceiq.cloudbreak.cloud.aws.view.AwsNetworkView) CloudStack(com.sequenceiq.cloudbreak.cloud.model.CloudStack) List(java.util.List) ModelContext(com.sequenceiq.cloudbreak.cloud.aws.CloudFormationTemplateBuilder.ModelContext) ActionWentFailException(com.sequenceiq.cloudbreak.service.Retry.ActionWentFailException) Group(com.sequenceiq.cloudbreak.cloud.model.Group) Entry(java.util.Map.Entry) Optional(java.util.Optional) AmazonAutoScaling(com.amazonaws.services.autoscaling.AmazonAutoScaling) HashMap(java.util.HashMap) Inject(javax.inject.Inject) DeleteKeyPairRequest(com.amazonaws.services.ec2.model.DeleteKeyPairRequest) DisassociateAddressRequest(com.amazonaws.services.ec2.model.DisassociateAddressRequest) Charset(java.nio.charset.Charset) AmazonEC2(com.amazonaws.services.ec2.AmazonEC2) Qualifier(org.springframework.beans.factory.annotation.Qualifier) ResumeProcessesRequest(com.amazonaws.services.autoscaling.model.ResumeProcessesRequest) PollTask(com.sequenceiq.cloudbreak.cloud.task.PollTask) DescribeImagesResult(com.amazonaws.services.ec2.model.DescribeImagesResult) Subnet(com.amazonaws.services.ec2.model.Subnet) ResourceConnector(com.sequenceiq.cloudbreak.cloud.ResourceConnector) Logger(org.slf4j.Logger) ResourceType(com.sequenceiq.cloudbreak.common.type.ResourceType) Address(com.amazonaws.services.ec2.model.Address) Vpc(com.amazonaws.services.ec2.model.Vpc) CloudResource(com.sequenceiq.cloudbreak.cloud.model.CloudResource) DescribeAutoScalingGroupsRequest(com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsRequest) AssociateAddressRequest(com.amazonaws.services.ec2.model.AssociateAddressRequest) DescribeAddressesResult(com.amazonaws.services.ec2.model.DescribeAddressesResult) Collections(java.util.Collections) AutoScalingGroup(com.amazonaws.services.autoscaling.model.AutoScalingGroup) Group(com.sequenceiq.cloudbreak.cloud.model.Group) AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.view.AwsCredentialView) AmazonAutoScalingClient(com.amazonaws.services.autoscaling.AmazonAutoScalingClient) AwsNetworkView(com.sequenceiq.cloudbreak.cloud.aws.view.AwsNetworkView) CloudResourceStatus(com.sequenceiq.cloudbreak.cloud.model.CloudResourceStatus) UpdateAutoScalingGroupRequest(com.amazonaws.services.autoscaling.model.UpdateAutoScalingGroupRequest) AmazonCloudFormationClient(com.amazonaws.services.cloudformation.AmazonCloudFormationClient)

Example 2 with AmazonCloudFormationClient

use of com.amazonaws.services.cloudformation.AmazonCloudFormationClient in project cloudbreak by hortonworks.

the class AwsStackValidator method validate.

@Override
public void validate(AuthenticatedContext ac, CloudStack cloudStack) {
    AwsCredentialView credentialView = new AwsCredentialView(ac.getCloudCredential());
    String regionName = ac.getCloudContext().getLocation().getRegion().value();
    AmazonCloudFormationClient cfClient = awsClient.createCloudFormationClient(credentialView, regionName);
    String cFStackName = cfStackUtil.getCfStackName(ac);
    try {
        cfClient.describeStacks(new DescribeStacksRequest().withStackName(cFStackName));
        throw new CloudConnectorException(String.format("Stack is already exists with the given name: %s", cFStackName));
    } catch (AmazonServiceException ignored) {
    }
}
Also used : AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.view.AwsCredentialView) DescribeStacksRequest(com.amazonaws.services.cloudformation.model.DescribeStacksRequest) CloudConnectorException(com.sequenceiq.cloudbreak.cloud.exception.CloudConnectorException) AmazonServiceException(com.amazonaws.AmazonServiceException) AmazonCloudFormationClient(com.amazonaws.services.cloudformation.AmazonCloudFormationClient)

Example 3 with AmazonCloudFormationClient

use of com.amazonaws.services.cloudformation.AmazonCloudFormationClient in project cloudbreak by hortonworks.

the class AwsDeleteVpcTest method deleteNetwork.

@AfterSuite
@Parameters({ "regionName", "vpcStackName" })
public void deleteNetwork(String regionName, @Optional("it-vpc-stack") String vpcStackName) {
    AmazonCloudFormationClient client = new AmazonCloudFormationClient();
    client.setRegion(RegionUtils.getRegion(regionName));
    client.deleteStack(new DeleteStackRequest().withStackName(vpcStackName));
}
Also used : DeleteStackRequest(com.amazonaws.services.cloudformation.model.DeleteStackRequest) AmazonCloudFormationClient(com.amazonaws.services.cloudformation.AmazonCloudFormationClient) Parameters(org.testng.annotations.Parameters) AfterSuite(org.testng.annotations.AfterSuite)

Example 4 with AmazonCloudFormationClient

use of com.amazonaws.services.cloudformation.AmazonCloudFormationClient in project cloudbreak by hortonworks.

the class AwsResourceConnector method launch.

@Override
public List<CloudResourceStatus> launch(AuthenticatedContext ac, CloudStack stack, PersistenceNotifier resourceNotifier, AdjustmentType adjustmentType, Long threshold) throws Exception {
    createKeyPair(ac, stack);
    String cFStackName = cfStackUtil.getCfStackName(ac);
    AwsCredentialView credentialView = new AwsCredentialView(ac.getCloudCredential());
    String regionName = ac.getCloudContext().getLocation().getRegion().value();
    AmazonCloudFormationClient cfClient = awsClient.createCloudFormationClient(credentialView, regionName);
    AmazonEC2Client amazonEC2Client = awsClient.createAccess(credentialView, regionName);
    AwsNetworkView awsNetworkView = new AwsNetworkView(stack.getNetwork());
    boolean existingVPC = awsNetworkView.isExistingVPC();
    boolean existingSubnet = awsNetworkView.isExistingSubnet();
    boolean mapPublicIpOnLaunch = isMapPublicOnLaunch(awsNetworkView, amazonEC2Client);
    try {
        cfClient.describeStacks(new DescribeStacksRequest().withStackName(cFStackName));
        LOGGER.info("Stack already exists: {}", cFStackName);
    } catch (AmazonServiceException ignored) {
        CloudResource cloudFormationStack = new Builder().type(ResourceType.CLOUDFORMATION_STACK).name(cFStackName).build();
        resourceNotifier.notifyAllocation(cloudFormationStack, ac.getCloudContext());
        String cidr = stack.getNetwork().getSubnet().getCidr();
        String subnet = isNoCIDRProvided(existingVPC, existingSubnet, cidr) ? findNonOverLappingCIDR(ac, stack) : cidr;
        AwsInstanceProfileView awsInstanceProfileView = new AwsInstanceProfileView(stack);
        ModelContext modelContext = new ModelContext().withAuthenticatedContext(ac).withStack(stack).withExistingVpc(existingVPC).withSnapshotId(getEbsSnapshotIdIfNeeded(ac, stack)).withExistingIGW(awsNetworkView.isExistingIGW()).withExistingSubnetCidr(existingSubnet ? getExistingSubnetCidr(ac, stack) : null).withExistingSubnetIds(existingSubnet ? awsNetworkView.getSubnetList() : null).mapPublicIpOnLaunch(mapPublicIpOnLaunch).withEnableInstanceProfile(awsInstanceProfileView.isEnableInstanceProfileStrategy()).withInstanceProfileAvailable(awsInstanceProfileView.isInstanceProfileAvailable()).withTemplate(stack.getTemplate()).withDefaultSubnet(subnet);
        String cfTemplate = cloudFormationTemplateBuilder.build(modelContext);
        LOGGER.debug("CloudFormationTemplate: {}", cfTemplate);
        cfClient.createStack(createCreateStackRequest(ac, stack, cFStackName, subnet, cfTemplate));
    }
    LOGGER.info("CloudFormation stack creation request sent with stack name: '{}' for stack: '{}'", cFStackName, ac.getCloudContext().getId());
    AmazonAutoScalingClient asClient = awsClient.createAutoScalingClient(credentialView, regionName);
    PollTask<Boolean> task = awsPollTaskFactory.newAwsCreateStackStatusCheckerTask(ac, cfClient, asClient, CREATE_COMPLETE, CREATE_FAILED, ERROR_STATUSES, cFStackName);
    try {
        Boolean statePollerResult = task.call();
        if (!task.completed(statePollerResult)) {
            syncPollingScheduler.schedule(task);
        }
    } catch (RuntimeException e) {
        throw new CloudConnectorException(e.getMessage(), e);
    }
    AmazonAutoScalingClient amazonASClient = awsClient.createAutoScalingClient(credentialView, regionName);
    saveS3AccessRoleArn(ac, stack, cFStackName, cfClient, resourceNotifier);
    saveGeneratedSubnet(ac, stack, cFStackName, cfClient, resourceNotifier);
    List<CloudResource> cloudResources = getCloudResources(ac, stack, cFStackName, cfClient, amazonEC2Client, amazonASClient, mapPublicIpOnLaunch);
    return check(ac, cloudResources);
}
Also used : AmazonEC2Client(com.amazonaws.services.ec2.AmazonEC2Client) DescribeStacksRequest(com.amazonaws.services.cloudformation.model.DescribeStacksRequest) CloudConnectorException(com.sequenceiq.cloudbreak.cloud.exception.CloudConnectorException) Builder(com.sequenceiq.cloudbreak.cloud.model.CloudResource.Builder) AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.view.AwsCredentialView) ModelContext(com.sequenceiq.cloudbreak.cloud.aws.CloudFormationTemplateBuilder.ModelContext) AwsNetworkView(com.sequenceiq.cloudbreak.cloud.aws.view.AwsNetworkView) AmazonAutoScalingClient(com.amazonaws.services.autoscaling.AmazonAutoScalingClient) AmazonServiceException(com.amazonaws.AmazonServiceException) CloudResource(com.sequenceiq.cloudbreak.cloud.model.CloudResource) AmazonCloudFormationClient(com.amazonaws.services.cloudformation.AmazonCloudFormationClient) AwsInstanceProfileView(com.sequenceiq.cloudbreak.cloud.aws.view.AwsInstanceProfileView)

Example 5 with AmazonCloudFormationClient

use of com.amazonaws.services.cloudformation.AmazonCloudFormationClient in project cloudbreak by hortonworks.

the class AwsResourceConnector method getCloudResources.

private List<CloudResource> getCloudResources(AuthenticatedContext ac, CloudStack stack, String cFStackName, AmazonCloudFormation client, AmazonEC2 amazonEC2Client, AmazonAutoScaling amazonASClient, boolean mapPublicIpOnLaunch) {
    List<CloudResource> cloudResources = new ArrayList<>();
    AmazonCloudFormationClient cloudFormationClient = awsClient.createCloudFormationClient(new AwsCredentialView(ac.getCloudCredential()), ac.getCloudContext().getLocation().getRegion().value());
    scheduleStatusChecks(stack, ac, cloudFormationClient);
    suspendAutoScaling(ac, stack);
    if (mapPublicIpOnLaunch) {
        Map<String, String> eipAllocationIds = getElasticIpAllocationIds(cFStackName, client);
        List<Group> gateways = getGatewayGroups(stack.getGroups());
        for (Group gateway : gateways) {
            List<String> eips = getEipsForGatewayGroup(eipAllocationIds, gateway);
            List<String> instanceIds = getInstancesForGroup(ac, amazonASClient, client, gateway);
            associateElasticIpsToInstances(amazonEC2Client, eips, instanceIds);
        }
    }
    return cloudResources;
}
Also used : AwsCredentialView(com.sequenceiq.cloudbreak.cloud.aws.view.AwsCredentialView) AutoScalingGroup(com.amazonaws.services.autoscaling.model.AutoScalingGroup) Group(com.sequenceiq.cloudbreak.cloud.model.Group) ArrayList(java.util.ArrayList) CloudResource(com.sequenceiq.cloudbreak.cloud.model.CloudResource) AmazonCloudFormationClient(com.amazonaws.services.cloudformation.AmazonCloudFormationClient)

Aggregations

AmazonCloudFormationClient (com.amazonaws.services.cloudformation.AmazonCloudFormationClient)9 AwsCredentialView (com.sequenceiq.cloudbreak.cloud.aws.view.AwsCredentialView)6 CloudConnectorException (com.sequenceiq.cloudbreak.cloud.exception.CloudConnectorException)5 AmazonServiceException (com.amazonaws.AmazonServiceException)4 DescribeStacksRequest (com.amazonaws.services.cloudformation.model.DescribeStacksRequest)4 AmazonEC2Client (com.amazonaws.services.ec2.AmazonEC2Client)4 AmazonAutoScalingClient (com.amazonaws.services.autoscaling.AmazonAutoScalingClient)3 DeleteStackRequest (com.amazonaws.services.cloudformation.model.DeleteStackRequest)3 CloudResource (com.sequenceiq.cloudbreak.cloud.model.CloudResource)3 AutoScalingGroup (com.amazonaws.services.autoscaling.model.AutoScalingGroup)2 CreateStackRequest (com.amazonaws.services.cloudformation.model.CreateStackRequest)2 Output (com.amazonaws.services.cloudformation.model.Output)2 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 AmazonAutoScaling (com.amazonaws.services.autoscaling.AmazonAutoScaling)1 DescribeAutoScalingGroupsRequest (com.amazonaws.services.autoscaling.model.DescribeAutoScalingGroupsRequest)1 DetachInstancesRequest (com.amazonaws.services.autoscaling.model.DetachInstancesRequest)1 ResumeProcessesRequest (com.amazonaws.services.autoscaling.model.ResumeProcessesRequest)1 SuspendProcessesRequest (com.amazonaws.services.autoscaling.model.SuspendProcessesRequest)1 UpdateAutoScalingGroupRequest (com.amazonaws.services.autoscaling.model.UpdateAutoScalingGroupRequest)1