Examples with Region com.amazonaws.services.ec2.model.Region used on opensource projects

Example 51 with Region

use of com.amazonaws.services.ec2.model.Region in project cloudbreak by hortonworks.

the class AwsCheckSpotInstance method checkSpotInstance.

@Parameters({ "region", "hostGroupToCheck", "scalingAdjustment" })
@Test
public void checkSpotInstance(Regions region, String hostGroupToCheck, @Optional Integer scalingAdjustment) {
    // GIVEN
    Integer spotInstanceCount = 0;
    IntegrationTestContext itContext = getItContext();
    String stackId = itContext.getContextParam(CloudbreakITContextConstants.STACK_ID);
    StackV1Endpoint stackV1Endpoint = getCloudbreakClient().stackV1Endpoint();
    StackResponse stackResponse = stackV1Endpoint.get(Long.valueOf(stackId), new HashSet<>());
    List<InstanceGroupResponse> instanceGroups = stackResponse.getInstanceGroups();
    Collection<String> instanceIdList = new ArrayList<>();
    List<String> hostGroupList = Arrays.asList(hostGroupToCheck.split(","));
    for (InstanceGroupResponse instanceGroup : instanceGroups) {
        if (hostGroupList.contains(instanceGroup.getGroup())) {
            Set<InstanceMetaDataJson> instanceMetaData = instanceGroup.getMetadata();
            for (InstanceMetaDataJson metaData : instanceMetaData) {
                instanceIdList.add(metaData.getInstanceId());
            }
        }
    }
    // WHEN
    AmazonEC2 ec2 = AmazonEC2ClientBuilder.standard().withRegion(region).build();
    DescribeSpotInstanceRequestsResult describeSpotInstanceRequestsResult = ec2.describeSpotInstanceRequests();
    List<SpotInstanceRequest> spotInstanceRequests = describeSpotInstanceRequestsResult.getSpotInstanceRequests();
    // THEN
    Assert.assertFalse(spotInstanceRequests.isEmpty());
    Collection<String> spotInstanceIdList = new ArrayList<>();
    for (SpotInstanceRequest request : spotInstanceRequests) {
        spotInstanceIdList.add(request.getInstanceId());
    }
    for (String id : instanceIdList) {
        Assert.assertTrue(spotInstanceIdList.contains(id));
        if (spotInstanceIdList.contains(id)) {
            spotInstanceCount += 1;
        }
    }
    if (scalingAdjustment != null) {
        Assert.assertNotNull(itContext.getContextParam(CloudbreakITContextConstants.INSTANCE_COUNT, List.class));
        Integer instanceCountPrev = 0;
        for (String hostGroup : hostGroupList) {
            List<Map<String, Integer>> instanceList = itContext.getContextParam(CloudbreakITContextConstants.INSTANCE_COUNT, List.class);
            Assert.assertTrue(instanceList.size() >= 2);
            instanceCountPrev += instanceList.get(instanceList.size() - 2).get(hostGroup);
        }
        Assert.assertEquals(Integer.valueOf(instanceCountPrev + scalingAdjustment), spotInstanceCount);
    }
}

Example 52 with Region

use of com.amazonaws.services.ec2.model.Region in project cloudbreak by hortonworks.

the class TagsUtil method checkTagsAws.

protected static void checkTagsAws(Regions region, Collection<String> instanceIdList, Map<String, String> tagsToCheckMap) {
    Map<String, String> extractedTagsToCheck = new HashMap<>();
    List<Tag> extractedTags;
    AmazonEC2 ec2 = AmazonEC2ClientBuilder.standard().withRegion(region).build();
    DescribeInstancesRequest describeInstancesRequest = new DescribeInstancesRequest();
    describeInstancesRequest.withInstanceIds(instanceIdList);
    DescribeInstancesResult describeInstancesResultAll = ec2.describeInstances(describeInstancesRequest);
    List<Reservation> reservationsAll = describeInstancesResultAll.getReservations();
    for (Reservation reservation : reservationsAll) {
        for (Instance instance : reservation.getInstances()) {
            extractedTags = instance.getTags();
            Assert.assertNotNull(extractedTags);
            for (Tag tag : extractedTags) {
                extractedTagsToCheck.put(tag.getKey(), tag.getValue());
            }
            checkTags(tagsToCheckMap, extractedTagsToCheck);
            extractedTags.clear();
        }
    }
}

Example 53 with Region

use of com.amazonaws.services.ec2.model.Region in project cloudbreak by hortonworks.

the class AwsMetaDataCollectorTest method collectMigratedExistingOneGroup.

@Test
public void collectMigratedExistingOneGroup() {
    List<CloudInstance> vms = new ArrayList<>();
    List<Volume> volumes = new ArrayList<>();
    InstanceAuthentication instanceAuthentication = new InstanceAuthentication("sshkey", "", "cloudbreak");
    vms.add(new CloudInstance("i-1", new InstanceTemplate("fla", "cbgateway", 5L, volumes, InstanceStatus.CREATED, null, 0L), instanceAuthentication));
    when(awsClient.createCloudFormationClient(any(AwsCredentialView.class), eq("region"))).thenReturn(amazonCFClient);
    when(awsClient.createAutoScalingClient(any(AwsCredentialView.class), eq("region"))).thenReturn(amazonASClient);
    when(awsClient.createAccess(any(AwsCredentialView.class), eq("region"))).thenReturn(amazonEC2Client);
    when(cloudFormationStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationClient.class), eq("cbgateway"))).thenReturn("cbgateway-AAA");
    List<String> gatewayIds = Collections.singletonList("i-1");
    when(cloudFormationStackUtil.getInstanceIds(any(AmazonAutoScalingClient.class), eq("cbgateway-AAA"))).thenReturn(gatewayIds);
    when(cloudFormationStackUtil.createDescribeInstancesRequest(eq(gatewayIds))).thenReturn(describeInstancesRequestGw);
    when(amazonEC2Client.describeInstances(describeInstancesRequestGw)).thenReturn(describeInstancesResultGw);
    Instance instance = Mockito.mock(Instance.class);
    when(instance.getInstanceId()).thenReturn("i-1");
    when(instance.getPrivateIpAddress()).thenReturn("privateIp");
    when(instance.getPublicIpAddress()).thenReturn("publicIp");
    List<Reservation> gatewayReservations = Collections.singletonList(getReservation(instance));
    when(describeInstancesResultGw.getReservations()).thenReturn(gatewayReservations);
    AuthenticatedContext ac = authenticatedContext();
    List<CloudVmMetaDataStatus> statuses = awsMetadataCollector.collect(ac, null, vms);
    verify(amazonEC2Client).createTags(any(CreateTagsRequest.class));
    Assert.assertEquals(1, statuses.size());
    Assert.assertEquals("i-1", statuses.get(0).getCloudVmInstanceStatus().getCloudInstance().getInstanceId());
    Assert.assertEquals("privateIp", statuses.get(0).getMetaData().getPrivateIp());
    Assert.assertEquals("publicIp", statuses.get(0).getMetaData().getPublicIp());
}

Example 54 with Region

use of com.amazonaws.services.ec2.model.Region in project cloudbreak by hortonworks.

the class AwsMetaDataCollectorTest method collectNewAndExistingOneGroup.

@Test
public void collectNewAndExistingOneGroup() {
    List<CloudInstance> vms = new ArrayList<>();
    List<Volume> volumes = new ArrayList<>();
    InstanceAuthentication instanceAuthentication = new InstanceAuthentication("sshkey", "", "cloudbreak");
    vms.add(new CloudInstance(null, new InstanceTemplate("fla", "cbgateway", 5L, volumes, InstanceStatus.CREATED, null, 0L), instanceAuthentication));
    vms.add(new CloudInstance("i-1", new InstanceTemplate("fla", "cbgateway", 5L, volumes, InstanceStatus.CREATED, null, 0L), instanceAuthentication));
    when(awsClient.createCloudFormationClient(any(AwsCredentialView.class), eq("region"))).thenReturn(amazonCFClient);
    when(awsClient.createAutoScalingClient(any(AwsCredentialView.class), eq("region"))).thenReturn(amazonASClient);
    when(awsClient.createAccess(any(AwsCredentialView.class), eq("region"))).thenReturn(amazonEC2Client);
    when(cloudFormationStackUtil.getAutoscalingGroupName(any(AuthenticatedContext.class), any(AmazonCloudFormationClient.class), eq("cbgateway"))).thenReturn("cbgateway-AAA");
    List<String> gatewayIds = Arrays.asList("i-1", "i-2");
    when(cloudFormationStackUtil.getInstanceIds(any(AmazonAutoScalingClient.class), eq("cbgateway-AAA"))).thenReturn(gatewayIds);
    when(cloudFormationStackUtil.createDescribeInstancesRequest(eq(gatewayIds))).thenReturn(describeInstancesRequestGw);
    when(amazonEC2Client.describeInstances(describeInstancesRequestGw)).thenReturn(describeInstancesResultGw);
    Instance instance1 = Mockito.mock(Instance.class);
    when(instance1.getInstanceId()).thenReturn("i-1");
    when(instance1.getPrivateIpAddress()).thenReturn("privateIp1");
    when(instance1.getPublicIpAddress()).thenReturn("publicIp1");
    Instance instance2 = Mockito.mock(Instance.class);
    when(instance2.getInstanceId()).thenReturn("i-2");
    when(instance2.getPrivateIpAddress()).thenReturn("privateIp2");
    when(instance2.getPublicIpAddress()).thenReturn("publicIp2");
    List<Reservation> gatewayReservations = Collections.singletonList(getReservation(instance1, instance2));
    when(describeInstancesResultGw.getReservations()).thenReturn(gatewayReservations);
    AuthenticatedContext ac = authenticatedContext();
    List<CloudVmMetaDataStatus> statuses = awsMetadataCollector.collect(ac, null, vms);
    verify(amazonEC2Client, times(2)).createTags(any(CreateTagsRequest.class));
    Assert.assertEquals(2, statuses.size());
    Assert.assertEquals("i-1", statuses.get(0).getCloudVmInstanceStatus().getCloudInstance().getInstanceId());
    Assert.assertEquals("privateIp1", statuses.get(0).getMetaData().getPrivateIp());
    Assert.assertEquals("publicIp1", statuses.get(0).getMetaData().getPublicIp());
    Assert.assertEquals("i-2", statuses.get(1).getCloudVmInstanceStatus().getCloudInstance().getInstanceId());
    Assert.assertEquals("privateIp2", statuses.get(1).getMetaData().getPrivateIp());
    Assert.assertEquals("publicIp2", statuses.get(1).getMetaData().getPublicIp());
}

Example 55 with Region

use of com.amazonaws.services.ec2.model.Region in project ice by Netflix.

the class ReservationCapacityPoller method poll.

@Override
protected void poll() throws Exception {
    ProcessorConfig config = ProcessorConfig.getInstance();
    // read from s3 if not exists
    File file = new File(config.localDir, "reservation_capacity.txt");
    if (!file.exists()) {
        logger.info("downloading " + file + "...");
        AwsUtils.downloadFileIfNotExist(config.workS3BucketName, config.workS3BucketPrefix, file);
        logger.info("downloaded " + file);
    }
    // read from file
    Map<String, ReservedInstances> reservations = Maps.newTreeMap();
    if (file.exists()) {
        BufferedReader reader = null;
        try {
            reader = new BufferedReader(new FileReader(file));
            String line;
            while ((line = reader.readLine()) != null) {
                String[] tokens = line.split(",");
                String accountId = tokens[0];
                String region = tokens[1];
                String reservationId = tokens[2];
                String zone = tokens[3];
                Long start = Long.parseLong(tokens[4]);
                long duration = Long.parseLong(tokens[5]);
                String instanceType = tokens[6];
                String productDescription = tokens[7];
                int instanceCount = Integer.parseInt(tokens[8]);
                String offeringType = tokens[9];
                String state = tokens[10];
                Long end = tokens.length > 11 ? Long.parseLong(tokens[11]) : null;
                float fixedPrice = tokens.length > 12 ? Float.parseFloat(tokens[12]) : 0;
                float usagePrice = tokens.length > 13 ? Float.parseFloat(tokens[13]) : 0;
                ReservedInstances reservation = new ReservedInstances().withAvailabilityZone(zone).withStart(new Date(start)).withDuration(duration).withInstanceType(instanceType).withProductDescription(productDescription).withInstanceCount(instanceCount).withOfferingType(offeringType).withState(state).withFixedPrice(fixedPrice).withUsagePrice(usagePrice);
                if (end != null)
                    reservation.setEnd(new Date(end));
                else
                    reservation.setEnd(new Date(start + duration * 1000));
                reservations.put(accountId + "," + region + "," + reservationId, reservation);
            }
        } catch (Exception e) {
            logger.error("error in reading " + file, e);
        } finally {
            if (reader != null)
                try {
                    reader.close();
                } catch (Exception e) {
                }
        }
    }
    logger.info("read " + reservations.size() + " reservations.");
    for (Account account : config.accountService.getReservationAccounts().keySet()) {
        try {
            AmazonEC2Client ec2Client;
            String assumeRole = config.accountService.getReservationAccessRoles().get(account);
            if (assumeRole != null) {
                String externalId = config.accountService.getReservationAccessExternalIds().get(account);
                final Credentials credentials = AwsUtils.getAssumedCredentials(account.id, assumeRole, externalId);
                ec2Client = new AmazonEC2Client(new AWSSessionCredentials() {

                    public String getAWSAccessKeyId() {
                        return credentials.getAccessKeyId();
                    }

                    public String getAWSSecretKey() {
                        return credentials.getSecretAccessKey();
                    }

                    public String getSessionToken() {
                        return credentials.getSessionToken();
                    }
                });
            } else
                ec2Client = new AmazonEC2Client(AwsUtils.awsCredentialsProvider.getCredentials(), AwsUtils.clientConfig);
            for (Region region : Region.getAllRegions()) {
                // just ignore GovCloud when polling for RIs in order to prevent AuthFailure errors.
                if (region == Region.US_GOV_WEST_1) {
                    continue;
                }
                ec2Client.setEndpoint("ec2." + region.name + ".amazonaws.com");
                try {
                    DescribeReservedInstancesResult result = ec2Client.describeReservedInstances();
                    for (ReservedInstances reservation : result.getReservedInstances()) {
                        String key = account.id + "," + region.name + "," + reservation.getReservedInstancesId();
                        reservations.put(key, reservation);
                        if (reservation.getEnd() == null)
                            reservation.setEnd(new Date(reservation.getStart().getTime() + reservation.getDuration() * 1000L));
                        if (reservation.getFixedPrice() == null)
                            reservation.setFixedPrice(0f);
                        if (reservation.getUsagePrice() == null)
                            reservation.setUsagePrice(0f);
                    }
                } catch (Exception e) {
                    logger.error("error in describeReservedInstances for " + region.name + " " + account.name, e);
                }
            }
            ec2Client.shutdown();
        } catch (Exception e) {
            logger.error("Error in describeReservedInstances for " + account.name, e);
        }
    }
    config.reservationService.updateEc2Reservations(reservations);
    updatedConfig = true;
    // archive to disk
    BufferedWriter writer = null;
    try {
        writer = new BufferedWriter(new FileWriter(file));
        for (String key : reservations.keySet()) {
            ReservedInstances reservation = reservations.get(key);
            String[] line = new String[] { key, reservation.getAvailabilityZone(), reservation.getStart().getTime() + "", reservation.getDuration().toString(), reservation.getInstanceType(), reservation.getProductDescription(), reservation.getInstanceCount().toString(), reservation.getOfferingType(), reservation.getState(), reservation.getEnd().getTime() + "", reservation.getFixedPrice() + "", reservation.getUsagePrice() + "" };
            writer.write(StringUtils.join(line, ","));
            writer.newLine();
        }
    } catch (Exception e) {
        logger.error("", e);
    } finally {
        if (writer != null)
            try {
                writer.close();
            } catch (Exception e) {
            }
    }
    logger.info("archived " + reservations.size() + " reservations.");
    // archive to s3
    logger.info("uploading " + file + "...");
    AwsUtils.upload(config.workS3BucketName, config.workS3BucketPrefix, config.localDir, file.getName());
    logger.info("uploaded " + file);
}