Examples with Subnet com.amazonaws.services.ec2.model.Subnet used on opensource projects

Example 1 with Subnet

use of com.amazonaws.services.ec2.model.Subnet in project herd by FINRAOS.

the class Ec2DaoImpl method getSubnets.

/**
 * This implementation uses the DescribeSubnets API.
 */
@Override
public List<Subnet> getSubnets(Collection<String> subnetIds, AwsParamsDto awsParamsDto) {
    AmazonEC2Client ec2Client = getEc2Client(awsParamsDto);
    DescribeSubnetsRequest describeSubnetsRequest = new DescribeSubnetsRequest();
    describeSubnetsRequest.setSubnetIds(subnetIds);
    try {
        DescribeSubnetsResult describeSubnetsResult = ec2Operations.describeSubnets(ec2Client, describeSubnetsRequest);
        return describeSubnetsResult.getSubnets();
    } catch (AmazonServiceException amazonServiceException) {
        /*
             * AWS throws a 400 error when any one of the specified subnet ID is not found.
             * We want to catch it and throw as an handled herd error as a 404 not found.
             */
        if (ERROR_CODE_SUBNET_ID_NOT_FOUND.equals(amazonServiceException.getErrorCode())) {
            throw new ObjectNotFoundException(amazonServiceException.getErrorMessage(), amazonServiceException);
        } else // Any other type of error we throw as is because they are unexpected.
        {
            throw amazonServiceException;
        }
    }
}

Example 2 with Subnet

use of com.amazonaws.services.ec2.model.Subnet in project herd by FINRAOS.

the class EmrPricingHelper method updateEmrClusterDefinitionWithBestPrice.

/**
 * Finds the best price for each master and core instances based on the subnets and master and core instance search parameters given in the definition.
 * <p/>
 * The results of the findings are used to update the given definition.
 * <p/>
 * If the instance's instanceSpotPrice is set, the instance definition will keep that value. If the instance's instanceMaxSearchPrice is set, the best price
 * will be found. If the found price is spot, the instanceSpotPrice will be set to the value of instanceMaxSearchPrice. If the found price is on-demand, the
 * instanceSpotPrice will be removed. The definition's subnetId will be set to the particular subnet which the best price is found. The value will always be
 * replaced by a single subnet ID.
 * <p/>
 * The definition's instanceMaxSearchPrice and instanceOnDemandThreshold will be removed by this operation.
 *
 * @param emrClusterAlternateKeyDto EMR cluster alternate key
 * @param emrClusterDefinition The EMR cluster definition with search criteria, and the definition that will be updated
 * @param awsParamsDto the AWS related parameters for access/secret keys and proxy details
 */
public void updateEmrClusterDefinitionWithBestPrice(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, EmrClusterDefinition emrClusterDefinition, AwsParamsDto awsParamsDto) {
    EmrVpcPricingState emrVpcPricingState = new EmrVpcPricingState();
    // Get total count of instances this definition will attempt to create
    int totalInstanceCount = getTotalInstanceCount(emrClusterDefinition);
    // Get the subnet information
    List<Subnet> subnets = getSubnets(emrClusterDefinition, awsParamsDto);
    for (Subnet subnet : subnets) {
        emrVpcPricingState.getSubnetAvailableIpAddressCounts().put(subnet.getSubnetId(), subnet.getAvailableIpAddressCount());
    }
    // Filter out subnets with not enough available IPs
    removeSubnetsWithAvailableIpsLessThan(subnets, totalInstanceCount);
    if (subnets.isEmpty()) {
        LOGGER.info(String.format("Insufficient IP availability. namespace=\"%s\" emrClusterDefinitionName=\"%s\" emrClusterName=\"%s\" " + "totalRequestedInstanceCount=%s emrVpcPricingState=%s", emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(), emrClusterAlternateKeyDto.getEmrClusterName(), totalInstanceCount, jsonHelper.objectToJson(emrVpcPricingState)));
        throw new ObjectNotFoundException(String.format("There are no subnets in the current VPC which have sufficient IP addresses available to run your " + "clusters. Try expanding the list of subnets or try again later. requestedInstanceCount=%s%n%s", totalInstanceCount, emrVpcPricingStateFormatter.format(emrVpcPricingState)));
    }
    // Best prices are accumulated in this list
    List<EmrClusterPriceDto> emrClusterPrices = new ArrayList<>();
    InstanceDefinition masterInstanceDefinition = getMasterInstanceDefinition(emrClusterDefinition);
    InstanceDefinition coreInstanceDefinition = getCoreInstanceDefinition(emrClusterDefinition);
    InstanceDefinition taskInstanceDefinition = getTaskInstanceDefinition(emrClusterDefinition);
    Set<String> requestedInstanceTypes = new HashSet<>();
    String masterInstanceType = masterInstanceDefinition.getInstanceType();
    requestedInstanceTypes.add(masterInstanceType);
    if (coreInstanceDefinition != null) {
        String coreInstanceType = coreInstanceDefinition.getInstanceType();
        requestedInstanceTypes.add(coreInstanceType);
    }
    if (taskInstanceDefinition != null) {
        String taskInstanceType = taskInstanceDefinition.getInstanceType();
        requestedInstanceTypes.add(taskInstanceType);
    }
    // Get AZs for the subnets
    for (AvailabilityZone availabilityZone : getAvailabilityZones(subnets, awsParamsDto)) {
        // Create a mapping of instance types to prices for more efficient, in-memory lookup
        // This method also validates that the given instance types are real instance types supported by AWS.
        Map<String, BigDecimal> instanceTypeOnDemandPrices = getInstanceTypeOnDemandPrices(availabilityZone, requestedInstanceTypes);
        // Create a mapping of instance types to prices for more efficient, in-memory lookup
        // When AWS does not return any spot price history for an instance type in an availability zone, the algorithm will not use that availability zone
        // when selecting the lowest price.
        Map<String, BigDecimal> instanceTypeSpotPrices = getInstanceTypeSpotPrices(availabilityZone, requestedInstanceTypes, awsParamsDto);
        emrVpcPricingState.getSpotPricesPerAvailabilityZone().put(availabilityZone.getZoneName(), instanceTypeSpotPrices);
        emrVpcPricingState.getOnDemandPricesPerAvailabilityZone().put(availabilityZone.getZoneName(), instanceTypeOnDemandPrices);
        // Get and compare master price
        BigDecimal masterSpotPrice = instanceTypeSpotPrices.get(masterInstanceType);
        BigDecimal masterOnDemandPrice = instanceTypeOnDemandPrices.get(masterInstanceType);
        Ec2PriceDto masterPrice = getBestInstancePrice(masterSpotPrice, masterOnDemandPrice, masterInstanceDefinition);
        // Get and compare core price
        Ec2PriceDto corePrice = null;
        if (coreInstanceDefinition != null) {
            String coreInstanceType = coreInstanceDefinition.getInstanceType();
            BigDecimal coreSpotPrice = instanceTypeSpotPrices.get(coreInstanceType);
            BigDecimal coreOnDemandPrice = instanceTypeOnDemandPrices.get(coreInstanceType);
            corePrice = getBestInstancePrice(coreSpotPrice, coreOnDemandPrice, coreInstanceDefinition);
        }
        // Get and compare task price
        Ec2PriceDto taskPrice = null;
        if (taskInstanceDefinition != null) {
            String taskInstanceType = taskInstanceDefinition.getInstanceType();
            BigDecimal taskSpotPrice = instanceTypeSpotPrices.get(taskInstanceType);
            BigDecimal taskOnDemandPrice = instanceTypeOnDemandPrices.get(taskInstanceType);
            taskPrice = getBestInstancePrice(taskSpotPrice, taskOnDemandPrice, taskInstanceDefinition);
        }
        // If prices were found
        if (masterPrice != null && (coreInstanceDefinition == null || corePrice != null) && (taskInstanceDefinition == null || taskPrice != null)) {
            // Add the pricing result to the result list
            emrClusterPrices.add(createEmrClusterPrice(availabilityZone, masterPrice, corePrice, taskPrice));
        }
    // If prices were not found for either master or core, this AZ cannot satisfy the search criteria. Ignore this AZ.
    }
    if (emrClusterPrices.isEmpty()) {
        LOGGER.info(String.format("No subnets which satisfied the best price search criteria. namespace=\"%s\" emrClusterDefinitionName=\"%s\" " + "emrClusterName=\"%s\" emrVpcPricingState=%s", emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(), emrClusterAlternateKeyDto.getEmrClusterName(), jsonHelper.objectToJson(emrVpcPricingState)));
        throw new ObjectNotFoundException(String.format("There were no subnets which satisfied your best price search criteria. If you explicitly opted to use spot EC2 instances, please confirm " + "that your instance types support spot pricing. Otherwise, try setting the max price or the on-demand threshold to a higher value.%n%s", emrVpcPricingStateFormatter.format(emrVpcPricingState)));
    }
    // Find the best prices from the result list
    EmrClusterPriceDto bestEmrClusterPrice = getEmrClusterPriceWithLowestCoreInstancePrice(emrClusterPrices);
    // Find the best subnet among the best AZ's
    Subnet bestEmrClusterSubnet = getBestSubnetForAvailabilityZone(bestEmrClusterPrice.getAvailabilityZone(), subnets);
    // Update the definition with the new calculated values
    updateInstanceDefinitionsWithBestPrice(emrClusterDefinition, bestEmrClusterSubnet, bestEmrClusterPrice);
}

Example 3 with Subnet

use of com.amazonaws.services.ec2.model.Subnet in project photon-model by vmware.

the class AWSSubnetTaskServiceTest method createAwsSubnet.

private Subnet createAwsSubnet() {
    if (this.isMock) {
        Subnet subnet = new Subnet();
        subnet.setSubnetId(UUID.randomUUID().toString());
        return subnet;
    }
    CreateSubnetRequest createRequest = new CreateSubnetRequest((String) this.awsTestContext.get(TestAWSSetupUtils.VPC_KEY), AWS_NON_EXISTING_SUBNET_CIDR);
    return this.client.createSubnet(createRequest).getSubnet();
}

Example 4 with Subnet

use of com.amazonaws.services.ec2.model.Subnet in project photon-model by vmware.

the class AWSSubnetTaskServiceTest method testCreateSubnetWithOutboundAccess.

@Test
public void testCreateSubnetWithOutboundAccess() throws Throwable {
    // provision a "public" subnet first
    SubnetState publicSubnetState = provisionSubnet(AWS_NON_EXISTING_PUBLIC_SUBNET_NAME, AWS_NON_EXISTING_PUBLIC_SUBNET_CIDR, null);
    assertNotNull(publicSubnetState.id);
    assertEquals(LifecycleState.READY, publicSubnetState.lifecycleState);
    SubnetState subnetState = provisionSubnet(AWS_NON_EXISTING_SUBNET_NAME, AWS_NON_EXISTING_SUBNET_CIDR, publicSubnetState.documentSelfLink);
    assertNotNull(subnetState.id);
    assertEquals(LifecycleState.READY, subnetState.lifecycleState);
    if (!this.isMock) {
        // Verify that the subnet was created.
        DescribeSubnetsRequest describeRequest = new DescribeSubnetsRequest().withSubnetIds(Collections.singletonList(subnetState.id));
        List<Subnet> subnets = this.client.describeSubnets(describeRequest).getSubnets();
        assertNotNull(subnets);
        assertEquals(1, subnets.size());
        // Verify that a NAT gateway was created
        assertNotNull(subnetState.customProperties);
        String natGatewayId = subnetState.customProperties.get(AWS_NAT_GATEWAY_ID);
        String routeTableId = subnetState.customProperties.get(AWS_ROUTE_TABLE_ID);
        String allocationId = subnetState.customProperties.get(AWS_ELASTIC_IP_ALLOCATION_ID);
        assertNotNull(natGatewayId);
        assertNotNull(routeTableId);
        assertNotNull(allocationId);
        DescribeNatGatewaysRequest describeNatGatewaysRequest = new DescribeNatGatewaysRequest().withNatGatewayIds(Collections.singletonList(natGatewayId));
        List<NatGateway> natGateways = this.client.describeNatGateways(describeNatGatewaysRequest).getNatGateways();
        assertNotNull(natGateways);
        assertEquals(1, natGateways.size());
        NatGateway natGateway = natGateways.get(0);
        assertEquals(publicSubnetState.id, natGateway.getSubnetId());
        assertNotNull(natGateway.getNatGatewayAddresses());
        assertEquals(1, natGateway.getNatGatewayAddresses().size());
        assertEquals(allocationId, natGateway.getNatGatewayAddresses().get(0).getAllocationId());
        assertEquals("available", natGateways.get(0).getState());
        // verify that a route table was created
        DescribeRouteTablesRequest describeRouteTablesRequest = new DescribeRouteTablesRequest().withRouteTableIds(Collections.singletonList(routeTableId));
        List<RouteTable> routeTables = this.client.describeRouteTables(describeRouteTablesRequest).getRouteTables();
        assertNotNull(routeTables);
        assertEquals(1, routeTables.size());
        RouteTable routeTable = routeTables.get(0);
        assertNotNull(routeTable.getAssociations());
        assertEquals(1, routeTable.getAssociations().size());
        assertEquals(subnetState.id, routeTable.getAssociations().get(0).getSubnetId());
        assertNotNull(routeTable.getRoutes());
        assertEquals(2, routeTable.getRoutes().size());
        boolean hasRouteToNatGateway = false;
        for (Route route : routeTable.getRoutes()) {
            if (route.getDestinationCidrBlock().equals("0.0.0.0/0") && route.getNatGatewayId() != null && route.getNatGatewayId().equals(natGatewayId)) {
                hasRouteToNatGateway = true;
                break;
            }
        }
        assertTrue(hasRouteToNatGateway);
        // Verify that an IP address allocation was created
        DescribeAddressesRequest describeAddressesRequest = new DescribeAddressesRequest().withAllocationIds(Collections.singletonList(allocationId));
        List<Address> addresses = this.client.describeAddresses(describeAddressesRequest).getAddresses();
        assertNotNull(addresses);
        assertEquals(1, addresses.size());
    }
    // delete the subnet
    kickOffSubnetProvision(InstanceRequestType.DELETE, subnetState, TaskStage.FINISHED);
    if (!this.isMock) {
        // Verify that the subnet was deleted.
        DescribeSubnetsRequest describeRequest = new DescribeSubnetsRequest().withSubnetIds(Collections.singletonList(subnetState.id));
        try {
            this.client.describeSubnets(describeRequest).getSubnets();
            fail("Subnet should not exist in AWS.");
        } catch (AmazonEC2Exception ex) {
            assertEquals(HttpResponseStatus.BAD_REQUEST.code(), ex.getStatusCode());
        }
        // Verify that the NAT gateway was deleted
        String natGatewayId = subnetState.customProperties.get(AWS_NAT_GATEWAY_ID);
        String routeTableId = subnetState.customProperties.get(AWS_ROUTE_TABLE_ID);
        String allocationId = subnetState.customProperties.get(AWS_ELASTIC_IP_ALLOCATION_ID);
        DescribeNatGatewaysRequest describeNatGatewaysRequest = new DescribeNatGatewaysRequest().withNatGatewayIds(Collections.singletonList(natGatewayId));
        List<NatGateway> natGateways = this.client.describeNatGateways(describeNatGatewaysRequest).getNatGateways();
        assertNotNull(natGateways);
        assertEquals(1, natGateways.size());
        assertEquals("deleted", natGateways.get(0).getState());
        // Verify that the route table was deleted
        DescribeRouteTablesRequest describeRouteTablesRequest = new DescribeRouteTablesRequest().withRouteTableIds(Collections.singletonList(routeTableId));
        try {
            this.client.describeRouteTables(describeRouteTablesRequest).getRouteTables();
            fail("Route table should not exist in AWS.");
        } catch (AmazonEC2Exception ex) {
            assertEquals(HttpResponseStatus.BAD_REQUEST.code(), ex.getStatusCode());
        }
        DescribeAddressesRequest describeAddressesRequest = new DescribeAddressesRequest().withAllocationIds(Collections.singletonList(allocationId));
        try {
            this.client.describeAddresses(describeAddressesRequest).getAddresses();
            fail("IP address allocation should not exist in AWS.");
        } catch (AmazonEC2Exception ex) {
            assertEquals(HttpResponseStatus.BAD_REQUEST.code(), ex.getStatusCode());
        }
    }
}

Example 5 with Subnet

use of com.amazonaws.services.ec2.model.Subnet in project photon-model by vmware.

the class AWSSubnetTaskServiceTest method deleteAwsSubnet.

public void deleteAwsSubnet() {
    if (this.isMock) {
        return;
    }
    DescribeSubnetsRequest subnetRequest = new DescribeSubnetsRequest().withFilters(new Filter(AWS_VPC_ID_FILTER, singletonList((String) this.awsTestContext.get(TestAWSSetupUtils.VPC_KEY)))).withFilters(new Filter(AWS_SUBNET_CIDR_FILTER, singletonList(AWS_NON_EXISTING_SUBNET_CIDR)));
    DescribeSubnetsResult subnetResult = this.client.describeSubnets(subnetRequest);
    subnetResult.getSubnets().forEach(subnet -> {
        DeleteSubnetRequest deleteRequest = new DeleteSubnetRequest(subnet.getSubnetId());
        this.client.deleteSubnet(deleteRequest);
    });
}