Examples with MasterInstanceDefinition org.finra.herd.model.api.xml.MasterInstanceDefinition used on opensource projects

Example 16 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrPricingHelper method getTotalInstanceCount.

/**
 * Returns the total number of requested instances. Returns the sum of master, core, and task instance counts. Task instance is optional.
 *
 * @param emrClusterDefinition the EMR cluster definition containing the instance definitions
 *
 * @return the total instance count
 */
private int getTotalInstanceCount(EmrClusterDefinition emrClusterDefinition) {
    InstanceDefinition masterInstanceDefinition = getMasterInstanceDefinition(emrClusterDefinition);
    InstanceDefinition coreInstanceDefinition = getCoreInstanceDefinition(emrClusterDefinition);
    InstanceDefinition taskInstanceDefinition = getTaskInstanceDefinition(emrClusterDefinition);
    // Get total count of instances this definition will attempt to create
    int totalInstanceCount = masterInstanceDefinition.getInstanceCount();
    if (coreInstanceDefinition != null) {
        totalInstanceCount += coreInstanceDefinition.getInstanceCount();
    }
    if (taskInstanceDefinition != null) {
        totalInstanceCount += taskInstanceDefinition.getInstanceCount();
    }
    return totalInstanceCount;
}

Example 17 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrPricingHelper method updateEmrClusterDefinitionWithBestPrice.

/**
 * Finds the best price for each master and core instances based on the subnets and master and core instance search parameters given in the definition.
 * <p/>
 * The results of the findings are used to update the given definition.
 * <p/>
 * If the instance's instanceSpotPrice is set, the instance definition will keep that value. If the instance's instanceMaxSearchPrice is set, the best price
 * will be found. If the found price is spot, the instanceSpotPrice will be set to the value of instanceMaxSearchPrice. If the found price is on-demand, the
 * instanceSpotPrice will be removed. The definition's subnetId will be set to the particular subnet which the best price is found. The value will always be
 * replaced by a single subnet ID.
 * <p/>
 * The definition's instanceMaxSearchPrice and instanceOnDemandThreshold will be removed by this operation.
 *
 * @param emrClusterAlternateKeyDto EMR cluster alternate key
 * @param emrClusterDefinition The EMR cluster definition with search criteria, and the definition that will be updated
 * @param awsParamsDto the AWS related parameters for access/secret keys and proxy details
 */
public void updateEmrClusterDefinitionWithBestPrice(EmrClusterAlternateKeyDto emrClusterAlternateKeyDto, EmrClusterDefinition emrClusterDefinition, AwsParamsDto awsParamsDto) {
    EmrVpcPricingState emrVpcPricingState = new EmrVpcPricingState();
    // Get total count of instances this definition will attempt to create
    int totalInstanceCount = getTotalInstanceCount(emrClusterDefinition);
    // Get the subnet information
    List<Subnet> subnets = getSubnets(emrClusterDefinition, awsParamsDto);
    for (Subnet subnet : subnets) {
        emrVpcPricingState.getSubnetAvailableIpAddressCounts().put(subnet.getSubnetId(), subnet.getAvailableIpAddressCount());
    }
    // Filter out subnets with not enough available IPs
    removeSubnetsWithAvailableIpsLessThan(subnets, totalInstanceCount);
    if (subnets.isEmpty()) {
        LOGGER.info(String.format("Insufficient IP availability. namespace=\"%s\" emrClusterDefinitionName=\"%s\" emrClusterName=\"%s\" " + "totalRequestedInstanceCount=%s emrVpcPricingState=%s", emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(), emrClusterAlternateKeyDto.getEmrClusterName(), totalInstanceCount, jsonHelper.objectToJson(emrVpcPricingState)));
        throw new ObjectNotFoundException(String.format("There are no subnets in the current VPC which have sufficient IP addresses available to run your " + "clusters. Try expanding the list of subnets or try again later. requestedInstanceCount=%s%n%s", totalInstanceCount, emrVpcPricingStateFormatter.format(emrVpcPricingState)));
    }
    // Best prices are accumulated in this list
    List<EmrClusterPriceDto> emrClusterPrices = new ArrayList<>();
    InstanceDefinition masterInstanceDefinition = getMasterInstanceDefinition(emrClusterDefinition);
    InstanceDefinition coreInstanceDefinition = getCoreInstanceDefinition(emrClusterDefinition);
    InstanceDefinition taskInstanceDefinition = getTaskInstanceDefinition(emrClusterDefinition);
    Set<String> requestedInstanceTypes = new HashSet<>();
    String masterInstanceType = masterInstanceDefinition.getInstanceType();
    requestedInstanceTypes.add(masterInstanceType);
    if (coreInstanceDefinition != null) {
        String coreInstanceType = coreInstanceDefinition.getInstanceType();
        requestedInstanceTypes.add(coreInstanceType);
    }
    if (taskInstanceDefinition != null) {
        String taskInstanceType = taskInstanceDefinition.getInstanceType();
        requestedInstanceTypes.add(taskInstanceType);
    }
    // Get AZs for the subnets
    for (AvailabilityZone availabilityZone : getAvailabilityZones(subnets, awsParamsDto)) {
        // Create a mapping of instance types to prices for more efficient, in-memory lookup
        // This method also validates that the given instance types are real instance types supported by AWS.
        Map<String, BigDecimal> instanceTypeOnDemandPrices = getInstanceTypeOnDemandPrices(availabilityZone, requestedInstanceTypes);
        // Create a mapping of instance types to prices for more efficient, in-memory lookup
        // When AWS does not return any spot price history for an instance type in an availability zone, the algorithm will not use that availability zone
        // when selecting the lowest price.
        Map<String, BigDecimal> instanceTypeSpotPrices = getInstanceTypeSpotPrices(availabilityZone, requestedInstanceTypes, awsParamsDto);
        emrVpcPricingState.getSpotPricesPerAvailabilityZone().put(availabilityZone.getZoneName(), instanceTypeSpotPrices);
        emrVpcPricingState.getOnDemandPricesPerAvailabilityZone().put(availabilityZone.getZoneName(), instanceTypeOnDemandPrices);
        // Get and compare master price
        BigDecimal masterSpotPrice = instanceTypeSpotPrices.get(masterInstanceType);
        BigDecimal masterOnDemandPrice = instanceTypeOnDemandPrices.get(masterInstanceType);
        Ec2PriceDto masterPrice = getBestInstancePrice(masterSpotPrice, masterOnDemandPrice, masterInstanceDefinition);
        // Get and compare core price
        Ec2PriceDto corePrice = null;
        if (coreInstanceDefinition != null) {
            String coreInstanceType = coreInstanceDefinition.getInstanceType();
            BigDecimal coreSpotPrice = instanceTypeSpotPrices.get(coreInstanceType);
            BigDecimal coreOnDemandPrice = instanceTypeOnDemandPrices.get(coreInstanceType);
            corePrice = getBestInstancePrice(coreSpotPrice, coreOnDemandPrice, coreInstanceDefinition);
        }
        // Get and compare task price
        Ec2PriceDto taskPrice = null;
        if (taskInstanceDefinition != null) {
            String taskInstanceType = taskInstanceDefinition.getInstanceType();
            BigDecimal taskSpotPrice = instanceTypeSpotPrices.get(taskInstanceType);
            BigDecimal taskOnDemandPrice = instanceTypeOnDemandPrices.get(taskInstanceType);
            taskPrice = getBestInstancePrice(taskSpotPrice, taskOnDemandPrice, taskInstanceDefinition);
        }
        // If prices were found
        if (masterPrice != null && (coreInstanceDefinition == null || corePrice != null) && (taskInstanceDefinition == null || taskPrice != null)) {
            // Add the pricing result to the result list
            emrClusterPrices.add(createEmrClusterPrice(availabilityZone, masterPrice, corePrice, taskPrice));
        }
    // If prices were not found for either master or core, this AZ cannot satisfy the search criteria. Ignore this AZ.
    }
    if (emrClusterPrices.isEmpty()) {
        LOGGER.info(String.format("No subnets which satisfied the best price search criteria. namespace=\"%s\" emrClusterDefinitionName=\"%s\" " + "emrClusterName=\"%s\" emrVpcPricingState=%s", emrClusterAlternateKeyDto.getNamespace(), emrClusterAlternateKeyDto.getEmrClusterDefinitionName(), emrClusterAlternateKeyDto.getEmrClusterName(), jsonHelper.objectToJson(emrVpcPricingState)));
        throw new ObjectNotFoundException(String.format("There were no subnets which satisfied your best price search criteria. If you explicitly opted to use spot EC2 instances, please confirm " + "that your instance types support spot pricing. Otherwise, try setting the max price or the on-demand threshold to a higher value.%n%s", emrVpcPricingStateFormatter.format(emrVpcPricingState)));
    }
    // Find the best prices from the result list
    EmrClusterPriceDto bestEmrClusterPrice = getEmrClusterPriceWithLowestCoreInstancePrice(emrClusterPrices);
    // Find the best subnet among the best AZ's
    Subnet bestEmrClusterSubnet = getBestSubnetForAvailabilityZone(bestEmrClusterPrice.getAvailabilityZone(), subnets);
    // Update the definition with the new calculated values
    updateInstanceDefinitionsWithBestPrice(emrClusterDefinition, bestEmrClusterSubnet, bestEmrClusterPrice);
}

Example 18 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrClusterDefinitionHelperTest method createValidEmrClusterDefinition.

/**
 * Creates a EMR cluster definition which does not cause validateEmrClusterDefinitionConfiguration() to throw an exception.
 * <p/>
 * - One subnet is specified - Master, core, and task instances are specified - Instance count, and instance type are specified for each instance
 * definition. - One node tag is specified
 *
 * @return A new instance of {@link EmrClusterDefinition}
 */
private EmrClusterDefinition createValidEmrClusterDefinition() {
    EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition();
    emrClusterDefinition.setSubnetId(MockEc2OperationsImpl.SUBNET_1);
    InstanceDefinitions instanceDefinitions = new InstanceDefinitions();
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(MockEc2OperationsImpl.INSTANCE_TYPE_1);
    instanceDefinitions.setMasterInstances(masterInstanceDefinition);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(1);
    coreInstanceDefinition.setInstanceType(MockEc2OperationsImpl.INSTANCE_TYPE_1);
    instanceDefinitions.setCoreInstances(coreInstanceDefinition);
    InstanceDefinition taskInstanceDefinition = new InstanceDefinition();
    taskInstanceDefinition.setInstanceCount(1);
    taskInstanceDefinition.setInstanceType(MockEc2OperationsImpl.INSTANCE_TYPE_1);
    instanceDefinitions.setTaskInstances(taskInstanceDefinition);
    emrClusterDefinition.setInstanceDefinitions(instanceDefinitions);
    List<NodeTag> nodeTags = new ArrayList<>();
    {
        nodeTags.add(new NodeTag("test_nodeTagName", "test_nodeTagValue"));
    }
    emrClusterDefinition.setNodeTags(nodeTags);
    return emrClusterDefinition;
}

Example 19 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrDaoImplTest method testGetInstanceGroupConfigs.

@Test
public void testGetInstanceGroupConfigs() {
    // Create objects required for testing.
    final Integer instanceCount = 0;
    final InstanceDefinitions instanceDefinitions = new InstanceDefinitions(new MasterInstanceDefinition(), new InstanceDefinition(), new InstanceDefinition());
    // Mock the external calls.
    when(emrHelper.isInstanceDefinitionsEmpty(instanceDefinitions)).thenReturn(false);
    // Call the method under test.
    List<InstanceGroupConfig> result = emrDaoImpl.getInstanceGroupConfigs(instanceDefinitions);
    // Verify the external calls.
    verify(emrHelper).isInstanceDefinitionsEmpty(instanceDefinitions);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(3, CollectionUtils.size(result));
    assertTrue(result.contains(new InstanceGroupConfig(InstanceRoleType.MASTER, null, instanceCount)));
    assertTrue(result.contains(new InstanceGroupConfig(InstanceRoleType.CORE, null, instanceCount)));
    assertTrue(result.contains(new InstanceGroupConfig(InstanceRoleType.TASK, null, instanceCount)));
}

Example 20 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPricePickMultipleInstancesAzPricesAreEqual.

/**
 * Tests case where instance count affect subnet selection. Tests case given - master instance is very expensive in one AZ but cheaper in the other - core
 * instance is very cheap in one AZ but more expensive in the other When - Enough instances are specified such that both AZ's are equal in total costs Then
 * - The result subnet is arbitrary, but should not error.
 * <p/>
 * Test case reference ClusterSpotPriceAlgorithm 13
 */
@Test
public void testBestPricePickMultipleInstancesAzPricesAreEqual() {
    String subnetId = SUBNET_1 + "," + SUBNET_4;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_2);
    masterInstanceDefinition.setInstanceSpotPrice(SPOT_PRICE_VERY_HIGH);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(2);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    coreInstanceDefinition.setInstanceSpotPrice(ON_DEMAND);
    InstanceDefinition taskInstanceDefinition = new InstanceDefinition();
    taskInstanceDefinition.setInstanceCount(2);
    taskInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    taskInstanceDefinition.setInstanceSpotPrice(ON_DEMAND);
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertEquals("master instance bid price", SPOT_PRICE_VERY_HIGH, emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice());
    assertEquals("core instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice());
    assertEquals("task instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getTaskInstances().getInstanceSpotPrice());
    assertTrue("selected subnet was neither SUBNET_1 or SUBNET_4", SUBNET_1.equals(emrClusterDefinition.getSubnetId()) || SUBNET_4.equals(emrClusterDefinition.getSubnetId()));
}