Examples with MasterInstanceDefinition org.finra.herd.model.api.xml.MasterInstanceDefinition used on opensource projects

Example 1 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrClusterDefinitionHelper method validateMasterInstanceDefinition.

/**
 * Converts the given master instance definition to a generic instance definition and delegates to validateInstanceDefinition(). Generates an appropriate
 * error message using the name "master".
 *
 * @param masterInstanceDefinition the master instance definition to validate
 *
 * @throws IllegalArgumentException when any validation error occurs
 */
private void validateMasterInstanceDefinition(MasterInstanceDefinition masterInstanceDefinition) {
    InstanceDefinition instanceDefinition = new InstanceDefinition();
    instanceDefinition.setInstanceCount(masterInstanceDefinition.getInstanceCount());
    instanceDefinition.setInstanceMaxSearchPrice(masterInstanceDefinition.getInstanceMaxSearchPrice());
    instanceDefinition.setInstanceOnDemandThreshold(masterInstanceDefinition.getInstanceOnDemandThreshold());
    instanceDefinition.setInstanceSpotPrice(masterInstanceDefinition.getInstanceSpotPrice());
    instanceDefinition.setInstanceType(masterInstanceDefinition.getInstanceType());
    validateInstanceDefinition("master", instanceDefinition, 1);
}

Example 2 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPricePickBestAz.

/**
 * Tests case where multiple subnets across multiple AZs are specified, and there are price differences. The algorithm should result in selecting the subnet
 * in the AZ with the lowest price.
 * <p/>
 * Test case reference ClusterSpotPriceAlgorithm 12
 */
@Test
public void testBestPricePickBestAz() {
    String subnetId = SUBNET_1 + "," + SUBNET_3;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(4);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    masterInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(5);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    coreInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND);
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertEquals("master instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice());
    assertEquals("core instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice());
    assertEquals("selected subnet", SUBNET_1, emrClusterDefinition.getSubnetId());
}

Example 3 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceSameAzMultipleSubnets.

/**
 * Tests case where multiple subnets are specified, but they all belong to the same AZ. In such case, the subnet with the most available IP addresses should
 * be selected.
 * <p/>
 * Test case reference ClusterSpotPriceAlgorithm 9
 */
@Test
public void testBestPriceSameAzMultipleSubnets() {
    String subnetId = SUBNET_1 + "," + SUBNET_2;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(1);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertEquals("selected subnet", SUBNET_2, emrClusterDefinition.getSubnetId());
}

Example 4 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceAlgorithmicMaxSearchPriceTooLowAndSpotPriceNotAvailable.

/**
 * Tests algorithmic case when the max search price is lower than on-demand price and spot price is not available. The update method should throw an error
 * indicating that no subnets satisfied the given criteria.
 */
@Test
public void testBestPriceAlgorithmicMaxSearchPriceTooLowAndSpotPriceNotAvailable() {
    String subnetId = SUBNET_1;
    // For master instance definition, use instance type that does not have spot price available.
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_4);
    masterInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND_LESS_ONE);
    InstanceDefinition coreInstanceDefinition = null;
    InstanceDefinition taskInstanceDefinition = null;
    // Try with master failing criteria
    try {
        updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
        fail();
    } catch (ObjectNotFoundException e) {
        // Set expected EMR VPC price state.
        EmrVpcPricingState expectedEmrVpcPricingState = new EmrVpcPricingState();
        expectedEmrVpcPricingState.setSubnetAvailableIpAddressCounts(new HashMap<String, Integer>() {

            {
                put(SUBNET_1, 10);
            }
        });
        expectedEmrVpcPricingState.setSpotPricesPerAvailabilityZone(new HashMap<String, Map<String, BigDecimal>>() {

            {
                put(AVAILABILITY_ZONE_1, new HashMap<>());
            }
        });
        expectedEmrVpcPricingState.setOnDemandPricesPerAvailabilityZone(new HashMap<String, Map<String, BigDecimal>>() {

            {
                put(AVAILABILITY_ZONE_1, new HashMap<String, BigDecimal>() {

                    {
                        put(INSTANCE_TYPE_4, ON_DEMAND);
                    }
                });
            }
        });
        assertEquals(String.format("There were no subnets which satisfied your best price search criteria. If you explicitly opted to use spot EC2 instances, please confirm " + "that your instance types support spot pricing. Otherwise, try setting the max price or the on-demand threshold to a higher value.%n%s", emrVpcPricingStateFormatter.format(expectedEmrVpcPricingState)), e.getMessage());
    }
}

Example 5 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceAlgorithmicPickSpotAndOnDemand.

/**
 * Tests algorithmic cases:
 * <p/>
 * Master spot < on-demand and on-demand threshold < on-demand. Therefore master should use spot. Core spot < on-demand and on-demand threshold = on-demand.
 * Therefore core should use on-demand.
 * <p/>
 * Test case reference ClusterSpotPriceAlgorithm 1, 2
 */
@Test
public void testBestPriceAlgorithmicPickSpotAndOnDemand() {
    String subnetId = SUBNET_1;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND);
    masterInstanceDefinition.setInstanceOnDemandThreshold(SPOT_PRICE_LOW_LESS_ONE);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    coreInstanceDefinition.setInstanceCount(1);
    coreInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND);
    coreInstanceDefinition.setInstanceOnDemandThreshold(SPOT_PRICE_LOW);
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertEquals("master instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice());
    assertNull("core instance was not on-demand", emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice());
}