Examples with EmrClusterDefinition org.finra.herd.model.api.xml.EmrClusterDefinition used on opensource projects

Example 51 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class CreateEmrClusterTest method testCreateClusterOverrideJson.

@Test
public void testCreateClusterOverrideJson() throws Exception {
    String clusterName = "testCluster" + Math.random();
    String override = jsonHelper.objectToJson(new EmrClusterDefinition());
    Map<String, Object> variables = createJob(clusterName, "false", "json", override);
    assertEquals("taskStatus", "SUCCESS", variables.get("createClusterServiceTask_taskStatus"));
    assertNotNull("emrClusterId", variables.get("createClusterServiceTask_emrClusterId"));
    assertNotNull("emrClusterStatus", variables.get("createClusterServiceTask_emrClusterStatus"));
    assertNotNull("emrClusterCreated", variables.get("createClusterServiceTask_emrClusterCreated"));
    assertNotNull("emrClusterDefinition", variables.get("createClusterServiceTask_emrClusterDefinition"));
    terminateCluster(TEST_ACTIVITI_NAMESPACE_CD, EMR_CLUSTER_DEFINITION_NAME, clusterName);
}

Example 52 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPricePickMultipleInstancesAzPricesAreEqual.

/**
 * Tests case where instance count affect subnet selection. Tests case given - master instance is very expensive in one AZ but cheaper in the other - core
 * instance is very cheap in one AZ but more expensive in the other When - Enough instances are specified such that both AZ's are equal in total costs Then
 * - The result subnet is arbitrary, but should not error.
 * <p/>
 * Test case reference ClusterSpotPriceAlgorithm 13
 */
@Test
public void testBestPricePickMultipleInstancesAzPricesAreEqual() {
    String subnetId = SUBNET_1 + "," + SUBNET_4;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_2);
    masterInstanceDefinition.setInstanceSpotPrice(SPOT_PRICE_VERY_HIGH);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(2);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    coreInstanceDefinition.setInstanceSpotPrice(ON_DEMAND);
    InstanceDefinition taskInstanceDefinition = new InstanceDefinition();
    taskInstanceDefinition.setInstanceCount(2);
    taskInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    taskInstanceDefinition.setInstanceSpotPrice(ON_DEMAND);
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertEquals("master instance bid price", SPOT_PRICE_VERY_HIGH, emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice());
    assertEquals("core instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice());
    assertEquals("task instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getTaskInstances().getInstanceSpotPrice());
    assertTrue("selected subnet was neither SUBNET_1 or SUBNET_4", SUBNET_1.equals(emrClusterDefinition.getSubnetId()) || SUBNET_4.equals(emrClusterDefinition.getSubnetId()));
}

Example 53 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceAlgorithmicMaxSearchPriceEqualsToOnDemandAndSpotPriceNotAvailable.

/**
 * Tests algorithmic case when the max search price equals to on-demand price and spot price is not available. Master should pick on-demand because spot is
 * not available and max >= on-demand. Core spot > on-demand, max search price = spot - Core should pick on-demand since on-demand is cheaper.
 */
@Test
public void testBestPriceAlgorithmicMaxSearchPriceEqualsToOnDemandAndSpotPriceNotAvailable() {
    String subnetId = SUBNET_1;
    // For master instance definition, use instance type that does not have spot price available.
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_4);
    masterInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(1);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_3);
    coreInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND);
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertNull("master instance was not on-demand", emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice());
    assertNull("core instance was not on-demand", emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice());
}

Example 54 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPricePickMultipleInstancesSelectCheaperCore.

/**
 * Tests case where instance count affect subnet selection. Tests case given - core instance is very expensive in one AZ but cheaper in the other - master
 * instance is very cheap in one AZ but more expensive in the other When - Enough instances are specified such that the AZ with the expensive core is
 * selected
 * <p/>
 * Even though the core is more expensive, the master is cheap enough to warrant the use of the expensive core AZ.
 * <p/>
 * Test case reference ClusterSpotPriceAlgorithm 13
 */
@Test
public void testBestPricePickMultipleInstancesSelectCheaperCore() {
    String subnetId = SUBNET_1 + "," + SUBNET_4;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    masterInstanceDefinition.setInstanceSpotPrice(ON_DEMAND);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(1);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_2);
    coreInstanceDefinition.setInstanceSpotPrice(SPOT_PRICE_VERY_HIGH);
    InstanceDefinition taskInstanceDefinition = new InstanceDefinition();
    taskInstanceDefinition.setInstanceCount(1);
    taskInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    taskInstanceDefinition.setInstanceSpotPrice(ON_DEMAND);
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertEquals("master instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice());
    assertEquals("core instance bid price", SPOT_PRICE_VERY_HIGH, emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice());
    assertEquals("task instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getTaskInstances().getInstanceSpotPrice());
    assertEquals("selected subnet", SUBNET_1, emrClusterDefinition.getSubnetId());
}

Example 55 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testCoreInstanceCount0SubnetInMultipleAzAssertSuccess.

@Test
public void testCoreInstanceCount0SubnetInMultipleAzAssertSuccess() {
    String subnetId = SUBNET_1 + "," + SUBNET_3;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(0);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    // we select the pricing randomly so either one can be chosen
    assertTrue(Arrays.asList(SUBNET_1, SUBNET_3).contains(emrClusterDefinition.getSubnetId()));
}