Examples with EmrClusterDefinition org.finra.herd.model.api.xml.EmrClusterDefinition used on opensource projects

Example 61 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceSubnetIsEmpty.

/**
 * Tests case where subnet is empty. This should default to searching all subnets.
 */
@Test
public void testBestPriceSubnetIsEmpty() {
    String subnetId = "";
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(1);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertTrue("subnet was not selected", StringUtils.isNotBlank(emrClusterDefinition.getSubnetId()));
}

Example 62 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceCheapestPriceDoenstHaveEnoughIp.

/**
 * Test case when multiple subnets are specified, but the subnet with AZ with the cheapest price does not have enough IP addresses available, the algorithm
 * should choose the subnet with enough availability, even when it is not the cheapest.
 * <p/>
 * Test case reference ClusterSpotPriceAlgorithm 10
 */
@Test
public void testBestPriceCheapestPriceDoenstHaveEnoughIp() {
    String subnetId = SUBNET_1 + "," + SUBNET_3;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(5);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    masterInstanceDefinition.setInstanceSpotPrice(ON_DEMAND);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(6);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    coreInstanceDefinition.setInstanceSpotPrice(ON_DEMAND);
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertEquals("master instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice());
    assertEquals("core instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice());
    assertEquals("selected subnet", SUBNET_3, emrClusterDefinition.getSubnetId());
}

Example 63 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceSubnetPermutations.

/**
 * Tests case where subnet list contains - Valid subnet with whitespace padding - Empty string - String with only whitespaces - Trailing and leading commas
 * <p/>
 * We expect that only the valid subnet should be used, after trimming. All other blank values should be ignored.
 */
@Test
public void testBestPriceSubnetPermutations() {
    String subnetId = ", \n\t\r" + SUBNET_1 + " \n\t\r,, \n\t\r,";
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(1);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertEquals("selected subnet", SUBNET_1, emrClusterDefinition.getSubnetId());
}

Example 64 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceAlgorithmicThresholdAboveOnDemandAndSearchBelowOnDemand.

/**
 * Tests 2 cases: Master instance spot < on-demand, search = on-demand, threshold > on-demand - Master should use on-demand since on-demand is above
 * threshold Core instance spot < on-demand, search < on-demand, threshold < on-demand - Core should use spot since on-demand is above max
 * <p/>
 * Test case reference ClusterSpotPriceAlgorithm 6, 7
 */
@Test
public void testBestPriceAlgorithmicThresholdAboveOnDemandAndSearchBelowOnDemand() {
    String subnetId = SUBNET_1;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    masterInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND);
    masterInstanceDefinition.setInstanceOnDemandThreshold(ON_DEMAND.subtract(SPOT_PRICE_LOW).add(ONE_UNIT));
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(1);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    coreInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND_LESS_ONE);
    coreInstanceDefinition.setInstanceOnDemandThreshold(ON_DEMAND_LESS_ONE.subtract(SPOT_PRICE_LOW));
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertNull("master instance was not on-demand", emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice());
    assertEquals("core instance bid price", ON_DEMAND_LESS_ONE, emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice());
}

Example 65 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrDaoTest method createEmrClusterAssertEncryptionDisabled.

@Test
public void createEmrClusterAssertEncryptionDisabled() throws Exception {
    /*
         * Use only minimum required options
         */
    String clusterName = "clusterName";
    EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition();
    InstanceDefinitions instanceDefinitions = new InstanceDefinitions();
    instanceDefinitions.setMasterInstances(new MasterInstanceDefinition(10, "masterInstanceType", NO_EMR_CLUSTER_DEFINITION_EBS_CONFIGURATION, NO_INSTANCE_SPOT_PRICE, NO_INSTANCE_MAX_SEARCH_PRICE, NO_INSTANCE_ON_DEMAND_THRESHOLD));
    instanceDefinitions.setCoreInstances(new InstanceDefinition(20, "coreInstanceType", NO_EMR_CLUSTER_DEFINITION_EBS_CONFIGURATION, NO_INSTANCE_SPOT_PRICE, NO_INSTANCE_MAX_SEARCH_PRICE, NO_INSTANCE_ON_DEMAND_THRESHOLD));
    emrClusterDefinition.setInstanceDefinitions(instanceDefinitions);
    emrClusterDefinition.setNodeTags(Arrays.asList(new NodeTag("tagName", "tagValue")));
    emrClusterDefinition.setEncryptionEnabled(false);
    String clusterId = "clusterId";
    when(mockEmrOperations.runEmrJobFlow(any(), any())).then(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocation) throws Throwable {
            RunJobFlowRequest runJobFlowRequest = invocation.getArgument(1);
            // No bootstrap action should be added
            assertEquals(0, runJobFlowRequest.getBootstrapActions().size());
            return clusterId;
        }
    });
    assertEquals(clusterId, emrDao.createEmrCluster(clusterName, emrClusterDefinition, new AwsParamsDto()));
}