Examples with MasterInstanceDefinition org.finra.herd.model.api.xml.MasterInstanceDefinition used on opensource projects

Example 31 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceCheapestPriceDoenstHaveEnoughIp.

/**
 * Test case when multiple subnets are specified, but the subnet with AZ with the cheapest price does not have enough IP addresses available, the algorithm
 * should choose the subnet with enough availability, even when it is not the cheapest.
 * <p/>
 * Test case reference ClusterSpotPriceAlgorithm 10
 */
@Test
public void testBestPriceCheapestPriceDoenstHaveEnoughIp() {
    String subnetId = SUBNET_1 + "," + SUBNET_3;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(5);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    masterInstanceDefinition.setInstanceSpotPrice(ON_DEMAND);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(6);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    coreInstanceDefinition.setInstanceSpotPrice(ON_DEMAND);
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertEquals("master instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice());
    assertEquals("core instance bid price", ON_DEMAND, emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice());
    assertEquals("selected subnet", SUBNET_3, emrClusterDefinition.getSubnetId());
}

Example 32 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceSubnetPermutations.

/**
 * Tests case where subnet list contains - Valid subnet with whitespace padding - Empty string - String with only whitespaces - Trailing and leading commas
 * <p/>
 * We expect that only the valid subnet should be used, after trimming. All other blank values should be ignored.
 */
@Test
public void testBestPriceSubnetPermutations() {
    String subnetId = ", \n\t\r" + SUBNET_1 + " \n\t\r,, \n\t\r,";
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(1);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertEquals("selected subnet", SUBNET_1, emrClusterDefinition.getSubnetId());
}

Example 33 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceAlgorithmicThresholdAboveOnDemandAndSearchBelowOnDemand.

/**
 * Tests 2 cases: Master instance spot < on-demand, search = on-demand, threshold > on-demand - Master should use on-demand since on-demand is above
 * threshold Core instance spot < on-demand, search < on-demand, threshold < on-demand - Core should use spot since on-demand is above max
 * <p/>
 * Test case reference ClusterSpotPriceAlgorithm 6, 7
 */
@Test
public void testBestPriceAlgorithmicThresholdAboveOnDemandAndSearchBelowOnDemand() {
    String subnetId = SUBNET_1;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    masterInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND);
    masterInstanceDefinition.setInstanceOnDemandThreshold(ON_DEMAND.subtract(SPOT_PRICE_LOW).add(ONE_UNIT));
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(1);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    coreInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND_LESS_ONE);
    coreInstanceDefinition.setInstanceOnDemandThreshold(ON_DEMAND_LESS_ONE.subtract(SPOT_PRICE_LOW));
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertNull("master instance was not on-demand", emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice());
    assertEquals("core instance bid price", ON_DEMAND_LESS_ONE, emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice());
}

Example 34 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrDaoTest method createEmrClusterAssertEncryptionDisabled.

@Test
public void createEmrClusterAssertEncryptionDisabled() throws Exception {
    /*
         * Use only minimum required options
         */
    String clusterName = "clusterName";
    EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition();
    InstanceDefinitions instanceDefinitions = new InstanceDefinitions();
    instanceDefinitions.setMasterInstances(new MasterInstanceDefinition(10, "masterInstanceType", NO_EMR_CLUSTER_DEFINITION_EBS_CONFIGURATION, NO_INSTANCE_SPOT_PRICE, NO_INSTANCE_MAX_SEARCH_PRICE, NO_INSTANCE_ON_DEMAND_THRESHOLD));
    instanceDefinitions.setCoreInstances(new InstanceDefinition(20, "coreInstanceType", NO_EMR_CLUSTER_DEFINITION_EBS_CONFIGURATION, NO_INSTANCE_SPOT_PRICE, NO_INSTANCE_MAX_SEARCH_PRICE, NO_INSTANCE_ON_DEMAND_THRESHOLD));
    emrClusterDefinition.setInstanceDefinitions(instanceDefinitions);
    emrClusterDefinition.setNodeTags(Arrays.asList(new NodeTag("tagName", "tagValue")));
    emrClusterDefinition.setEncryptionEnabled(false);
    String clusterId = "clusterId";
    when(mockEmrOperations.runEmrJobFlow(any(), any())).then(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocation) throws Throwable {
            RunJobFlowRequest runJobFlowRequest = invocation.getArgument(1);
            // No bootstrap action should be added
            assertEquals(0, runJobFlowRequest.getBootstrapActions().size());
            return clusterId;
        }
    });
    assertEquals(clusterId, emrDao.createEmrCluster(clusterName, emrClusterDefinition, new AwsParamsDto()));
}

Example 35 with MasterInstanceDefinition

use of org.finra.herd.model.api.xml.MasterInstanceDefinition in project herd by FINRAOS.

the class EmrDaoTest method createEmrClusterAssertCallRunEmrJobFlowRequiredParamsOnly.

@Test
public void createEmrClusterAssertCallRunEmrJobFlowRequiredParamsOnly() throws Exception {
    String clusterName = "clusterName";
    EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition();
    InstanceDefinitions instanceDefinitions = new InstanceDefinitions();
    instanceDefinitions.setMasterInstances(new MasterInstanceDefinition(10, "masterInstanceType", NO_EMR_CLUSTER_DEFINITION_EBS_CONFIGURATION, NO_INSTANCE_SPOT_PRICE, NO_INSTANCE_MAX_SEARCH_PRICE, NO_INSTANCE_ON_DEMAND_THRESHOLD));
    instanceDefinitions.setCoreInstances(new InstanceDefinition(20, "coreInstanceType", NO_EMR_CLUSTER_DEFINITION_EBS_CONFIGURATION, NO_INSTANCE_SPOT_PRICE, NO_INSTANCE_MAX_SEARCH_PRICE, NO_INSTANCE_ON_DEMAND_THRESHOLD));
    emrClusterDefinition.setInstanceDefinitions(instanceDefinitions);
    emrClusterDefinition.setNodeTags(Arrays.asList(new NodeTag("tagName", "tagValue")));
    String clusterId = "clusterId";
    when(mockEmrOperations.runEmrJobFlow(any(), any())).then(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocation) throws Throwable {
            /*
                 * Assert that the given EMR cluster definition produced the correct RunJobFlowRequest
                 */
            RunJobFlowRequest runJobFlowRequest = invocation.getArgument(1);
            JobFlowInstancesConfig jobFlowInstancesConfig = runJobFlowRequest.getInstances();
            List<InstanceGroupConfig> instanceGroupConfigs = jobFlowInstancesConfig.getInstanceGroups();
            assertEquals(2, instanceGroupConfigs.size());
            {
                InstanceGroupConfig instanceGroupConfig = instanceGroupConfigs.get(0);
                assertEquals(10, instanceGroupConfig.getInstanceCount().intValue());
                assertEquals("masterInstanceType", instanceGroupConfig.getInstanceType());
            }
            {
                InstanceGroupConfig instanceGroupConfig = instanceGroupConfigs.get(1);
                assertEquals(20, instanceGroupConfig.getInstanceCount().intValue());
                assertEquals("coreInstanceType", instanceGroupConfig.getInstanceType());
            }
            assertEquals(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_EC2_NODE_IAM_PROFILE_NAME), runJobFlowRequest.getJobFlowRole());
            assertEquals(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_SERVICE_IAM_ROLE_NAME), runJobFlowRequest.getServiceRole());
            List<StepConfig> stepConfigs = runJobFlowRequest.getSteps();
            assertEquals(0, stepConfigs.size());
            List<Tag> tags = runJobFlowRequest.getTags();
            assertEquals(1, tags.size());
            {
                Tag tag = tags.get(0);
                assertEquals("tagName", tag.getKey());
                assertEquals("tagValue", tag.getValue());
            }
            return clusterId;
        }
    });
    assertEquals(clusterId, emrDao.createEmrCluster(clusterName, emrClusterDefinition, new AwsParamsDto()));
}