Examples with EmrClusterDefinition org.finra.herd.model.api.xml.EmrClusterDefinition used on opensource projects

Example 6 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrPricingHelperTest method testBestPriceAlgorithmicOnDemandOverMaxAndSpotGreaterThanOnDemand.

/**
 * Tests 2 cases: Master spot is less than on-demand, max search price is less than on-demand, threshold is greater than on-demand. - Master should pick
 * spot because even though on-demand is within the threshold, it is above the max Core spot > on-demand, max search price = spot - Core should pick
 * on-demand since on-demand is cheaper
 * <p/>
 * Test case reference ClusterSpotPriceAlgorithm 4, 5
 */
@Test
public void testBestPriceAlgorithmicOnDemandOverMaxAndSpotGreaterThanOnDemand() {
    String subnetId = SUBNET_1;
    MasterInstanceDefinition masterInstanceDefinition = new MasterInstanceDefinition();
    masterInstanceDefinition.setInstanceCount(1);
    masterInstanceDefinition.setInstanceType(INSTANCE_TYPE_1);
    masterInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND_LESS_ONE);
    masterInstanceDefinition.setInstanceOnDemandThreshold(SPOT_PRICE_LOW_PLUS_ONE);
    InstanceDefinition coreInstanceDefinition = new InstanceDefinition();
    coreInstanceDefinition.setInstanceCount(1);
    coreInstanceDefinition.setInstanceType(INSTANCE_TYPE_3);
    coreInstanceDefinition.setInstanceMaxSearchPrice(ON_DEMAND);
    InstanceDefinition taskInstanceDefinition = null;
    EmrClusterDefinition emrClusterDefinition = updateEmrClusterDefinitionWithBestPrice(subnetId, masterInstanceDefinition, coreInstanceDefinition, taskInstanceDefinition);
    assertBestPriceCriteriaRemoved(emrClusterDefinition);
    assertEquals("master instance bid price", ON_DEMAND_LESS_ONE, emrClusterDefinition.getInstanceDefinitions().getMasterInstances().getInstanceSpotPrice());
    assertNull("core instance was not on-demand", emrClusterDefinition.getInstanceDefinitions().getCoreInstances().getInstanceSpotPrice());
}

Example 7 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrClusterDefinitionServiceImpl method createEmrClusterDefinitionFromEntity.

/**
 * Creates the EMR cluster definition information from the persisted entity.
 *
 * @param emrClusterDefinitionEntity the EMR cluster definition entity
 *
 * @return the EMR cluster definition information
 */
private EmrClusterDefinitionInformation createEmrClusterDefinitionFromEntity(EmrClusterDefinitionEntity emrClusterDefinitionEntity) throws Exception {
    // Unmarshal EMR cluster definition XML into JAXB object.
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, emrClusterDefinitionEntity.getConfiguration());
    // Create a new instance of EMR cluster definition information.
    EmrClusterDefinitionInformation emrClusterDefinitionInformation = new EmrClusterDefinitionInformation();
    emrClusterDefinitionInformation.setId(emrClusterDefinitionEntity.getId());
    emrClusterDefinitionInformation.setEmrClusterDefinitionKey(new EmrClusterDefinitionKey(emrClusterDefinitionEntity.getNamespace().getCode(), emrClusterDefinitionEntity.getName()));
    emrClusterDefinitionInformation.setEmrClusterDefinition(emrClusterDefinition);
    return emrClusterDefinitionInformation;
}

Example 8 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrDaoTest method createEmrClusterAssertCallRunEmrJobFlowWithInstanceFleetAndMultipleSubnets.

@Test
public void createEmrClusterAssertCallRunEmrJobFlowWithInstanceFleetAndMultipleSubnets() throws Exception {
    // Create objects required for testing.
    final String clusterName = "clusterName";
    final String clusterId = "clusterId";
    final String name = STRING_VALUE;
    final String instanceFleetType = STRING_VALUE_2;
    final Integer targetOnDemandCapacity = INTEGER_VALUE;
    final Integer targetSpotCapacity = INTEGER_VALUE_2;
    final List<EmrClusterDefinitionInstanceTypeConfig> emrClusterDefinitionInstanceTypeConfigs = null;
    final EmrClusterDefinitionLaunchSpecifications emrClusterDefinitionLaunchSpecifications = null;
    final EmrClusterDefinitionInstanceFleet emrClusterDefinitionInstanceFleet = new EmrClusterDefinitionInstanceFleet(name, instanceFleetType, targetOnDemandCapacity, targetSpotCapacity, emrClusterDefinitionInstanceTypeConfigs, emrClusterDefinitionLaunchSpecifications);
    // Create an EMR cluster definition with instance fleet configuration and multiple EC2 subnet IDs.
    EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition();
    emrClusterDefinition.setInstanceFleets(Arrays.asList(emrClusterDefinitionInstanceFleet));
    emrClusterDefinition.setSubnetId(String.format("%s , %s  ", EC2_SUBNET, EC2_SUBNET_2));
    emrClusterDefinition.setNodeTags(Arrays.asList(new NodeTag("tagName", "tagValue")));
    when(mockEmrOperations.runEmrJobFlow(any(), any())).then(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocation) throws Throwable {
            // Assert that the given EMR cluster definition produced the correct RunJobFlowRequest.
            RunJobFlowRequest runJobFlowRequest = invocation.getArgument(1);
            JobFlowInstancesConfig jobFlowInstancesConfig = runJobFlowRequest.getInstances();
            assertEquals(0, CollectionUtils.size(jobFlowInstancesConfig.getInstanceGroups()));
            final List<InstanceTypeConfig> expectedInstanceTypeConfigs = null;
            assertEquals(Arrays.asList(new InstanceFleetConfig().withName(name).withInstanceFleetType(instanceFleetType).withTargetOnDemandCapacity(targetOnDemandCapacity).withTargetSpotCapacity(targetSpotCapacity).withInstanceTypeConfigs(expectedInstanceTypeConfigs).withLaunchSpecifications(null)), jobFlowInstancesConfig.getInstanceFleets());
            assertNull(jobFlowInstancesConfig.getEc2SubnetId());
            assertEquals(2, CollectionUtils.size(jobFlowInstancesConfig.getEc2SubnetIds()));
            assertTrue(jobFlowInstancesConfig.getEc2SubnetIds().contains(EC2_SUBNET));
            assertTrue(jobFlowInstancesConfig.getEc2SubnetIds().contains(EC2_SUBNET_2));
            assertEquals(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_EC2_NODE_IAM_PROFILE_NAME), runJobFlowRequest.getJobFlowRole());
            assertEquals(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_SERVICE_IAM_ROLE_NAME), runJobFlowRequest.getServiceRole());
            List<StepConfig> stepConfigs = runJobFlowRequest.getSteps();
            assertEquals(0, stepConfigs.size());
            List<Tag> tags = runJobFlowRequest.getTags();
            assertEquals(1, tags.size());
            {
                Tag tag = tags.get(0);
                assertEquals("tagName", tag.getKey());
                assertEquals("tagValue", tag.getValue());
            }
            return clusterId;
        }
    });
    assertEquals(clusterId, emrDao.createEmrCluster(clusterName, emrClusterDefinition, new AwsParamsDto()));
}

Example 9 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrDaoTest method createEmrClusterAssertInstallOozieDisabled.

@Test
public void createEmrClusterAssertInstallOozieDisabled() throws Exception {
    /*
         * Use only minimum required options
         */
    String clusterName = "clusterName";
    EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition();
    InstanceDefinitions instanceDefinitions = new InstanceDefinitions();
    instanceDefinitions.setMasterInstances(new MasterInstanceDefinition(10, "masterInstanceType", NO_EMR_CLUSTER_DEFINITION_EBS_CONFIGURATION, NO_INSTANCE_SPOT_PRICE, NO_INSTANCE_MAX_SEARCH_PRICE, NO_INSTANCE_ON_DEMAND_THRESHOLD));
    instanceDefinitions.setCoreInstances(new InstanceDefinition(20, "coreInstanceType", NO_EMR_CLUSTER_DEFINITION_EBS_CONFIGURATION, NO_INSTANCE_SPOT_PRICE, NO_INSTANCE_MAX_SEARCH_PRICE, NO_INSTANCE_ON_DEMAND_THRESHOLD));
    emrClusterDefinition.setInstanceDefinitions(instanceDefinitions);
    emrClusterDefinition.setNodeTags(Arrays.asList(new NodeTag("tagName", "tagValue")));
    emrClusterDefinition.setInstallOozie(false);
    String clusterId = "clusterId";
    when(mockEmrOperations.runEmrJobFlow(any(), any())).then(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocation) throws Throwable {
            RunJobFlowRequest runJobFlowRequest = invocation.getArgument(1);
            // The oozie step should be skipped.
            assertEquals(0, runJobFlowRequest.getSteps().size());
            return clusterId;
        }
    });
    assertEquals(clusterId, emrDao.createEmrCluster(clusterName, emrClusterDefinition, new AwsParamsDto()));
}

Example 10 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class AddEmrMasterSecurityGroupTest method testAddEmrMasterSecurityGroup.

@Test
public void testAddEmrMasterSecurityGroup() throws Exception {
    // Create a pipe-separated list of security group IDs.
    final String securityGroupIds = EC2_SECURITY_GROUP_1 + "|" + EC2_SECURITY_GROUP_2;
    // Create EC2 on-demand pricing entities required for testing.
    ec2OnDemandPricingDaoTestHelper.createEc2OnDemandPricingEntities();
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Create a trusting AWS account.
    trustingAccountDaoTestHelper.createTrustingAccountEntity(AWS_ACCOUNT_ID, AWS_ROLE_ARN);
    // Create an EMR cluster definition.
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    // Create an EMR cluster definition override with an AWS account ID.
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    emrClusterDefinitionOverride.setAccountId(AWS_ACCOUNT_ID);
    // Create an EMR cluster.
    emrService.createCluster(new EmrClusterCreateRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, NO_DRY_RUN, emrClusterDefinitionOverride));
    List<FieldExtension> fieldExtensionList = new ArrayList<>();
    fieldExtensionList.add(buildFieldExtension("namespace", "${namespace}"));
    fieldExtensionList.add(buildFieldExtension("emrClusterDefinitionName", "${emrClusterDefinitionName}"));
    fieldExtensionList.add(buildFieldExtension("emrClusterName", "${emrClusterName}"));
    fieldExtensionList.add(buildFieldExtension("securityGroupIds", "${securityGroupIds}"));
    fieldExtensionList.add(buildFieldExtension("accountId", "${accountId}"));
    List<Parameter> parameters = new ArrayList<>();
    parameters.add(buildParameter("namespace", NAMESPACE));
    parameters.add(buildParameter("emrClusterDefinitionName", EMR_CLUSTER_DEFINITION_NAME));
    parameters.add(buildParameter("emrClusterName", EMR_CLUSTER_NAME));
    parameters.add(buildParameter("securityGroupIds", securityGroupIds));
    parameters.add(buildParameter("accountId", AWS_ACCOUNT_ID));
    Map<String, Object> variableValuesToValidate = new HashMap<>();
    variableValuesToValidate.put(AddEmrMasterSecurityGroup.VARIABLE_EMR_MASTER_SECURITY_GROUPS, securityGroupIds);
    testActivitiServiceTaskSuccess(AddEmrMasterSecurityGroup.class.getCanonicalName(), fieldExtensionList, parameters, variableValuesToValidate);
}