Examples with EmrClusterDefinitionInstanceFleet org.finra.herd.model.api.xml.EmrClusterDefinitionInstanceFleet used on opensource projects

Example 1 with EmrClusterDefinitionInstanceFleet

use of org.finra.herd.model.api.xml.EmrClusterDefinitionInstanceFleet in project herd by FINRAOS.

the class EmrDaoImplTest method testGetInstanceFleets.

@Test
public void testGetInstanceFleets() {
    // Create objects required for testing.
    final String name = STRING_VALUE;
    final String instanceFleetType = STRING_VALUE_2;
    final Integer targetOnDemandCapacity = INTEGER_VALUE;
    final Integer targetSpotCapacity = INTEGER_VALUE_2;
    final List<EmrClusterDefinitionInstanceTypeConfig> emrClusterDefinitionInstanceTypeConfigs = null;
    final EmrClusterDefinitionLaunchSpecifications emrClusterDefinitionLaunchSpecifications = null;
    final EmrClusterDefinitionInstanceFleet emrClusterDefinitionInstanceFleet = new EmrClusterDefinitionInstanceFleet(name, instanceFleetType, targetOnDemandCapacity, targetSpotCapacity, emrClusterDefinitionInstanceTypeConfigs, emrClusterDefinitionLaunchSpecifications);
    // Call the method under test.
    List<InstanceFleetConfig> result = emrDaoImpl.getInstanceFleets(Arrays.asList(emrClusterDefinitionInstanceFleet));
    // Verify the external calls.
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    final List<InstanceTypeConfig> expectedInstanceTypeConfigs = null;
    assertEquals(Arrays.asList(new InstanceFleetConfig().withName(name).withInstanceFleetType(instanceFleetType).withTargetOnDemandCapacity(targetOnDemandCapacity).withTargetSpotCapacity(targetSpotCapacity).withInstanceTypeConfigs(expectedInstanceTypeConfigs).withLaunchSpecifications(null)), result);
}

Example 2 with EmrClusterDefinitionInstanceFleet

use of org.finra.herd.model.api.xml.EmrClusterDefinitionInstanceFleet in project herd by FINRAOS.

the class EmrDaoTest method createEmrClusterAssertCallRunEmrJobFlowWithInstanceFleetAndMultipleSubnets.

@Test
public void createEmrClusterAssertCallRunEmrJobFlowWithInstanceFleetAndMultipleSubnets() throws Exception {
    // Create objects required for testing.
    final String clusterName = "clusterName";
    final String clusterId = "clusterId";
    final String name = STRING_VALUE;
    final String instanceFleetType = STRING_VALUE_2;
    final Integer targetOnDemandCapacity = INTEGER_VALUE;
    final Integer targetSpotCapacity = INTEGER_VALUE_2;
    final List<EmrClusterDefinitionInstanceTypeConfig> emrClusterDefinitionInstanceTypeConfigs = null;
    final EmrClusterDefinitionLaunchSpecifications emrClusterDefinitionLaunchSpecifications = null;
    final EmrClusterDefinitionInstanceFleet emrClusterDefinitionInstanceFleet = new EmrClusterDefinitionInstanceFleet(name, instanceFleetType, targetOnDemandCapacity, targetSpotCapacity, emrClusterDefinitionInstanceTypeConfigs, emrClusterDefinitionLaunchSpecifications);
    // Create an EMR cluster definition with instance fleet configuration and multiple EC2 subnet IDs.
    EmrClusterDefinition emrClusterDefinition = new EmrClusterDefinition();
    emrClusterDefinition.setInstanceFleets(Arrays.asList(emrClusterDefinitionInstanceFleet));
    emrClusterDefinition.setSubnetId(String.format("%s , %s  ", EC2_SUBNET, EC2_SUBNET_2));
    emrClusterDefinition.setNodeTags(Arrays.asList(new NodeTag("tagName", "tagValue")));
    when(mockEmrOperations.runEmrJobFlow(any(), any())).then(new Answer<String>() {

        @Override
        public String answer(InvocationOnMock invocation) throws Throwable {
            // Assert that the given EMR cluster definition produced the correct RunJobFlowRequest.
            RunJobFlowRequest runJobFlowRequest = invocation.getArgument(1);
            JobFlowInstancesConfig jobFlowInstancesConfig = runJobFlowRequest.getInstances();
            assertEquals(0, CollectionUtils.size(jobFlowInstancesConfig.getInstanceGroups()));
            final List<InstanceTypeConfig> expectedInstanceTypeConfigs = null;
            assertEquals(Arrays.asList(new InstanceFleetConfig().withName(name).withInstanceFleetType(instanceFleetType).withTargetOnDemandCapacity(targetOnDemandCapacity).withTargetSpotCapacity(targetSpotCapacity).withInstanceTypeConfigs(expectedInstanceTypeConfigs).withLaunchSpecifications(null)), jobFlowInstancesConfig.getInstanceFleets());
            assertNull(jobFlowInstancesConfig.getEc2SubnetId());
            assertEquals(2, CollectionUtils.size(jobFlowInstancesConfig.getEc2SubnetIds()));
            assertTrue(jobFlowInstancesConfig.getEc2SubnetIds().contains(EC2_SUBNET));
            assertTrue(jobFlowInstancesConfig.getEc2SubnetIds().contains(EC2_SUBNET_2));
            assertEquals(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_EC2_NODE_IAM_PROFILE_NAME), runJobFlowRequest.getJobFlowRole());
            assertEquals(herdStringHelper.getRequiredConfigurationValue(ConfigurationValue.EMR_DEFAULT_SERVICE_IAM_ROLE_NAME), runJobFlowRequest.getServiceRole());
            List<StepConfig> stepConfigs = runJobFlowRequest.getSteps();
            assertEquals(0, stepConfigs.size());
            List<Tag> tags = runJobFlowRequest.getTags();
            assertEquals(1, tags.size());
            {
                Tag tag = tags.get(0);
                assertEquals("tagName", tag.getKey());
                assertEquals("tagValue", tag.getValue());
            }
            return clusterId;
        }
    });
    assertEquals(clusterId, emrDao.createEmrCluster(clusterName, emrClusterDefinition, new AwsParamsDto()));
}

Example 3 with EmrClusterDefinitionInstanceFleet

use of org.finra.herd.model.api.xml.EmrClusterDefinitionInstanceFleet in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterOverrideInstanceFleetsWithInstanceDefinitions.

@Test
public void testCreateEmrClusterOverrideInstanceFleetsWithInstanceDefinitions() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Create an EMR cluster definition that uses instance fleets instead of instance definitions.
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    InstanceDefinitions instanceDefinitions = expectedEmrClusterDefinition.getInstanceDefinitions();
    expectedEmrClusterDefinition.setInstanceDefinitions(null);
    expectedEmrClusterDefinition.setInstanceFleets(Arrays.asList(new EmrClusterDefinitionInstanceFleet()));
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, xmlHelper.objectToXml(expectedEmrClusterDefinition));
    // Update the count of master instances in the instance definitions.
    instanceDefinitions.getMasterInstances().setInstanceCount(instanceDefinitions.getMasterInstances().getInstanceCount() + 1);
    // Create a new EMR cluster create request that replaces instance fleets with instance definitions.
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    emrClusterDefinitionOverride.setInstanceDefinitions(instanceDefinitions);
    emrClusterDefinitionOverride.setInstanceFleets(new ArrayList<>());
    request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
    // Update the expected EMR cluster definition.
    expectedEmrClusterDefinition.setInstanceDefinitions(instanceDefinitions);
    expectedEmrClusterDefinition.setInstanceFleets(new ArrayList<>());
    // Create the cluster.
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertNotNull(emrCluster.getId());
    assertNull(emrCluster.isDryRun());
    assertTrue(emrCluster.isEmrClusterCreated());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
}

Example 4 with EmrClusterDefinitionInstanceFleet

use of org.finra.herd.model.api.xml.EmrClusterDefinitionInstanceFleet in project herd by FINRAOS.

the class EmrDaoImpl method getInstanceFleets.

/**
 * Creates an instance fleet configuration that describes the EC2 instances and instance configurations for clusters that use this feature.
 *
 * @param emrClusterDefinitionInstanceFleets the list of instance fleet configurations from the EMR cluster definition
 *
 * @return the instance fleet configuration
 */
protected List<InstanceFleetConfig> getInstanceFleets(List<EmrClusterDefinitionInstanceFleet> emrClusterDefinitionInstanceFleets) {
    List<InstanceFleetConfig> instanceFleets = null;
    if (!CollectionUtils.isEmpty(emrClusterDefinitionInstanceFleets)) {
        instanceFleets = new ArrayList<>();
        for (EmrClusterDefinitionInstanceFleet emrClusterDefinitionInstanceFleet : emrClusterDefinitionInstanceFleets) {
            if (emrClusterDefinitionInstanceFleet != null) {
                InstanceFleetConfig instanceFleetConfig = new InstanceFleetConfig();
                instanceFleetConfig.setName(emrClusterDefinitionInstanceFleet.getName());
                instanceFleetConfig.setInstanceFleetType(emrClusterDefinitionInstanceFleet.getInstanceFleetType());
                instanceFleetConfig.setTargetOnDemandCapacity(emrClusterDefinitionInstanceFleet.getTargetOnDemandCapacity());
                instanceFleetConfig.setTargetSpotCapacity(emrClusterDefinitionInstanceFleet.getTargetSpotCapacity());
                instanceFleetConfig.setInstanceTypeConfigs(getInstanceTypeConfigs(emrClusterDefinitionInstanceFleet.getInstanceTypeConfigs()));
                instanceFleetConfig.setLaunchSpecifications(getLaunchSpecifications(emrClusterDefinitionInstanceFleet.getLaunchSpecifications()));
                instanceFleets.add(instanceFleetConfig);
            }
        }
    }
    return instanceFleets;
}

Example 5 with EmrClusterDefinitionInstanceFleet

use of org.finra.herd.model.api.xml.EmrClusterDefinitionInstanceFleet in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterWithInstanceFleets.

/**
 * This method tests the scenario where instance fleet configuration is specified
 */
@Test
public void testCreateEmrClusterWithInstanceFleets() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Retrieve the EMR cluster definition.
    String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml);
    // Update the EMR cluster definition to use instance fleets instead of instance group definitions.
    emrClusterDefinition.setInstanceFleets(Arrays.asList(new EmrClusterDefinitionInstanceFleet()));
    emrClusterDefinition.setInstanceDefinitions(null);
    // Create an entity for the the updated EMR cluster definition.
    configXml = xmlHelper.objectToXml(emrClusterDefinition);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);
    // Create a new EMR cluster create request.
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
}