Examples with EmrClusterDefinition org.finra.herd.model.api.xml.EmrClusterDefinition used on opensource projects

Example 36 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterAdditionalInfo.

/**
 * This method tests additionalInfo
 */
@Test
public void testCreateEmrClusterAdditionalInfo() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml);
    // Set the additional Info
    emrClusterDefinition.setAdditionalInfo("{ami64: \"ami-e82af080\", amiHvm64: \"ami-e82af080\", hadoopVersion: \"2.4.0\", hadoopConfigurationVersion: \"3.1\"}");
    configXml = xmlHelper.objectToXml(emrClusterDefinition);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);
    // Create a new EMR cluster create request.
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    emrService.createCluster(request);
}

Example 37 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterOverrideInstanceFleetsWithInstanceDefinitions.

@Test
public void testCreateEmrClusterOverrideInstanceFleetsWithInstanceDefinitions() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Create an EMR cluster definition that uses instance fleets instead of instance definitions.
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    InstanceDefinitions instanceDefinitions = expectedEmrClusterDefinition.getInstanceDefinitions();
    expectedEmrClusterDefinition.setInstanceDefinitions(null);
    expectedEmrClusterDefinition.setInstanceFleets(Arrays.asList(new EmrClusterDefinitionInstanceFleet()));
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, xmlHelper.objectToXml(expectedEmrClusterDefinition));
    // Update the count of master instances in the instance definitions.
    instanceDefinitions.getMasterInstances().setInstanceCount(instanceDefinitions.getMasterInstances().getInstanceCount() + 1);
    // Create a new EMR cluster create request that replaces instance fleets with instance definitions.
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    emrClusterDefinitionOverride.setInstanceDefinitions(instanceDefinitions);
    emrClusterDefinitionOverride.setInstanceFleets(new ArrayList<>());
    request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
    // Update the expected EMR cluster definition.
    expectedEmrClusterDefinition.setInstanceDefinitions(instanceDefinitions);
    expectedEmrClusterDefinition.setInstanceFleets(new ArrayList<>());
    // Create the cluster.
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertNotNull(emrCluster.getId());
    assertNull(emrCluster.isDryRun());
    assertTrue(emrCluster.isEmrClusterCreated());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
}

Example 38 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrCluster.

/**
 * This method tests the happy path scenario by providing all the parameters
 */
@Test
public void testCreateEmrCluster() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    assertEquals("scaleDownBehavior", expectedEmrClusterDefinition.getScaleDownBehavior());
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertNull(emrCluster.getAccountId());
    assertNotNull(emrCluster.getId());
    assertNull(emrCluster.isDryRun());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertTrue(emrCluster.isEmrClusterCreated());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
}

Example 39 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterWithTaskInstances.

/**
 * This method tests the scenario where task instances are there
 */
@Test
public void testCreateEmrClusterWithTaskInstances() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml);
    InstanceDefinition taskDef = new InstanceDefinition();
    taskDef.setInstanceCount(1);
    // This could be any EC2 instance type supported in EMR.
    taskDef.setInstanceType(MockEc2OperationsImpl.INSTANCE_TYPE_1);
    emrClusterDefinition.getInstanceDefinitions().setTaskInstances(taskDef);
    configXml = xmlHelper.objectToXml(emrClusterDefinition);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);
    // Create a new EMR cluster create request.
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
}

Example 40 with EmrClusterDefinition

use of org.finra.herd.model.api.xml.EmrClusterDefinition in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterBlankParams.

/**
 * This method tests the blank values for various parameters
 */
@Test
public void testCreateEmrClusterBlankParams() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml);
    List<ScriptDefinition> scriptDefinitions = emrClusterDefinition.getCustomBootstrapActionAll();
    scriptDefinitions.get(0).setScriptArguments(null);
    emrClusterDefinition.setCustomBootstrapActionAll(scriptDefinitions);
    scriptDefinitions = emrClusterDefinition.getCustomBootstrapActionMaster();
    scriptDefinitions.get(0).setScriptArguments(null);
    emrClusterDefinition.setCustomBootstrapActionMaster(scriptDefinitions);
    emrClusterDefinition.setVisibleToAll(null);
    emrClusterDefinition.setServiceIamRole(null);
    emrClusterDefinition.setAmiVersion(null);
    emrClusterDefinition.setServiceIamRole(emrClusterDefinition.getEc2NodeIamProfileName());
    emrClusterDefinition.setScaleDownBehavior(null);
    configXml = xmlHelper.objectToXml(emrClusterDefinition);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    emrService.createCluster(request);
}