Search in sources :

Example 16 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterOverrideHadoopConfigurations.

@Test
public void testCreateEmrClusterOverrideHadoopConfigurations() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    ArrayList<Serializable> hadoopConfigurations = new ArrayList<>();
    hadoopConfigurations.add(new ConfigurationFiles());
    hadoopConfigurations.add(new KeyValuePairConfigurations());
    emrClusterDefinitionOverride.setHadoopConfigurations(hadoopConfigurations);
    expectedEmrClusterDefinition.setHadoopConfigurations(emrClusterDefinitionOverride.getHadoopConfigurations());
    request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertNotNull(emrCluster.getId());
    assertNull(emrCluster.isDryRun());
    assertTrue(emrCluster.isEmrClusterCreated());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
}
Also used : NamespaceEntity(org.finra.herd.model.jpa.NamespaceEntity) Serializable(java.io.Serializable) EmrClusterDefinition(org.finra.herd.model.api.xml.EmrClusterDefinition) EmrClusterCreateRequest(org.finra.herd.model.api.xml.EmrClusterCreateRequest) ArrayList(java.util.ArrayList) KeyValuePairConfigurations(org.finra.herd.model.api.xml.KeyValuePairConfigurations) EmrCluster(org.finra.herd.model.api.xml.EmrCluster) ConfigurationFiles(org.finra.herd.model.api.xml.ConfigurationFiles) Test(org.junit.Test)

Example 17 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testEmrAddStepsAmazonObjectNotFound.

/**
 * This method tests the error cases for AmazonExceptions for Illegal Argument
 */
@Test(expected = ObjectNotFoundException.class)
public void testEmrAddStepsAmazonObjectNotFound() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    emrService.createCluster(request);
    // Create a new EMR cluster create request.
    EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
    stepRequest.setStepName(MockAwsOperationsHelper.AMAZON_NOT_FOUND);
    emrService.addStepToCluster(stepRequest);
}
Also used : NamespaceEntity(org.finra.herd.model.jpa.NamespaceEntity) EmrClusterCreateRequest(org.finra.herd.model.api.xml.EmrClusterCreateRequest) EmrShellStepAddRequest(org.finra.herd.model.api.xml.EmrShellStepAddRequest) Test(org.junit.Test)

Example 18 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterStartupSteps.

/**
 * This method tests the cluster creation with the startup hadoop jar steps being added.
 */
@Test
public void testCreateEmrClusterStartupSteps() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertNotNull(emrCluster.getId());
    assertNull(emrCluster.isDryRun());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertTrue(emrCluster.isEmrClusterCreated());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
    ListStepsRequest listStepsRequest = (new ListStepsRequest()).withClusterId(emrCluster.getId());
    ListStepsResult listStepsResult = emrOperations.listStepsRequest(null, listStepsRequest);
    List<StepSummary> addedSteps = listStepsResult.getSteps();
    // Validate that the step was added.
    for (HadoopJarStep hadoopJarStep : expectedEmrClusterDefinition.getHadoopJarSteps()) {
        boolean stepFound = false;
        for (StepSummary stepSummary : addedSteps) {
            if (stepSummary.getName().equals(hadoopJarStep.getStepName())) {
                // Step found
                stepFound = true;
                break;
            }
        }
        assertTrue(stepFound);
    }
}
Also used : ListStepsResult(com.amazonaws.services.elasticmapreduce.model.ListStepsResult) NamespaceEntity(org.finra.herd.model.jpa.NamespaceEntity) EmrClusterDefinition(org.finra.herd.model.api.xml.EmrClusterDefinition) EmrClusterCreateRequest(org.finra.herd.model.api.xml.EmrClusterCreateRequest) StepSummary(com.amazonaws.services.elasticmapreduce.model.StepSummary) EmrCluster(org.finra.herd.model.api.xml.EmrCluster) ListStepsRequest(com.amazonaws.services.elasticmapreduce.model.ListStepsRequest) HadoopJarStep(org.finra.herd.model.api.xml.HadoopJarStep) EmrHadoopJarStep(org.finra.herd.model.api.xml.EmrHadoopJarStep) Test(org.junit.Test)

Example 19 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterSecurityConfigurationReleaseLabelWithPrefix.

@Test
public void testCreateEmrClusterSecurityConfigurationReleaseLabelWithPrefix() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml);
    // Set the security configuration along with the EMR release label starting with an "emr-" prefix.
    // This is needed since security configuration is not supported prior to EMR version 4.8.0.
    emrClusterDefinition.setSecurityConfiguration("securityConfiguration");
    emrClusterDefinition.setReleaseLabel("emr-4.8.0");
    configXml = xmlHelper.objectToXml(emrClusterDefinition);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);
    // Create a new EMR cluster create request.
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    emrService.createCluster(request);
}
Also used : NamespaceEntity(org.finra.herd.model.jpa.NamespaceEntity) EmrClusterDefinition(org.finra.herd.model.api.xml.EmrClusterDefinition) EmrClusterCreateRequest(org.finra.herd.model.api.xml.EmrClusterCreateRequest) Test(org.junit.Test)

Example 20 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterDryRunTrueNoOverride.

@Test
public void testCreateEmrClusterDryRunTrueNoOverride() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    // with dry run set to true
    request.setDryRun(true);
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertNull(emrCluster.getId());
    assertNotNull(emrCluster.isDryRun());
    assertTrue(emrCluster.isDryRun());
    assertFalse(emrCluster.isEmrClusterCreated());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    assertEmrClusterCreationLogNotExist(emrCluster);
}
Also used : NamespaceEntity(org.finra.herd.model.jpa.NamespaceEntity) EmrClusterDefinition(org.finra.herd.model.api.xml.EmrClusterDefinition) EmrClusterCreateRequest(org.finra.herd.model.api.xml.EmrClusterCreateRequest) EmrCluster(org.finra.herd.model.api.xml.EmrCluster) Test(org.junit.Test)

Aggregations

EmrClusterCreateRequest (org.finra.herd.model.api.xml.EmrClusterCreateRequest)68 Test (org.junit.Test)64 NamespaceEntity (org.finra.herd.model.jpa.NamespaceEntity)59 EmrClusterDefinition (org.finra.herd.model.api.xml.EmrClusterDefinition)39 EmrCluster (org.finra.herd.model.api.xml.EmrCluster)33 EmrClusterAlternateKeyDto (org.finra.herd.model.dto.EmrClusterAlternateKeyDto)10 EmrShellStepAddRequest (org.finra.herd.model.api.xml.EmrShellStepAddRequest)9 ArrayList (java.util.ArrayList)6 EmrMasterSecurityGroupAddRequest (org.finra.herd.model.api.xml.EmrMasterSecurityGroupAddRequest)6 EmrClusterDefinitionInstanceFleet (org.finra.herd.model.api.xml.EmrClusterDefinitionInstanceFleet)3 EmrHadoopJarStep (org.finra.herd.model.api.xml.EmrHadoopJarStep)3 EmrHadoopJarStepAddRequest (org.finra.herd.model.api.xml.EmrHadoopJarStepAddRequest)3 EmrShellStep (org.finra.herd.model.api.xml.EmrShellStep)3 Serializable (java.io.Serializable)2 Method (java.lang.reflect.Method)2 HashMap (java.util.HashMap)2 ObjectNotFoundException (org.finra.herd.model.ObjectNotFoundException)2 EmrHiveStepAddRequest (org.finra.herd.model.api.xml.EmrHiveStepAddRequest)2 EmrMasterSecurityGroup (org.finra.herd.model.api.xml.EmrMasterSecurityGroup)2 EmrPigStepAddRequest (org.finra.herd.model.api.xml.EmrPigStepAddRequest)2