use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.
the class EmrServiceTest method testCreateEmrClusterOverrideHadoopConfigurations.
@Test
public void testCreateEmrClusterOverrideHadoopConfigurations() throws Exception {
// Create the namespace entity.
NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
// Create a new EMR cluster create request
EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
ArrayList<Serializable> hadoopConfigurations = new ArrayList<>();
hadoopConfigurations.add(new ConfigurationFiles());
hadoopConfigurations.add(new KeyValuePairConfigurations());
emrClusterDefinitionOverride.setHadoopConfigurations(hadoopConfigurations);
expectedEmrClusterDefinition.setHadoopConfigurations(emrClusterDefinitionOverride.getHadoopConfigurations());
request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
EmrCluster emrCluster = emrService.createCluster(request);
// Validate the returned object against the input.
assertNotNull(emrCluster);
assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
assertNotNull(emrCluster.getId());
assertNull(emrCluster.isDryRun());
assertTrue(emrCluster.isEmrClusterCreated());
assertNotNull(emrCluster.getEmrClusterDefinition());
assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
}
use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.
the class EmrServiceTest method testEmrAddStepsAmazonObjectNotFound.
/**
* This method tests the error cases for AmazonExceptions for Illegal Argument
*/
@Test(expected = ObjectNotFoundException.class)
public void testEmrAddStepsAmazonObjectNotFound() throws Exception {
// Create the namespace entity.
NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
emrService.createCluster(request);
// Create a new EMR cluster create request.
EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
stepRequest.setStepName(MockAwsOperationsHelper.AMAZON_NOT_FOUND);
emrService.addStepToCluster(stepRequest);
}
use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.
the class EmrServiceTest method testCreateEmrClusterStartupSteps.
/**
* This method tests the cluster creation with the startup hadoop jar steps being added.
*/
@Test
public void testCreateEmrClusterStartupSteps() throws Exception {
// Create the namespace entity.
NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
// Create a new EMR cluster create request
EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
EmrCluster emrCluster = emrService.createCluster(request);
// Validate the returned object against the input.
assertNotNull(emrCluster);
assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
assertNotNull(emrCluster.getId());
assertNull(emrCluster.isDryRun());
assertNotNull(emrCluster.getEmrClusterDefinition());
assertTrue(emrCluster.isEmrClusterCreated());
assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
ListStepsRequest listStepsRequest = (new ListStepsRequest()).withClusterId(emrCluster.getId());
ListStepsResult listStepsResult = emrOperations.listStepsRequest(null, listStepsRequest);
List<StepSummary> addedSteps = listStepsResult.getSteps();
// Validate that the step was added.
for (HadoopJarStep hadoopJarStep : expectedEmrClusterDefinition.getHadoopJarSteps()) {
boolean stepFound = false;
for (StepSummary stepSummary : addedSteps) {
if (stepSummary.getName().equals(hadoopJarStep.getStepName())) {
// Step found
stepFound = true;
break;
}
}
assertTrue(stepFound);
}
}
use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.
the class EmrServiceTest method testCreateEmrClusterSecurityConfigurationReleaseLabelWithPrefix.
@Test
public void testCreateEmrClusterSecurityConfigurationReleaseLabelWithPrefix() throws Exception {
// Create the namespace entity.
NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());
EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml);
// Set the security configuration along with the EMR release label starting with an "emr-" prefix.
// This is needed since security configuration is not supported prior to EMR version 4.8.0.
emrClusterDefinition.setSecurityConfiguration("securityConfiguration");
emrClusterDefinition.setReleaseLabel("emr-4.8.0");
configXml = xmlHelper.objectToXml(emrClusterDefinition);
emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);
// Create a new EMR cluster create request.
EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
emrService.createCluster(request);
}
use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.
the class EmrServiceTest method testCreateEmrClusterDryRunTrueNoOverride.
@Test
public void testCreateEmrClusterDryRunTrueNoOverride() throws Exception {
// Create the namespace entity.
NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
// Create a new EMR cluster create request
EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
// with dry run set to true
request.setDryRun(true);
EmrCluster emrCluster = emrService.createCluster(request);
// Validate the returned object against the input.
assertNotNull(emrCluster);
assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
assertNull(emrCluster.getId());
assertNotNull(emrCluster.isDryRun());
assertTrue(emrCluster.isDryRun());
assertFalse(emrCluster.isEmrClusterCreated());
assertNotNull(emrCluster.getEmrClusterDefinition());
assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
assertEmrClusterCreationLogNotExist(emrCluster);
}
Aggregations