Examples with EmrClusterCreateRequest org.finra.herd.model.api.xml.EmrClusterCreateRequest used on opensource projects

Example 1 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class AddEmrMasterSecurityGroupTest method testAddEmrMasterSecurityGroup.

@Test
public void testAddEmrMasterSecurityGroup() throws Exception {
    // Create a pipe-separated list of security group IDs.
    final String securityGroupIds = EC2_SECURITY_GROUP_1 + "|" + EC2_SECURITY_GROUP_2;
    // Create EC2 on-demand pricing entities required for testing.
    ec2OnDemandPricingDaoTestHelper.createEc2OnDemandPricingEntities();
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Create a trusting AWS account.
    trustingAccountDaoTestHelper.createTrustingAccountEntity(AWS_ACCOUNT_ID, AWS_ROLE_ARN);
    // Create an EMR cluster definition.
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    // Create an EMR cluster definition override with an AWS account ID.
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    emrClusterDefinitionOverride.setAccountId(AWS_ACCOUNT_ID);
    // Create an EMR cluster.
    emrService.createCluster(new EmrClusterCreateRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, NO_DRY_RUN, emrClusterDefinitionOverride));
    List<FieldExtension> fieldExtensionList = new ArrayList<>();
    fieldExtensionList.add(buildFieldExtension("namespace", "${namespace}"));
    fieldExtensionList.add(buildFieldExtension("emrClusterDefinitionName", "${emrClusterDefinitionName}"));
    fieldExtensionList.add(buildFieldExtension("emrClusterName", "${emrClusterName}"));
    fieldExtensionList.add(buildFieldExtension("securityGroupIds", "${securityGroupIds}"));
    fieldExtensionList.add(buildFieldExtension("accountId", "${accountId}"));
    List<Parameter> parameters = new ArrayList<>();
    parameters.add(buildParameter("namespace", NAMESPACE));
    parameters.add(buildParameter("emrClusterDefinitionName", EMR_CLUSTER_DEFINITION_NAME));
    parameters.add(buildParameter("emrClusterName", EMR_CLUSTER_NAME));
    parameters.add(buildParameter("securityGroupIds", securityGroupIds));
    parameters.add(buildParameter("accountId", AWS_ACCOUNT_ID));
    Map<String, Object> variableValuesToValidate = new HashMap<>();
    variableValuesToValidate.put(AddEmrMasterSecurityGroup.VARIABLE_EMR_MASTER_SECURITY_GROUPS, securityGroupIds);
    testActivitiServiceTaskSuccess(AddEmrMasterSecurityGroup.class.getCanonicalName(), fieldExtensionList, parameters, variableValuesToValidate);
}

Example 2 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrRestControllerTest method testCreateEmrCluster.

@Test
public void testCreateEmrCluster() throws Exception {
    // Create an EMR cluster definition override.
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    // Create an EMR cluster create request.
    EmrClusterCreateRequest emrClusterCreateRequest = new EmrClusterCreateRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, DRY_RUN, emrClusterDefinitionOverride);
    // Create an EMR cluster.
    EmrCluster emrCluster = new EmrCluster();
    emrCluster.setId(EMR_CLUSTER_ID);
    // Mock the external calls.
    when(emrService.createCluster(emrClusterCreateRequest)).thenReturn(emrCluster);
    // Call the method under test.
    EmrCluster result = emrRestController.createEmrCluster(emrClusterCreateRequest);
    // Verify the external calls.
    verify(emrService).createCluster(emrClusterCreateRequest);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(emrCluster, result);
}

Example 3 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterInstanceNotDefined.

/**
 * This method tests the error cases for Illegal Arguments for Instances not defined properly
 */
@Test
public void testCreateEmrClusterInstanceNotDefined() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Create the test EMR cluster definition entity with missing instance definitions.
    String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_MINIMAL_CLASSPATH).getInputStream());
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml);
    emrClusterDefinition.setInstanceDefinitions(null);
    configXml = xmlHelper.objectToXml(emrClusterDefinition);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);
    // Try to create a new EMR cluster using EMR cluster definition with missing both instance definitions and instance fleets.
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    try {
        emrService.createCluster(request);
        fail();
    } catch (IllegalArgumentException e) {
        assertEquals("Instance group definitions or instance fleets must be specified.", e.getMessage());
    }
}

Example 4 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testAddSecurityGroup.

/**
 * This method tests the happy path scenario for adding security groups
 */
@Test
public void testAddSecurityGroup() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    emrService.createCluster(request);
    // Create the Add security group.
    EmrMasterSecurityGroupAddRequest emrMasterSecurityGroupAddRequest = getNewEmrAddSecurityGroupMasterRequest(request.getEmrClusterName());
    EmrMasterSecurityGroup emrMasterSecurityGroup = emrService.addSecurityGroupsToClusterMaster(emrMasterSecurityGroupAddRequest);
    // Validate the returned object against the input.
    assertNotNull(emrMasterSecurityGroup);
    assertTrue(emrMasterSecurityGroup.getNamespace().equals(request.getNamespace()));
    assertTrue(emrMasterSecurityGroup.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrMasterSecurityGroup.getEmrClusterName().equals(request.getEmrClusterName()));
}

Example 5 with EmrClusterCreateRequest

use of org.finra.herd.model.api.xml.EmrClusterCreateRequest in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterOverrideList.

@Test
public void testCreateEmrClusterOverrideList() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    emrClusterDefinitionOverride.setApplications(Collections.<EmrClusterDefinitionApplication>emptyList());
    expectedEmrClusterDefinition.setApplications(emrClusterDefinitionOverride.getApplications());
    emrClusterDefinitionOverride.setConfigurations(Collections.<EmrClusterDefinitionConfiguration>emptyList());
    expectedEmrClusterDefinition.setConfigurations(emrClusterDefinitionOverride.getConfigurations());
    emrClusterDefinitionOverride.setCustomBootstrapActionAll(Collections.<ScriptDefinition>emptyList());
    expectedEmrClusterDefinition.setCustomBootstrapActionAll(emrClusterDefinitionOverride.getCustomBootstrapActionAll());
    emrClusterDefinitionOverride.setCustomBootstrapActionMaster(Collections.<ScriptDefinition>emptyList());
    expectedEmrClusterDefinition.setCustomBootstrapActionMaster(emrClusterDefinitionOverride.getCustomBootstrapActionMaster());
    emrClusterDefinitionOverride.setDaemonConfigurations(Collections.<Parameter>emptyList());
    expectedEmrClusterDefinition.setDaemonConfigurations(emrClusterDefinitionOverride.getDaemonConfigurations());
    emrClusterDefinitionOverride.setHadoopConfigurations(Collections.<Serializable>emptyList());
    expectedEmrClusterDefinition.setHadoopConfigurations(emrClusterDefinitionOverride.getHadoopConfigurations());
    emrClusterDefinitionOverride.setHadoopJarSteps(Collections.<HadoopJarStep>emptyList());
    expectedEmrClusterDefinition.setHadoopJarSteps(emrClusterDefinitionOverride.getHadoopJarSteps());
    emrClusterDefinitionOverride.setAdditionalMasterSecurityGroups(Collections.emptyList());
    expectedEmrClusterDefinition.setAdditionalMasterSecurityGroups(emrClusterDefinitionOverride.getAdditionalMasterSecurityGroups());
    emrClusterDefinitionOverride.setAdditionalSlaveSecurityGroups(Collections.emptyList());
    expectedEmrClusterDefinition.setAdditionalSlaveSecurityGroups(emrClusterDefinitionOverride.getAdditionalSlaveSecurityGroups());
    List<NodeTag> nodeTags = expectedEmrClusterDefinition.getNodeTags();
    nodeTags.add(new NodeTag("testTag", "test"));
    emrClusterDefinitionOverride.setNodeTags(nodeTags);
    expectedEmrClusterDefinition.setNodeTags(nodeTags);
    request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertNotNull(emrCluster.getId());
    assertNull(emrCluster.isDryRun());
    assertTrue(emrCluster.isEmrClusterCreated());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
}