Examples with EmrCluster org.finra.herd.model.api.xml.EmrCluster used on opensource projects

Example 26 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class EmrServiceTest method testGetEmrClusterByIdWithStepId.

/**
 * This method tests the scenario with providing step Id.
 */
@Test
public void testGetEmrClusterByIdWithStepId() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrCluster emrCluster = emrService.createCluster(request);
    // Add a running step
    // Create the Add step request.
    EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
    stepRequest.setStepName(MockEmrOperationsImpl.MOCK_STEP_RUNNING_NAME);
    EmrShellStep emrShellStep = (EmrShellStep) emrService.addStepToCluster(stepRequest);
    String stepId = emrShellStep.getId();
    EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder().withNamespace(NAMESPACE).withEmrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME).withEmrClusterName(request.getEmrClusterName()).build();
    EmrCluster emrClusterGet = emrService.getCluster(emrClusterAlternateKeyDto, emrCluster.getId(), stepId, true, null, false);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertNotNull(emrClusterGet);
    assertTrue(emrCluster.getId().equals(emrClusterGet.getId()));
    assertTrue(emrCluster.getNamespace().equals(emrClusterGet.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(emrClusterGet.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(emrClusterGet.getEmrClusterName()));
    assertTrue(stepId.equals(emrClusterGet.getActiveStep().getId()));
    assertTrue(stepId.equals(emrClusterGet.getStep().getId()));
    // Test the non verbose flow
    emrClusterGet = emrService.getCluster(emrClusterAlternateKeyDto, emrCluster.getId(), stepId, false, null, false);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertNotNull(emrClusterGet);
    assertTrue(emrCluster.getId().equals(emrClusterGet.getId()));
    assertTrue(emrCluster.getNamespace().equals(emrClusterGet.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(emrClusterGet.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(emrClusterGet.getEmrClusterName()));
    assertTrue(stepId.equals(emrClusterGet.getActiveStep().getId()));
    assertTrue(stepId.equals(emrClusterGet.getStep().getId()));
}

Example 27 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterDryRunFalseNoOverride.

@Test
public void testCreateEmrClusterDryRunFalseNoOverride() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    // with dry run set to false
    request.setDryRun(false);
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertNotNull(emrCluster.getId());
    assertNotNull(emrCluster.isDryRun());
    assertFalse(emrCluster.isDryRun());
    assertTrue(emrCluster.isEmrClusterCreated());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
}

Example 28 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterWithInstanceFleets.

/**
 * This method tests the scenario where instance fleet configuration is specified
 */
@Test
public void testCreateEmrClusterWithInstanceFleets() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Retrieve the EMR cluster definition.
    String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml);
    // Update the EMR cluster definition to use instance fleets instead of instance group definitions.
    emrClusterDefinition.setInstanceFleets(Arrays.asList(new EmrClusterDefinitionInstanceFleet()));
    emrClusterDefinition.setInstanceDefinitions(null);
    // Create an entity for the the updated EMR cluster definition.
    configXml = xmlHelper.objectToXml(emrClusterDefinition);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);
    // Create a new EMR cluster create request.
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
}

Example 29 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterWithSecurityGroups.

@Test
public void testCreateEmrClusterWithSecurityGroups() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Retrieve the EMR cluster definition.
    String configXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition emrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, configXml);
    // Update the EMR cluster definition to use instance fleets instead of instance group definitions.
    emrClusterDefinition.setMasterSecurityGroup(EMR_MASTER_SECURITY_GROUP);
    emrClusterDefinition.setSlaveSecurityGroup(EMR_SLAVE_SECURITY_GROUP);
    // Create an entity for the the updated EMR cluster definition.
    configXml = xmlHelper.objectToXml(emrClusterDefinition);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, configXml);
    // Create a new EMR cluster create request.
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinition().getMasterSecurityGroup().equals(EMR_MASTER_SECURITY_GROUP));
    assertTrue(emrCluster.getEmrClusterDefinition().getSlaveSecurityGroup().equals(EMR_SLAVE_SECURITY_GROUP));
}

Example 30 with EmrCluster

use of org.finra.herd.model.api.xml.EmrCluster in project herd by FINRAOS.

the class EmrServiceTest method testCreateEmrClusterOverrideInstanceDefinitionsWithInstanceFleets.

@Test
public void testCreateEmrClusterOverrideInstanceDefinitionsWithInstanceFleets() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    String definitionXml = IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream());
    EmrClusterDefinition expectedEmrClusterDefinition = xmlHelper.unmarshallXmlToObject(EmrClusterDefinition.class, definitionXml);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, definitionXml);
    // Create a new EMR cluster create request that replaces instance definitions with instance fleets.
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrClusterDefinition emrClusterDefinitionOverride = new EmrClusterDefinition();
    emrClusterDefinitionOverride.setInstanceDefinitions(new InstanceDefinitions());
    emrClusterDefinitionOverride.setInstanceFleets(Arrays.asList(new EmrClusterDefinitionInstanceFleet()));
    request.setEmrClusterDefinitionOverride(emrClusterDefinitionOverride);
    // Update the expected EMR cluster definition.
    expectedEmrClusterDefinition.setInstanceDefinitions(new InstanceDefinitions());
    expectedEmrClusterDefinition.setInstanceFleets(Arrays.asList(new EmrClusterDefinitionInstanceFleet()));
    // Create the cluster.
    EmrCluster emrCluster = emrService.createCluster(request);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertTrue(emrCluster.getNamespace().equals(request.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(request.getEmrClusterName()));
    assertNotNull(emrCluster.getId());
    assertNull(emrCluster.isDryRun());
    assertTrue(emrCluster.isEmrClusterCreated());
    assertNotNull(emrCluster.getEmrClusterDefinition());
    assertEquals(expectedEmrClusterDefinition, emrCluster.getEmrClusterDefinition());
    validateEmrClusterCreationLogUnique(emrCluster, expectedEmrClusterDefinition);
}