Examples with EmrShellStep org.finra.herd.model.api.xml.EmrShellStep used on opensource projects

Example 1 with EmrShellStep

use of org.finra.herd.model.api.xml.EmrShellStep in project herd by FINRAOS.

the class EmrShellStepHelper method buildResponseFromRequest.

@Override
public Object buildResponseFromRequest(Object stepRequest) {
    EmrShellStepAddRequest emrShellStepAddRequest = (EmrShellStepAddRequest) stepRequest;
    EmrShellStep step = new EmrShellStep();
    step.setNamespace(emrShellStepAddRequest.getNamespace());
    step.setEmrClusterDefinitionName(emrShellStepAddRequest.getEmrClusterDefinitionName());
    step.setEmrClusterName(emrShellStepAddRequest.getEmrClusterName());
    step.setStepName(emrShellStepAddRequest.getStepName().trim());
    step.setScriptLocation(emrShellStepAddRequest.getScriptLocation().trim().replaceAll(getS3ManagedReplaceString(), emrHelper.getS3StagingLocation()));
    // Add the script arguments
    if (!CollectionUtils.isEmpty(emrShellStepAddRequest.getScriptArguments())) {
        List<String> scriptArguments = new ArrayList<>();
        step.setScriptArguments(scriptArguments);
        for (String argument : emrShellStepAddRequest.getScriptArguments()) {
            scriptArguments.add(argument.trim());
        }
    }
    step.setContinueOnError(emrShellStepAddRequest.isContinueOnError());
    return step;
}

Example 2 with EmrShellStep

use of org.finra.herd.model.api.xml.EmrShellStep in project herd by FINRAOS.

the class EmrRestControllerTest method testAddShellStepToEmrCluster.

@Test
public void testAddShellStepToEmrCluster() throws Exception {
    // Create an add step request.
    EmrShellStepAddRequest emrShellStepAddRequest = new EmrShellStepAddRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_SCRIPT_LOCATION, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID, AWS_ACCOUNT_ID);
    // Create an add step response.
    EmrShellStep emrShellStep = new EmrShellStep(EMR_STEP_ID, NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_SCRIPT_LOCATION, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID);
    // Mock the external calls.
    when(emrService.addStepToCluster(emrShellStepAddRequest)).thenReturn(emrShellStep);
    // Call the method under test.
    EmrShellStep result = emrRestController.addShellStepToEmrCluster(emrShellStepAddRequest);
    // Verify the external calls.
    verify(emrService).addStepToCluster(emrShellStepAddRequest);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(emrShellStep, result);
}

Example 3 with EmrShellStep

use of org.finra.herd.model.api.xml.EmrShellStep in project herd by FINRAOS.

the class EmrServiceTest method testEmrAddSteps.

/**
 * This method tests the happy path scenario by providing all the parameters
 */
@Test
public void testEmrAddSteps() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    emrService.createCluster(request);
    // Create the Add step request.
    EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
    EmrShellStep emrShellStep = (EmrShellStep) emrService.addStepToCluster(stepRequest);
    // Validate the returned object against the input.
    assertNotNull(emrShellStep);
    assertTrue(emrShellStep.getNamespace().equals(request.getNamespace()));
    assertTrue(emrShellStep.getEmrClusterDefinitionName().equals(request.getEmrClusterDefinitionName()));
    assertTrue(emrShellStep.getEmrClusterName().equals(request.getEmrClusterName()));
}

Example 4 with EmrShellStep

use of org.finra.herd.model.api.xml.EmrShellStep in project herd by FINRAOS.

the class EmrServiceTest method testGetEmrClusterByIdWithStepId.

/**
 * This method tests the scenario with providing step Id.
 */
@Test
public void testGetEmrClusterByIdWithStepId() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    // Create a new EMR cluster create request
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrCluster emrCluster = emrService.createCluster(request);
    // Add a running step
    // Create the Add step request.
    EmrShellStepAddRequest stepRequest = getNewEmrShellStepAddRequest(request.getEmrClusterName());
    stepRequest.setStepName(MockEmrOperationsImpl.MOCK_STEP_RUNNING_NAME);
    EmrShellStep emrShellStep = (EmrShellStep) emrService.addStepToCluster(stepRequest);
    String stepId = emrShellStep.getId();
    EmrClusterAlternateKeyDto emrClusterAlternateKeyDto = EmrClusterAlternateKeyDto.builder().withNamespace(NAMESPACE).withEmrClusterDefinitionName(EMR_CLUSTER_DEFINITION_NAME).withEmrClusterName(request.getEmrClusterName()).build();
    EmrCluster emrClusterGet = emrService.getCluster(emrClusterAlternateKeyDto, emrCluster.getId(), stepId, true, null, false);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertNotNull(emrClusterGet);
    assertTrue(emrCluster.getId().equals(emrClusterGet.getId()));
    assertTrue(emrCluster.getNamespace().equals(emrClusterGet.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(emrClusterGet.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(emrClusterGet.getEmrClusterName()));
    assertTrue(stepId.equals(emrClusterGet.getActiveStep().getId()));
    assertTrue(stepId.equals(emrClusterGet.getStep().getId()));
    // Test the non verbose flow
    emrClusterGet = emrService.getCluster(emrClusterAlternateKeyDto, emrCluster.getId(), stepId, false, null, false);
    // Validate the returned object against the input.
    assertNotNull(emrCluster);
    assertNotNull(emrClusterGet);
    assertTrue(emrCluster.getId().equals(emrClusterGet.getId()));
    assertTrue(emrCluster.getNamespace().equals(emrClusterGet.getNamespace()));
    assertTrue(emrCluster.getEmrClusterDefinitionName().equals(emrClusterGet.getEmrClusterDefinitionName()));
    assertTrue(emrCluster.getEmrClusterName().equals(emrClusterGet.getEmrClusterName()));
    assertTrue(stepId.equals(emrClusterGet.getActiveStep().getId()));
    assertTrue(stepId.equals(emrClusterGet.getStep().getId()));
}

Example 5 with EmrShellStep

use of org.finra.herd.model.api.xml.EmrShellStep in project herd by FINRAOS.

the class EmrShellStepHelper method getEmrStepConfig.

@Override
public StepConfig getEmrStepConfig(Object step) {
    EmrShellStep emrShellStep = (EmrShellStep) step;
    // Hadoop Jar provided by Amazon for running Shell Scripts
    String hadoopJarForShellScript = configurationHelper.getProperty(ConfigurationValue.EMR_SHELL_SCRIPT_JAR);
    // Default ActionOnFailure is to cancel the execution and wait
    ActionOnFailure actionOnFailure = ActionOnFailure.CANCEL_AND_WAIT;
    if (emrShellStep.isContinueOnError() != null && emrShellStep.isContinueOnError()) {
        // Override based on user input
        actionOnFailure = ActionOnFailure.CONTINUE;
    }
    // Add the script location
    List<String> argsList = new ArrayList<>();
    argsList.add(emrShellStep.getScriptLocation().trim());
    // Add the script arguments
    if (!CollectionUtils.isEmpty(emrShellStep.getScriptArguments())) {
        for (String argument : emrShellStep.getScriptArguments()) {
            argsList.add(argument.trim());
        }
    }
    // Return the StepConfig object
    HadoopJarStepConfig jarConfig = new HadoopJarStepConfig(hadoopJarForShellScript).withArgs(argsList);
    return new StepConfig().withName(emrShellStep.getStepName().trim()).withActionOnFailure(actionOnFailure).withHadoopJarStep(jarConfig);
}