Examples with StepConfig com.amazonaws.services.elasticmapreduce.model.StepConfig used on opensource projects

Example 1 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class EmrPigStepHelper method getEmrStepConfig.

@Override
public StepConfig getEmrStepConfig(Object step) {
    EmrPigStep pigStep = (EmrPigStep) step;
    // Default ActionOnFailure is to cancel the execution and wait
    ActionOnFailure actionOnFailure = ActionOnFailure.CANCEL_AND_WAIT;
    if (pigStep.isContinueOnError() != null && pigStep.isContinueOnError()) {
        // Override based on user input
        actionOnFailure = ActionOnFailure.CONTINUE;
    }
    // If there are no arguments to hive script
    if (CollectionUtils.isEmpty(pigStep.getScriptArguments())) {
        // Just build the StepConfig object and return
        return new StepConfig().withName(pigStep.getStepName().trim()).withActionOnFailure(actionOnFailure).withHadoopJarStep(new StepFactory().newRunPigScriptStep(pigStep.getScriptLocation().trim()));
    } else // If there are arguments specified
    {
        return new StepConfig().withName(pigStep.getStepName().trim()).withActionOnFailure(actionOnFailure).withHadoopJarStep(new StepFactory().newRunPigScriptStep(pigStep.getScriptLocation().trim(), pigStep.getScriptArguments().toArray(new String[pigStep.getScriptArguments().size()])));
    }
}

Example 2 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class EmrHelperTest method testEmrHadoopJarStepConfigContinueOnError.

@Test
public void testEmrHadoopJarStepConfigContinueOnError() throws Exception {
    StepConfig stepConfig = emrHelper.getEmrHadoopJarStepConfig("step_name", "jar_location", null, null, true);
    assertNotNull("step not retuned", stepConfig);
    assertEquals("name not found", "step_name", stepConfig.getName());
    assertEquals("jar not found", "jar_location", stepConfig.getHadoopJarStep().getJar());
}

Example 3 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class EmrHelperTest method testEmrHadoopJarStepConfigNoContinueOnError.

@Test
public void testEmrHadoopJarStepConfigNoContinueOnError() throws Exception {
    StepConfig stepConfig = emrHelper.getEmrHadoopJarStepConfig("step_name", "jar_location", null, null, null);
    assertNotNull("step not retuned", stepConfig);
    assertEquals("name not found", "step_name", stepConfig.getName());
    assertEquals("jar not found", "jar_location", stepConfig.getHadoopJarStep().getJar());
}

Example 4 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class MockEmrOperationsImpl method createNewCluster.

private MockEmrJobFlow createNewCluster(RunJobFlowRequest jobFlowRequest, String status, StatusChangeReason reason, StatusTimeline timeline) {
    MockEmrJobFlow cluster = new MockEmrJobFlow();
    cluster.setJobFlowId(getNewJobFlowId());
    cluster.setJobFlowName(jobFlowRequest.getName());
    cluster.setStatus(status);
    cluster.setStatusTimeline(timeline);
    cluster.setStatusChangeReason(reason);
    emrClusters.put(cluster.getJobFlowId(), cluster);
    // Add the steps
    for (StepConfig stepConfig : jobFlowRequest.getSteps()) {
        addClusterStep(cluster.getJobFlowId(), stepConfig);
    }
    return cluster;
}

Example 5 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class EmrDaoTest method addEmrStepCallsAddJobFlowSteps.

@Test
public void addEmrStepCallsAddJobFlowSteps() throws Exception {
    String clusterName = "clusterName";
    StepConfig emrStepConfig = new StepConfig();
    String clusterId = "clusterId";
    String stepId = "stepId";
    /*
         * Mock the EmrOperations.listEmrClusters() call to return a known result.
         */
    ListClustersResult listClustersResult = new ListClustersResult();
    ClusterSummary clusterSummary = new ClusterSummary();
    clusterSummary.setId(clusterId);
    clusterSummary.setName(clusterName);
    listClustersResult.setClusters(Arrays.asList(clusterSummary));
    when(mockEmrOperations.listEmrClusters(any(), any())).thenReturn(listClustersResult);
    /*
         * Mock EmrOperations.addJobFlowStepsRequest() and assert parameters passed in.
         */
    when(mockEmrOperations.addJobFlowStepsRequest(any(), any())).thenAnswer(new Answer<List<String>>() {

        @Override
        public List<String> answer(InvocationOnMock invocation) throws Throwable {
            AddJobFlowStepsRequest addJobFlowStepsRequest = invocation.getArgument(1);
            assertEquals(clusterId, addJobFlowStepsRequest.getJobFlowId());
            List<StepConfig> steps = addJobFlowStepsRequest.getSteps();
            assertEquals(1, steps.size());
            assertEquals(emrStepConfig, steps.get(0));
            // return a single step with the given stepId
            return Arrays.asList(stepId);
        }
    });
    assertEquals(stepId, emrDao.addEmrStep(clusterId, emrStepConfig, new AwsParamsDto()));
}