Examples with StepConfig com.amazonaws.services.elasticmapreduce.model.StepConfig used on opensource projects

Example 6 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class EmrDaoImpl method getStepConfig.

/**
 * Create the step config list of objects for hive/pig installation.
 *
 * @param emrClusterDefinition the EMR definition name value.
 *
 * @return list of step configuration that contains all the steps for the given configuration.
 */
private List<StepConfig> getStepConfig(EmrClusterDefinition emrClusterDefinition) {
    StepFactory stepFactory = new StepFactory();
    List<StepConfig> appSteps = new ArrayList<>();
    // Create install hive step and add to the StepConfig list
    if (StringUtils.isNotBlank(emrClusterDefinition.getHiveVersion())) {
        StepConfig installHive = new StepConfig().withName("Hive " + emrClusterDefinition.getHiveVersion()).withActionOnFailure(ActionOnFailure.TERMINATE_JOB_FLOW).withHadoopJarStep(stepFactory.newInstallHiveStep(emrClusterDefinition.getHiveVersion()));
        appSteps.add(installHive);
    }
    // Create install Pig step and add to the StepConfig List
    if (StringUtils.isNotBlank(emrClusterDefinition.getPigVersion())) {
        StepConfig installPig = new StepConfig().withName("Pig " + emrClusterDefinition.getPigVersion()).withActionOnFailure(ActionOnFailure.TERMINATE_JOB_FLOW).withHadoopJarStep(stepFactory.newInstallPigStep(emrClusterDefinition.getPigVersion()));
        appSteps.add(installPig);
    }
    // Add the hadoop jar steps that need to be added.
    if (!CollectionUtils.isEmpty(emrClusterDefinition.getHadoopJarSteps())) {
        for (HadoopJarStep hadoopJarStep : emrClusterDefinition.getHadoopJarSteps()) {
            StepConfig stepConfig = emrHelper.getEmrHadoopJarStepConfig(hadoopJarStep.getStepName(), hadoopJarStep.getJarLocation(), hadoopJarStep.getMainClass(), hadoopJarStep.getScriptArguments(), hadoopJarStep.isContinueOnError());
            appSteps.add(stepConfig);
        }
    }
    return appSteps;
}

Example 7 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class EmrHelperTest method testEmrHadoopJarStepConfigWithArguments.

@Test
public void testEmrHadoopJarStepConfigWithArguments() throws Exception {
    List<String> arguments = new ArrayList<>();
    arguments.add("arg1");
    StepConfig stepConfig = emrHelper.getEmrHadoopJarStepConfig("step_name", "jar_location", null, arguments, false);
    assertNotNull("step not retuned", stepConfig);
    assertEquals("name not found", "step_name", stepConfig.getName());
    assertEquals("jar not found", "jar_location", stepConfig.getHadoopJarStep().getJar());
    assertNotNull("arguments not found", stepConfig.getHadoopJarStep().getArgs());
}

Example 8 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class EmrHelperTest method testEmrHadoopJarStepConfig.

@Test
public void testEmrHadoopJarStepConfig() throws Exception {
    StepConfig stepConfig = emrHelper.getEmrHadoopJarStepConfig("step_name", "jar_location", null, null, false);
    assertNotNull("step not retuned", stepConfig);
    assertEquals("name not found", "step_name", stepConfig.getName());
    assertEquals("jar not found", "jar_location", stepConfig.getHadoopJarStep().getJar());
}

Example 9 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class MockEmrOperationsImpl method addJobFlowStepsRequest.

/**
 * Add Job Flow Step to AmazonElasticMapReduceClient
 */
@Override
public List<String> addJobFlowStepsRequest(AmazonElasticMapReduceClient emrClient, AddJobFlowStepsRequest addJobFlowStepsRequest) {
    if (addJobFlowStepsRequest.getSteps() != null && addJobFlowStepsRequest.getSteps().get(0) != null) {
        StepConfig firstStep = addJobFlowStepsRequest.getSteps().get(0);
        if (firstStep.getName().equals(MockAwsOperationsHelper.AMAZON_BAD_REQUEST)) {
            AmazonServiceException badRequestException = new AmazonServiceException(MockAwsOperationsHelper.AMAZON_BAD_REQUEST);
            badRequestException.setStatusCode(HttpStatus.SC_BAD_REQUEST);
            throw badRequestException;
        } else if (firstStep.getName().equals(MockAwsOperationsHelper.AMAZON_NOT_FOUND)) {
            AmazonServiceException notFoundException = new AmazonServiceException(MockAwsOperationsHelper.AMAZON_NOT_FOUND);
            notFoundException.setStatusCode(HttpStatus.SC_NOT_FOUND);
            throw notFoundException;
        } else if (firstStep.getName().equals(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION)) {
            throw new AmazonServiceException(MockAwsOperationsHelper.AMAZON_SERVICE_EXCEPTION);
        }
    }
    MockEmrJobFlow cluster = getClusterById(addJobFlowStepsRequest.getJobFlowId());
    if (cluster == null) {
        throw new AmazonServiceException("No Cluster exists with jobFlowId: " + addJobFlowStepsRequest.getJobFlowId());
    }
    List<String> jobIds = new ArrayList<>();
    for (StepConfig step : addJobFlowStepsRequest.getSteps()) {
        jobIds.add(addClusterStep(cluster.getJobFlowId(), step).getJobFlowId());
    }
    return jobIds;
}

Example 10 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class EmrDaoImpl method addEmrStep.

@Override
public String addEmrStep(String clusterId, StepConfig emrStepConfig, AwsParamsDto awsParamsDto) throws Exception {
    List<StepConfig> steps = new ArrayList<>();
    steps.add(emrStepConfig);
    // Add the job flow request
    AddJobFlowStepsRequest jobFlowStepRequest = new AddJobFlowStepsRequest(clusterId, steps);
    List<String> emrStepIds = emrOperations.addJobFlowStepsRequest(getEmrClient(awsParamsDto), jobFlowStepRequest);
    return emrStepIds.get(0);
}