Examples with StepConfig com.amazonaws.services.elasticmapreduce.model.StepConfig used on opensource projects

Example 11 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class EmrHiveStepHelper method getEmrStepConfig.

@Override
public StepConfig getEmrStepConfig(Object step) {
    EmrHiveStep emrHiveStep = (EmrHiveStep) step;
    // Default ActionOnFailure is to cancel the execution and wait
    ActionOnFailure actionOnFailure = ActionOnFailure.CANCEL_AND_WAIT;
    if (emrHiveStep.isContinueOnError() != null && emrHiveStep.isContinueOnError()) {
        // Override based on user input
        actionOnFailure = ActionOnFailure.CONTINUE;
    }
    // If there are no arguments to hive script
    if (CollectionUtils.isEmpty(emrHiveStep.getScriptArguments())) {
        // Just build the StepConfig object and return
        return new StepConfig().withName(emrHiveStep.getStepName().trim()).withActionOnFailure(actionOnFailure).withHadoopJarStep(new StepFactory().newRunHiveScriptStep(emrHiveStep.getScriptLocation().trim()));
    } else // If there are arguments specified
    {
        // For each argument, add "-d" option
        List<String> hiveArgs = new ArrayList<>();
        for (String hiveArg : emrHiveStep.getScriptArguments()) {
            hiveArgs.add("-d");
            hiveArgs.add(hiveArg);
        }
        // Return the StepConfig object
        return new StepConfig().withName(emrHiveStep.getStepName().trim()).withActionOnFailure(actionOnFailure).withHadoopJarStep(new StepFactory().newRunHiveScriptStep(emrHiveStep.getScriptLocation().trim(), hiveArgs.toArray(new String[hiveArgs.size()])));
    }
}

Example 12 with StepConfig

use of com.amazonaws.services.elasticmapreduce.model.StepConfig in project herd by FINRAOS.

the class EmrShellStepHelper method getEmrStepConfig.

@Override
public StepConfig getEmrStepConfig(Object step) {
    EmrShellStep emrShellStep = (EmrShellStep) step;
    // Hadoop Jar provided by Amazon for running Shell Scripts
    String hadoopJarForShellScript = configurationHelper.getProperty(ConfigurationValue.EMR_SHELL_SCRIPT_JAR);
    // Default ActionOnFailure is to cancel the execution and wait
    ActionOnFailure actionOnFailure = ActionOnFailure.CANCEL_AND_WAIT;
    if (emrShellStep.isContinueOnError() != null && emrShellStep.isContinueOnError()) {
        // Override based on user input
        actionOnFailure = ActionOnFailure.CONTINUE;
    }
    // Add the script location
    List<String> argsList = new ArrayList<>();
    argsList.add(emrShellStep.getScriptLocation().trim());
    // Add the script arguments
    if (!CollectionUtils.isEmpty(emrShellStep.getScriptArguments())) {
        for (String argument : emrShellStep.getScriptArguments()) {
            argsList.add(argument.trim());
        }
    }
    // Return the StepConfig object
    HadoopJarStepConfig jarConfig = new HadoopJarStepConfig(hadoopJarForShellScript).withArgs(argsList);
    return new StepConfig().withName(emrShellStep.getStepName().trim()).withActionOnFailure(actionOnFailure).withHadoopJarStep(jarConfig);
}