Examples with EmrHadoopJarStep org.finra.herd.model.api.xml.EmrHadoopJarStep used on opensource projects

Example 1 with EmrHadoopJarStep

use of org.finra.herd.model.api.xml.EmrHadoopJarStep in project herd by FINRAOS.

the class EmrRestControllerTest method testAddHadoopJarStepToEmrCluster.

@Test
public void testAddHadoopJarStepToEmrCluster() throws Exception {
    // Create an add step request.
    EmrHadoopJarStepAddRequest emrHadoopJarStepAddRequest = new EmrHadoopJarStepAddRequest(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_JAR_LOCATION, EMR_STEP_MAIN_CLASS, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID, AWS_ACCOUNT_ID);
    // Create an add step response.
    EmrHadoopJarStep emrHadoopJarStep = new EmrHadoopJarStep(EMR_STEP_ID, NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, EMR_CLUSTER_NAME, EMR_STEP_NAME, EMR_STEP_JAR_LOCATION, EMR_STEP_MAIN_CLASS, Arrays.asList(ATTRIBUTE_NAME_1_MIXED_CASE), CONTINUE_ON_ERROR, EMR_CLUSTER_ID);
    // Mock the external calls.
    when(emrService.addStepToCluster(emrHadoopJarStepAddRequest)).thenReturn(emrHadoopJarStep);
    // Call the method under test.
    EmrHadoopJarStep result = emrRestController.addHadoopJarStepToEmrCluster(emrHadoopJarStepAddRequest);
    // Verify the external calls.
    verify(emrService).addStepToCluster(emrHadoopJarStepAddRequest);
    verifyNoMoreInteractionsHelper();
    // Validate the results.
    assertEquals(emrHadoopJarStep, result);
}

Example 2 with EmrHadoopJarStep

use of org.finra.herd.model.api.xml.EmrHadoopJarStep in project herd by FINRAOS.

the class EmrServiceTest method testEmrAddStepsHadoopNoMainClass.

/**
 * This method tests the happy path scenario by testing all the step types
 */
@Test
public void testEmrAddStepsHadoopNoMainClass() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    emrService.createCluster(request);
    // TODO: Why are we adding EMR steps to a list, but not doing anything with them once they're added?
    List<Serializable> emrSteps = new ArrayList<>();
    EmrHadoopJarStepAddRequest hadoopJarStepRequest = new EmrHadoopJarStepAddRequest();
    hadoopJarStepRequest.setNamespace(request.getNamespace());
    hadoopJarStepRequest.setEmrClusterDefinitionName(request.getEmrClusterDefinitionName());
    hadoopJarStepRequest.setEmrClusterName(request.getEmrClusterName());
    hadoopJarStepRequest.setStepName("Hadoop Jar");
    hadoopJarStepRequest.setJarLocation("s3://test-bucket-managed/app-a/test/hadoop-mapreduce-examples-2.4.0.jar");
    emrSteps.add(hadoopJarStepRequest);
    EmrHadoopJarStep emrHadoopJarStep = (EmrHadoopJarStep) emrService.addStepToCluster(hadoopJarStepRequest);
    assertNotNull(emrHadoopJarStep);
    assertNotNull(emrHadoopJarStep.getId());
}

Example 3 with EmrHadoopJarStep

use of org.finra.herd.model.api.xml.EmrHadoopJarStep in project herd by FINRAOS.

the class EmrHelperTest method testEmrAddStepsAllTypesNegativeTestCase.

/**
 * This method tests the negative test cases scenario by testing all the step types
 */
@Test
public void testEmrAddStepsAllTypesNegativeTestCase() throws Exception {
    // Create a namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Create an EMR cluster definition entity.
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    // Create EC2 on-demand pricing entities.
    ec2OnDemandPricingDaoTestHelper.createEc2OnDemandPricingEntities();
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    EmrCluster emrCluster = emrService.createCluster(request);
    EmrStepHelper stepHelper;
    // Shell script arguments
    List<String> shellScriptArgs = new ArrayList<>();
    shellScriptArgs.add("Hello");
    shellScriptArgs.add("herd");
    shellScriptArgs.add("How Are You");
    List<Object> steps = new ArrayList<>();
    // Shell step parameters
    EmrShellStep shellStep = new EmrShellStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    shellStep.setScriptLocation("s3://test-bucket-managed/app-a/test/test_script.sh");
    shellStep.setStepName("Test Shell Script");
    shellStep.setScriptArguments(shellScriptArgs);
    shellStep.setContinueOnError(true);
    steps.add(shellStep);
    // Hive step parameters
    EmrHiveStep hiveStep = new EmrHiveStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    hiveStep.setStepName("Test Hive");
    hiveStep.setScriptLocation("s3://test-bucket-managed/app-a/test/test_hive.hql");
    hiveStep.setContinueOnError(true);
    steps.add(hiveStep);
    // Pig step parameter
    EmrPigStep pigStep = new EmrPigStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    pigStep.setStepName("Test Pig");
    pigStep.setContinueOnError(true);
    pigStep.setScriptLocation("s3://test-bucket-managed/app-a/test/test_pig.pig");
    steps.add(pigStep);
    // Oozie step that includes a shell script to install oozie
    shellStep = new EmrShellStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null);
    shellStep.setScriptLocation("s3://test-bucket-managed/app-a/bootstrap/install_oozie.sh");
    shellStep.setStepName("Install Oozie");
    List<String> shellScriptArgsOozie = new ArrayList<>();
    shellScriptArgsOozie.add("s3://test-bucket-managed/app-a/bootstrap/oozie-4.0.1-distro.tar");
    shellStep.setScriptArguments(shellScriptArgsOozie);
    steps.add(shellStep);
    // Hadoop jar step configuration
    EmrHadoopJarStep hadoopJarStep = new EmrHadoopJarStep(NAMESPACE, EMR_CLUSTER_DEFINITION_NAME, request.getEmrClusterName(), null, null, null, null, null, null, null);
    hadoopJarStep.setContinueOnError(true);
    hadoopJarStep.setStepName("Hadoop Jar");
    hadoopJarStep.setJarLocation("s3://test-bucket-managed/app-a/test/hadoop-mapreduce-examples-2.4.0.jar");
    hadoopJarStep.setMainClass("wordcount");
    steps.add(hadoopJarStep);
    for (Object emrStep : steps) {
        stepHelper = emrStepHelperFactory.getStepHelper(emrStep.getClass().getName());
        emrDao.addEmrStep(emrCluster.getId(), stepHelper.getEmrStepConfig(emrStep), emrHelper.getAwsParamsDto());
    }
}

Example 4 with EmrHadoopJarStep

use of org.finra.herd.model.api.xml.EmrHadoopJarStep in project herd by FINRAOS.

the class EmrHadoopJarStepHelper method buildResponseFromRequest.

@Override
public Object buildResponseFromRequest(Object stepRequest) {
    EmrHadoopJarStepAddRequest emrHadoopJarStepAddRequest = (EmrHadoopJarStepAddRequest) stepRequest;
    EmrHadoopJarStep step = new EmrHadoopJarStep();
    step.setNamespace(emrHadoopJarStepAddRequest.getNamespace());
    step.setEmrClusterDefinitionName(emrHadoopJarStepAddRequest.getEmrClusterDefinitionName());
    step.setEmrClusterName(emrHadoopJarStepAddRequest.getEmrClusterName());
    step.setStepName(emrHadoopJarStepAddRequest.getStepName().trim());
    step.setJarLocation(emrHadoopJarStepAddRequest.getJarLocation().trim().replaceAll(getS3ManagedReplaceString(), emrHelper.getS3StagingLocation()));
    if (emrHadoopJarStepAddRequest.getMainClass() != null) {
        step.setMainClass(emrHadoopJarStepAddRequest.getMainClass().trim());
    }
    // Add the script arguments
    if (!CollectionUtils.isEmpty(emrHadoopJarStepAddRequest.getScriptArguments())) {
        List<String> scriptArguments = new ArrayList<>();
        step.setScriptArguments(scriptArguments);
        for (String argument : emrHadoopJarStepAddRequest.getScriptArguments()) {
            scriptArguments.add(argument.trim());
        }
    }
    step.setContinueOnError(emrHadoopJarStepAddRequest.isContinueOnError());
    return step;
}