Examples with EmrStepHelper org.finra.herd.service.helper.EmrStepHelper used on opensource projects

Example 1 with EmrStepHelper

use of org.finra.herd.service.helper.EmrStepHelper in project herd by FINRAOS.

the class BaseAddEmrStep method addEmrStepAndSetWorkflowVariables.

/**
 * Adds the Step to EMR cluster, and sets the step id as workflow variable.
 *
 * @param request the EmrStepsAddRequest
 * @param execution the DelegateExecution
 *
 * @throws Exception if any problems were encountered.
 */
protected void addEmrStepAndSetWorkflowVariables(Object request, DelegateExecution execution) throws Exception {
    // Add the step.
    Object emrStep = emrService.addStepToCluster(request);
    EmrStepHelper stepHelper = emrStepHelperFactory.getStepHelper(request.getClass().getName());
    String stepId = stepHelper.getStepId(emrStep);
    // Set workflow variables based on the new step.
    // Set the stepId in workflow variable.
    setTaskWorkflowVariable(execution, VARIABLE_EMR_STEP_ID, stepId);
}

Example 2 with EmrStepHelper

use of org.finra.herd.service.helper.EmrStepHelper in project herd by FINRAOS.

the class BaseAddEmrStep method populateCommonParams.

/**
 * Populates common parameters.
 *
 * @param request the request.
 * @param execution the delegate execution.
 */
protected void populateCommonParams(Object request, DelegateExecution execution) {
    EmrStepHelper stepHelper = emrStepHelperFactory.getStepHelper(request.getClass().getName());
    stepHelper.setRequestStepName(request, getStepName(execution));
    stepHelper.setRequestContinueOnError(request, getContinueOnError(execution));
    stepHelper.setRequestNamespace(request, getNamespace(execution));
    stepHelper.setRequestEmrClusterDefinitionName(request, getEmrClusterDefinitionName(execution));
    stepHelper.setRequestEmrClusterName(request, getEmrClusterName(execution));
    stepHelper.setRequestEmrClusterId(request, getEmrClusterId(execution));
    stepHelper.setRequestAccountId(request, getAccountId(execution));
}

Example 3 with EmrStepHelper

use of org.finra.herd.service.helper.EmrStepHelper in project herd by FINRAOS.

the class BaseAddEmrStepTest method populateCommonParamsAssertEmrStepHelperPopulatedCorrectly.

@Test
public void populateCommonParamsAssertEmrStepHelperPopulatedCorrectly() {
    Object request = "request";
    DelegateExecution execution = mock(DelegateExecution.class);
    EmrStepHelper emrStepHelper = mock(EmrStepHelper.class);
    when(emrStepHelperFactory.getStepHelper(any())).thenReturn(emrStepHelper);
    String namespaceString = "namespaceString";
    when(activitiHelper.getExpressionVariableAsString(same(namespace), any())).thenReturn(namespaceString);
    String stepNameString = "stepNameString";
    when(activitiHelper.getExpressionVariableAsString(same(stepName), any())).thenReturn(stepNameString);
    Boolean continueOnErrorBoolean = false;
    when(activitiHelper.getExpressionVariableAsBoolean(same(continueOnError), any(), any(), anyBoolean(), any())).thenReturn(continueOnErrorBoolean);
    String emrClusterDefinitionNameString = "emrClusterDefinitionNameString";
    when(activitiHelper.getExpressionVariableAsString(same(emrClusterDefinitionName), any())).thenReturn(emrClusterDefinitionNameString);
    String emrClusterNameString = "emrClusterNameString";
    when(activitiHelper.getExpressionVariableAsString(same(emrClusterName), any())).thenReturn(emrClusterNameString);
    String emrClusterIdString = "emrClusterIdString";
    when(activitiHelper.getExpressionVariableAsString(same(emrClusterId), any())).thenReturn(emrClusterIdString);
    baseAddEmrStep.populateCommonParams(request, execution);
    verify(emrStepHelper).setRequestStepName(request, stepNameString);
    verify(emrStepHelper).setRequestContinueOnError(request, continueOnErrorBoolean);
    verify(emrStepHelper).setRequestNamespace(request, namespaceString);
    verify(emrStepHelper).setRequestEmrClusterDefinitionName(request, emrClusterDefinitionNameString);
    verify(emrStepHelper).setRequestEmrClusterName(request, emrClusterNameString);
    verify(emrStepHelper).setRequestEmrClusterId(request, emrClusterIdString);
}

Example 4 with EmrStepHelper

use of org.finra.herd.service.helper.EmrStepHelper in project herd by FINRAOS.

the class EmrServiceWithAccountIdTest method testEmrAddStepsAllTypes.

/**
 * This method tests the happy path scenario by testing all the step types.
 */
@Test
public void testEmrAddStepsAllTypes() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    // Create a trusting AWS account.
    trustingAccountDaoTestHelper.createTrustingAccountEntity(AWS_ACCOUNT_ID, AWS_ROLE_ARN);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequestWithAccountId();
    emrService.createCluster(request);
    List<Object> emrSteps = new ArrayList<>();
    List<String> shellScriptArgs = new ArrayList<>();
    shellScriptArgs.add("Hello");
    shellScriptArgs.add("herd");
    shellScriptArgs.add("How Are You");
    EmrShellStepAddRequest shellStepRequest = new EmrShellStepAddRequest();
    shellStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_script.sh");
    shellStepRequest.setStepName("Test Shell Script");
    shellStepRequest.setScriptArguments(shellScriptArgs);
    emrSteps.add(shellStepRequest);
    EmrHiveStepAddRequest hiveStepRequest = new EmrHiveStepAddRequest();
    List<String> scriptArgs1 = new ArrayList<>();
    scriptArgs1.add("arg2=sampleArg");
    scriptArgs1.add("arg1=tables");
    hiveStepRequest.setStepName("Test Hive");
    hiveStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_hive.hql");
    hiveStepRequest.setScriptArguments(scriptArgs1);
    emrSteps.add(hiveStepRequest);
    EmrPigStepAddRequest pigStepRequest = new EmrPigStepAddRequest();
    pigStepRequest.setStepName("Test Pig");
    pigStepRequest.setScriptArguments(shellScriptArgs);
    pigStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_pig.pig");
    emrSteps.add(pigStepRequest);
    shellStepRequest = new EmrShellStepAddRequest();
    shellStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/bootstrap/install_oozie.sh");
    shellStepRequest.setStepName("Install Oozie");
    List<String> shellScriptArgsOozie = new ArrayList<>();
    shellScriptArgsOozie.add("s3://test-bucket-managed/app-a/bootstrap/oozie-4.0.1-distro.tar");
    shellStepRequest.setScriptArguments(shellScriptArgsOozie);
    emrSteps.add(shellStepRequest);
    EmrHadoopJarStepAddRequest hadoopJarStepRequest = new EmrHadoopJarStepAddRequest();
    List<String> scriptArgs2 = new ArrayList<>();
    scriptArgs2.add("oozie_run");
    scriptArgs2.add("wordcountOutput");
    hadoopJarStepRequest.setStepName("Hadoop Jar");
    hadoopJarStepRequest.setJarLocation("s3://test-bucket-managed/app-a/test/hadoop-mapreduce-examples-2.4.0.jar");
    hadoopJarStepRequest.setMainClass("wordcount");
    hadoopJarStepRequest.setScriptArguments(scriptArgs2);
    emrSteps.add(hadoopJarStepRequest);
    EmrStepHelper stepHelper;
    for (Object emrStepAddRequest : emrSteps) {
        stepHelper = emrStepHelperFactory.getStepHelper(emrStepAddRequest.getClass().getName());
        stepHelper.setRequestNamespace(emrStepAddRequest, NAMESPACE);
        stepHelper.setRequestEmrClusterDefinitionName(emrStepAddRequest, EMR_CLUSTER_DEFINITION_NAME);
        stepHelper.setRequestEmrClusterName(emrStepAddRequest, request.getEmrClusterName());
        stepHelper.setRequestAccountId(emrStepAddRequest, AWS_ACCOUNT_ID);
        Object emrStep = emrService.addStepToCluster(emrStepAddRequest);
        assertNotNull(emrStep);
        assertNotNull(stepHelper.getStepId(emrStep));
        Method getNameMethod = emrStep.getClass().getMethod("getStepName");
        String emrStepName = (String) getNameMethod.invoke(emrStep);
        assertEquals(stepHelper.getRequestStepName(emrStepAddRequest), emrStepName);
        Method isContinueOnErrorMethod = emrStep.getClass().getMethod("isContinueOnError");
        Object emrStepIsContinueOnError = isContinueOnErrorMethod.invoke(emrStep);
        assertEquals(stepHelper.isRequestContinueOnError(emrStepAddRequest), emrStepIsContinueOnError);
    }
}

Example 5 with EmrStepHelper

use of org.finra.herd.service.helper.EmrStepHelper in project herd by FINRAOS.

the class EmrServiceTest method testEmrAddStepsAllTypes.

/**
 * This method tests the happy path scenario by testing all the step types
 */
@Test
public void testEmrAddStepsAllTypes() throws Exception {
    // Create the namespace entity.
    NamespaceEntity namespaceEntity = namespaceDaoTestHelper.createNamespaceEntity(NAMESPACE);
    emrClusterDefinitionDaoTestHelper.createEmrClusterDefinitionEntity(namespaceEntity, EMR_CLUSTER_DEFINITION_NAME, IOUtils.toString(resourceLoader.getResource(EMR_CLUSTER_DEFINITION_XML_FILE_WITH_CLASSPATH).getInputStream()));
    EmrClusterCreateRequest request = getNewEmrClusterCreateRequest();
    emrService.createCluster(request);
    List<Object> emrSteps = new ArrayList<>();
    List<String> shellScriptArgs = new ArrayList<>();
    shellScriptArgs.add("Hello");
    shellScriptArgs.add("herd");
    shellScriptArgs.add("How Are You");
    EmrShellStepAddRequest shellStepRequest = new EmrShellStepAddRequest();
    shellStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_script.sh");
    shellStepRequest.setStepName("Test Shell Script");
    shellStepRequest.setScriptArguments(shellScriptArgs);
    emrSteps.add(shellStepRequest);
    EmrHiveStepAddRequest hiveStepRequest = new EmrHiveStepAddRequest();
    List<String> scriptArgs1 = new ArrayList<>();
    scriptArgs1.add("arg2=sampleArg");
    scriptArgs1.add("arg1=tables");
    hiveStepRequest.setStepName("Test Hive");
    hiveStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_hive.hql");
    hiveStepRequest.setScriptArguments(scriptArgs1);
    emrSteps.add(hiveStepRequest);
    EmrPigStepAddRequest pigStepRequest = new EmrPigStepAddRequest();
    pigStepRequest.setStepName("Test Pig");
    pigStepRequest.setScriptArguments(shellScriptArgs);
    pigStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/test/test_pig.pig");
    emrSteps.add(pigStepRequest);
    shellStepRequest = new EmrShellStepAddRequest();
    shellStepRequest.setScriptLocation("s3://test-bucket-managed/app-a/bootstrap/install_oozie.sh");
    shellStepRequest.setStepName("Install Oozie");
    List<String> shellScriptArgsOozie = new ArrayList<>();
    shellScriptArgsOozie.add("s3://test-bucket-managed/app-a/bootstrap/oozie-4.0.1-distro.tar");
    shellStepRequest.setScriptArguments(shellScriptArgsOozie);
    emrSteps.add(shellStepRequest);
    EmrHadoopJarStepAddRequest hadoopJarStepRequest = new EmrHadoopJarStepAddRequest();
    List<String> scriptArgs2 = new ArrayList<>();
    scriptArgs2.add("oozie_run");
    scriptArgs2.add("wordcountOutput");
    hadoopJarStepRequest.setStepName("Hadoop Jar");
    hadoopJarStepRequest.setJarLocation("s3://test-bucket-managed/app-a/test/hadoop-mapreduce-examples-2.4.0.jar");
    hadoopJarStepRequest.setMainClass("wordcount");
    hadoopJarStepRequest.setScriptArguments(scriptArgs2);
    emrSteps.add(hadoopJarStepRequest);
    EmrStepHelper stepHelper;
    for (Object emrStepAddRequest : emrSteps) {
        stepHelper = emrStepHelperFactory.getStepHelper(emrStepAddRequest.getClass().getName());
        stepHelper.setRequestNamespace(emrStepAddRequest, NAMESPACE);
        stepHelper.setRequestEmrClusterDefinitionName(emrStepAddRequest, EMR_CLUSTER_DEFINITION_NAME);
        stepHelper.setRequestEmrClusterName(emrStepAddRequest, request.getEmrClusterName());
        Object emrStep = emrService.addStepToCluster(emrStepAddRequest);
        assertNotNull(emrStep);
        assertNotNull(stepHelper.getStepId(emrStep));
        Method getNameMethod = emrStep.getClass().getMethod("getStepName");
        String emrStepName = (String) getNameMethod.invoke(emrStep);
        assertEquals(stepHelper.getRequestStepName(emrStepAddRequest), emrStepName);
        Method isContinueOnErrorMethod = emrStep.getClass().getMethod("isContinueOnError");
        Object emrStepIsContinueOnError = isContinueOnErrorMethod.invoke(emrStep);
        assertEquals(stepHelper.isRequestContinueOnError(emrStepAddRequest), emrStepIsContinueOnError);
    }
}