Examples with Job org.finra.herd.model.api.xml.Job used on opensource projects

Example 66 with Job

use of org.finra.herd.model.api.xml.Job in project herd by FINRAOS.

the class CheckEmrClusterTest method testCheckClusterByClusterIdStepIdRequestedStepHasNoId.

/**
 * This method tests the check EMR cluster activiti task with cluster Id and step Id specified and requested step contains no id in response
 */
@Test
public void testCheckClusterByClusterIdStepIdRequestedStepHasNoId() throws Exception {
    List<FieldExtension> fieldExtensions = getOptionalFieldExtensions();
    FieldExtension fieldExtension = new FieldExtension();
    fieldExtension.setFieldName("emrStepId");
    fieldExtension.setExpression("${addHiveStepServiceTask_emrStepId}");
    fieldExtensions.add(fieldExtension);
    List<Parameter> parameters = new ArrayList<>();
    parameters.add(new Parameter("clusterName", "testCluster1"));
    parameters.add(new Parameter("hiveStepName", MockEmrOperationsImpl.MOCK_STEP_WITHOUT_ID_NAME));
    parameters.add(new Parameter("hiveScriptLocation", "a_hive_step_location"));
    parameters.add(new Parameter("shellStepName", MockEmrOperationsImpl.MOCK_STEP_RUNNING_NAME));
    parameters.add(new Parameter("shellScriptLocation", "a_shell_step_location"));
    parameters.add(new Parameter("verbose", "false"));
    parameters.add(new Parameter("retrieveInstanceFleets", "false"));
    // Run a job with Activiti XML that will start cluster, check status and terminate.
    Job job = jobServiceTestHelper.createJobForCreateClusterForActivitiXml(getCheckClusterActivitiXml(fieldExtensions), parameters);
    assertNotNull(job);
    HistoricProcessInstance hisInstance = activitiHistoryService.createHistoricProcessInstanceQuery().processInstanceId(job.getId()).includeProcessVariables().singleResult();
    Map<String, Object> variables = hisInstance.getProcessVariables();
    String hiveStepId = (String) variables.get("addHiveStepServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + "emrStepId");
    assertNotNull(hiveStepId);
    String emrStepId = (String) variables.get("checkClusterServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + "step_id");
    assertNull(emrStepId);
    String emrStepJarLocation = (String) variables.get("checkClusterServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + "step_jarLocation");
    assertNull(emrStepJarLocation);
    String shellStepId = (String) variables.get("addShellStepServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + "emrStepId");
    assertNotNull(shellStepId);
    String activeStepId = (String) variables.get("checkClusterServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + "activeStep_id");
    assertEquals(shellStepId, activeStepId);
    String activeStepJarLocation = (String) variables.get("checkClusterServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + "activeStep_jarLocation");
    assertNull(activeStepJarLocation);
}

Example 67 with Job

use of org.finra.herd.model.api.xml.Job in project herd by FINRAOS.

the class CheckEmrClusterTest method testCheckClusterByClusterIdStepIdActiveStepHasNoId.

@Test
public void testCheckClusterByClusterIdStepIdActiveStepHasNoId() throws Exception {
    List<FieldExtension> fieldExtensions = getOptionalFieldExtensions();
    FieldExtension fieldExtension = new FieldExtension();
    fieldExtension.setFieldName("emrStepId");
    fieldExtension.setExpression("${addHiveStepServiceTask_emrStepId}");
    fieldExtensions.add(fieldExtension);
    List<Parameter> parameters = new ArrayList<>();
    parameters.add(new Parameter("clusterName", "testCluster1"));
    parameters.add(new Parameter("hiveStepName", "a_hive_step"));
    parameters.add(new Parameter("hiveScriptLocation", "a_hive_step_location"));
    parameters.add(new Parameter("shellStepName", MockEmrOperationsImpl.MOCK_STEP_RUNNING_WITHOUT_ID_NAME));
    parameters.add(new Parameter("shellScriptLocation", "a_shell_step_location"));
    parameters.add(new Parameter("verbose", "false"));
    parameters.add(new Parameter("retrieveInstanceFleets", "false"));
    // Run a job with Activiti XML that will start cluster, check status and terminate.
    Job job = jobServiceTestHelper.createJobForCreateClusterForActivitiXml(getCheckClusterActivitiXml(fieldExtensions), parameters);
    assertNotNull(job);
    HistoricProcessInstance hisInstance = activitiHistoryService.createHistoricProcessInstanceQuery().processInstanceId(job.getId()).includeProcessVariables().singleResult();
    Map<String, Object> variables = hisInstance.getProcessVariables();
    String hiveStepId = (String) variables.get("addHiveStepServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + "emrStepId");
    assertNotNull(hiveStepId);
    String emrStepId = (String) variables.get("checkClusterServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + "step_id");
    assertEquals(hiveStepId, emrStepId);
    String emrStepJarLocation = (String) variables.get("checkClusterServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + "step_jarLocation");
    assertNull(emrStepJarLocation);
    String activeStepId = (String) variables.get("checkClusterServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + "activeStep_id");
    assertNull(activeStepId);
}

Example 68 with Job

use of org.finra.herd.model.api.xml.Job in project herd by FINRAOS.

the class JobServiceImpl method getJob.

/**
 * Gets a job by the given jobId, and displays verbose information if verbose flag is true. Does namespace permission check if namespace permissions flag is
 * true.
 *
 * @param jobId the job ID
 * @param verbose specifies if verbose mode is enabled or not
 * @param checkNamespacePermissions specifies whether to check namespace permissions or not
 *
 * @return the job information
 */
private Job getJob(String jobId, boolean verbose, boolean checkNamespacePermissions) {
    String jobIdLocal = jobId;
    // Validate and trim required elements
    Assert.hasText(jobIdLocal, "A job Id must be specified.");
    jobIdLocal = jobIdLocal.trim();
    // Get process instance from Activiti runtime.
    ProcessInstance processInstance = activitiService.getProcessInstanceById(jobIdLocal);
    // Get historic process instance.
    HistoricProcessInstance historicProcessInstance = activitiService.getHistoricProcessInstanceByProcessInstanceId(jobIdLocal);
    /*
         * Check permissions against namespace of the job.
         * Cannot be done through the annotation because the namespace is not part of the request.
         * TODO refactor this so it gets namespace from JobDefinitionEntity instead of parsing process definition key.
         */
    if (checkNamespacePermissions) {
        String processDefinitionKey = null;
        if (processInstance != null) {
            processDefinitionKey = processInstance.getProcessDefinitionKey();
        } else if (historicProcessInstance != null) {
            processDefinitionKey = activitiService.getProcessDefinitionById(historicProcessInstance.getProcessDefinitionId()).getKey();
        }
        if (processDefinitionKey != null) {
            checkPermissions(processDefinitionKey, new NamespacePermissionEnum[] { NamespacePermissionEnum.READ });
        }
    }
    Job job = new Job();
    job.setId(jobIdLocal);
    if (processInstance == null && historicProcessInstance == null) {
        // Process not found
        throw new ObjectNotFoundException(String.format("Job with Id: \"%s\" doesn't exist.", jobIdLocal));
    } else if (processInstance != null) {
        // Process is still running or suspended.
        job.setStatus(processInstance.isSuspended() ? JobStatusEnum.SUSPENDED : JobStatusEnum.RUNNING);
        // Check if there are errors.
        List<org.activiti.engine.runtime.Job> erroredJobs = activitiService.getJobsWithExceptionByProcessInstanceId(processInstance.getProcessInstanceId());
        if (!CollectionUtils.isEmpty(erroredJobs)) {
            // Errors found.
            List<WorkflowError> workflowErrors = new ArrayList<>();
            for (org.activiti.engine.runtime.Job erroredJob : erroredJobs) {
                WorkflowError workflowError = new WorkflowError();
                workflowError.setErrorMessage(erroredJob.getExceptionMessage());
                workflowError.setRetriesLeft(erroredJob.getRetries());
                workflowError.setErrorStackTrace(activitiService.getJobExceptionStacktrace(erroredJob.getId()));
                workflowErrors.add(workflowError);
            }
            job.setWorkflowErrors(workflowErrors);
        }
        if (processInstance.getActivityId() != null) {
            // Set current workflow step.
            WorkflowStep currentStep = new WorkflowStep();
            currentStep.setId(processInstance.getActivityId());
            job.setCurrentWorkflowStep(currentStep);
        }
        // Set the workflow variables.
        populateWorkflowParameters(job, processInstance.getProcessVariables());
        // If verbose, set activiti workflow xml.
        if (verbose) {
            populateActivitiXml(job, processInstance.getProcessDefinitionId());
        }
    } else {
        // Process completed
        job.setStatus(JobStatusEnum.COMPLETED);
        // Set the workflow variables.
        populateWorkflowParameters(job, historicProcessInstance.getProcessVariables());
        job.setDeleteReason(historicProcessInstance.getDeleteReason());
        // If verbose, set activiti workflow xml.
        if (verbose) {
            populateActivitiXml(job, historicProcessInstance.getProcessDefinitionId());
        }
    }
    if (historicProcessInstance != null) {
        // If verbose, set completed steps.
        if (verbose) {
            populateCompletedActivitiSteps(job, activitiService.getHistoricActivityInstancesByProcessInstanceId(jobIdLocal));
        }
        // Set the start time always since all jobs will have a start time.
        job.setStartTime(HerdDateUtils.getXMLGregorianCalendarValue(historicProcessInstance.getStartTime()));
        // Set the end time if the historic process instance has it set (the job is completed).
        if (historicProcessInstance.getEndTime() != null) {
            job.setEndTime(HerdDateUtils.getXMLGregorianCalendarValue(historicProcessInstance.getEndTime()));
        }
    }
    return job;
}

Example 69 with Job

use of org.finra.herd.model.api.xml.Job in project herd by FINRAOS.

the class AddEmrStepsTest method createJobAndCheckStepStatusSuccess.

private void createJobAndCheckStepStatusSuccess(String activitiXml, List<Parameter> parameters) throws Exception {
    Job job = jobServiceTestHelper.createJobForCreateClusterForActivitiXml(activitiXml, parameters);
    assertNotNull(job);
    HistoricProcessInstance hisInstance = activitiHistoryService.createHistoricProcessInstanceQuery().processInstanceId(job.getId()).includeProcessVariables().singleResult();
    Map<String, Object> variables = hisInstance.getProcessVariables();
    String addStepServiceTaskStatus = (String) variables.get("addStepServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + ActivitiRuntimeHelper.VARIABLE_STATUS);
    assertEquals(ActivitiRuntimeHelper.TASK_STATUS_SUCCESS, addStepServiceTaskStatus);
    String addStepId = (String) variables.get("addStepServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + BaseAddEmrStep.VARIABLE_EMR_STEP_ID);
    assertNotNull(addStepId);
}

Example 70 with Job

use of org.finra.herd.model.api.xml.Job in project herd by FINRAOS.

the class AddEmrStepsTest method createJobAndCheckStepStatusFailure.

private void createJobAndCheckStepStatusFailure(String activitiXml, List<Parameter> parameters) throws Exception {
    Job job = jobServiceTestHelper.createJobForCreateClusterForActivitiXml(activitiXml, parameters);
    assertNotNull(job);
    HistoricProcessInstance hisInstance = activitiHistoryService.createHistoricProcessInstanceQuery().processInstanceId(job.getId()).includeProcessVariables().singleResult();
    Map<String, Object> variables = hisInstance.getProcessVariables();
    String addStepServiceTaskStatus = (String) variables.get("addStepServiceTask" + ActivitiRuntimeHelper.TASK_VARIABLE_MARKER + ActivitiRuntimeHelper.VARIABLE_STATUS);
    assertEquals(ActivitiRuntimeHelper.TASK_STATUS_ERROR, addStepServiceTaskStatus);
}