Examples with Job org.finra.herd.model.api.xml.Job used on opensource projects

Example 21 with Job

use of org.finra.herd.model.api.xml.Job in project herd by FINRAOS.

the class JobServiceTest method testSignalJobWithS3Properties.

/**
 * Signals job with S3 properties set. Parameters should be populated from the properties.
 *
 * @throws Exception
 */
@Test
public void testSignalJobWithS3Properties() throws Exception {
    jobDefinitionServiceTestHelper.createJobDefinition(ACTIVITI_XML_TEST_RECEIVE_TASK_WITH_CLASSPATH);
    Parameter parameter = new Parameter("testName", "testValue");
    S3PropertiesLocation s3PropertiesLocation = getS3PropertiesLocation("s3BucketName", "s3ObjectKey", parameter);
    // Start the job.
    Job job = jobService.createAndStartJob(jobServiceTestHelper.createJobCreateRequest(TEST_ACTIVITI_NAMESPACE_CD, TEST_ACTIVITI_JOB_NAME));
    JobSignalRequest jobSignalRequest = new JobSignalRequest(job.getId(), "receivetask1", null, null);
    jobSignalRequest.setS3PropertiesLocation(s3PropertiesLocation);
    Job signalJob = jobService.signalJob(jobSignalRequest);
    assertParameterEquals(parameter, signalJob.getParameters());
}

Example 22 with Job

use of org.finra.herd.model.api.xml.Job in project herd by FINRAOS.

the class JobServiceTest method testCreateJobWithS3PropertiesJobDefinitionWrongDatafixSafety.

/**
 * Tests an edge case where the job definition was persisted with some S3 properties location, but due to some datafix, the S3 properties location's object
 * key was removed, but not the bucket name. The service should still work, it would simply ignore the definition's S3 properties location.
 *
 * @throws Exception
 */
@Test
public void testCreateJobWithS3PropertiesJobDefinitionWrongDatafixSafety() throws Exception {
    // Create the namespace entity.
    namespaceDaoTestHelper.createNamespaceEntity(TEST_ACTIVITI_NAMESPACE_CD);
    // Create a job definition create request using hard coded test values.
    JobDefinitionCreateRequest jobDefinitionCreateRequest = jobDefinitionServiceTestHelper.createJobDefinitionCreateRequest();
    jobDefinitionCreateRequest.setS3PropertiesLocation(getS3PropertiesLocation("testBucketName", "testObjectKey", new Parameter("testName", "testValue")));
    jobDefinitionCreateRequest.setParameters(null);
    // Create the job definition.
    JobDefinition jobDefinition = jobDefinitionService.createJobDefinition(jobDefinitionCreateRequest, false);
    Integer jobDefinitionId = jobDefinition.getId();
    JobDefinitionEntity jobDefinitionEntity = herdDao.findById(JobDefinitionEntity.class, jobDefinitionId);
    jobDefinitionEntity.setS3ObjectKey(null);
    // Create a job create request using hard coded test values.
    JobCreateRequest jobCreateRequest = jobServiceTestHelper.createJobCreateRequest(TEST_ACTIVITI_NAMESPACE_CD, TEST_ACTIVITI_JOB_NAME);
    jobCreateRequest.setParameters(null);
    // Create the job.
    Job resultJob = jobService.createAndStartJob(jobCreateRequest);
    Assert.assertNotNull("resultJob parameters", resultJob.getParameters());
}

Example 23 with Job

use of org.finra.herd.model.api.xml.Job in project herd by FINRAOS.

the class JobServiceTest method testCreateJobWithS3PropertiesPrecedenceJobRequestS3OverridesDefinitionParams.

/**
 * Creates a job where the definition has S3 properties and parameters and request has S3 properties. The job create request's S3 properties should take
 * precedence if there are name clashes.
 *
 * @throws Exception
 */
@Test
public void testCreateJobWithS3PropertiesPrecedenceJobRequestS3OverridesDefinitionParams() throws Exception {
    Parameter jobDefinitionS3Parameter = new Parameter("testName", "testValue1");
    Parameter jobDefinitionRequestParameter = new Parameter("testName", "testValue2");
    Parameter jobCreateRequestS3Parameter = new Parameter("testName", "expectedValue");
    String s3BucketName = "s3BucketName";
    S3PropertiesLocation jobDefinitionS3PropertiesLocation = getS3PropertiesLocation(s3BucketName, "jobDefinitionObjectKey", jobDefinitionS3Parameter);
    S3PropertiesLocation jobCreateRequestS3PropertiesLocation = getS3PropertiesLocation(s3BucketName, "jobCreateRequestObjectKey", jobCreateRequestS3Parameter);
    Job resultJob = createJobWithParameters(jobDefinitionS3PropertiesLocation, Arrays.asList(jobDefinitionRequestParameter), jobCreateRequestS3PropertiesLocation, null);
    List<Parameter> actualParameters = resultJob.getParameters();
    assertParameterEquals(jobCreateRequestS3Parameter, actualParameters);
}

Example 24 with Job

use of org.finra.herd.model.api.xml.Job in project herd by FINRAOS.

the class JobServiceTest method testSignalJobWithCheckEmrClusterTask.

@Test
public void testSignalJobWithCheckEmrClusterTask() throws Exception {
    // Create EC2 on-demand pricing entities required for testing.
    ec2OnDemandPricingDaoTestHelper.createEc2OnDemandPricingEntities();
    // Create a list of parameters for the job.
    List<Parameter> parameters = new ArrayList<>();
    Parameter parameter = new Parameter("clusterName", EMR_CLUSTER_NAME);
    parameters.add(parameter);
    // Run a job with Activiti XML that will start cluster, check status, wait on receive task and terminate.
    Job job = jobServiceTestHelper.createJobForCreateCluster(ACTIVITI_XML_CHECK_CLUSTER_AND_RECEIVE_TASK_WITH_CLASSPATH, parameters);
    assertNotNull(job);
    // Job should be waiting at receive task.
    Job getJobResponse = jobService.getJob(job.getId(), false);
    assertEquals(JobStatusEnum.RUNNING, getJobResponse.getStatus());
    assertEquals("receiveTask", getJobResponse.getCurrentWorkflowStep().getId());
    // Validate that create and check cluster tasks were successful.
    assertTrue(getJobResponse.getParameters().contains(new Parameter("createClusterServiceTask_taskStatus", ActivitiRuntimeHelper.TASK_STATUS_SUCCESS)));
    assertTrue(getJobResponse.getParameters().contains(new Parameter("checkClusterServiceTask_taskStatus", ActivitiRuntimeHelper.TASK_STATUS_SUCCESS)));
    // Signal job to continue.
    Parameter signalParameter = new Parameter(PARAMETER_NAME, PARAMETER_VALUE);
    JobSignalRequest jobSignalRequest = new JobSignalRequest(job.getId(), "receiveTask", Collections.singletonList(signalParameter), null);
    Job signalJobResponse = jobService.signalJob(jobSignalRequest);
    // Validate the signal job response.
    assertEquals(JobStatusEnum.RUNNING, signalJobResponse.getStatus());
    assertEquals("receiveTask", signalJobResponse.getCurrentWorkflowStep().getId());
    assertTrue(signalJobResponse.getParameters().contains(signalParameter));
    // Validate the cluster status information.
    Map<String, Parameter> jobParameters = jobServiceTestHelper.toMap(signalJobResponse.getParameters());
    assertTrue(jobParameters.containsKey("checkClusterServiceTask_emrClusterStatus_creationTime"));
    assertTrue(jobParameters.containsKey("checkClusterServiceTask_emrClusterStatus_readyTime"));
    assertTrue(jobParameters.containsKey("checkClusterServiceTask_emrClusterStatus_endTime"));
    // Job should have been completed.
    getJobResponse = jobService.getJob(job.getId(), false);
    assertEquals(JobStatusEnum.COMPLETED, getJobResponse.getStatus());
    assertTrue(getJobResponse.getParameters().contains(signalParameter));
    // Get the process variables.
    HistoricProcessInstance historicProcessInstance = activitiHistoryService.createHistoricProcessInstanceQuery().processInstanceId(job.getId()).includeProcessVariables().singleResult();
    Map<String, Object> processVariables = historicProcessInstance.getProcessVariables();
    // Validate the cluster status information.
    assertTrue(processVariables.containsKey("checkClusterServiceTask_emrClusterStatus_creationTime"));
    assertNotNull(processVariables.get("checkClusterServiceTask_emrClusterStatus_creationTime"));
    assertTrue(processVariables.containsKey("checkClusterServiceTask_emrClusterStatus_readyTime"));
    assertNull(processVariables.get("checkClusterServiceTask_emrClusterStatus_readyTime"));
    assertTrue(processVariables.containsKey("checkClusterServiceTask_emrClusterStatus_endTime"));
    assertNull(processVariables.get("checkClusterServiceTask_emrClusterStatus_endTime"));
}

Example 25 with Job

use of org.finra.herd.model.api.xml.Job in project herd by FINRAOS.

the class JobServiceTest method testDeleteJobSuspendedJobWithMultipleSubProcesses.

@Test
@Transactional(propagation = Propagation.NOT_SUPPORTED)
public void testDeleteJobSuspendedJobWithMultipleSubProcesses() throws Exception {
    // Create and persist a test job definition.
    executeJdbcTestHelper.prepareHerdDatabaseForExecuteJdbcWithReceiveTaskTest(TEST_ACTIVITI_NAMESPACE_CD, TEST_ACTIVITI_JOB_NAME, ACTIVITI_XML_TEST_MULTIPLE_SUB_PROCESSES);
    try {
        // Get the job definition entity and ensure it exists.
        JobDefinitionEntity jobDefinitionEntity = jobDefinitionDao.getJobDefinitionByAltKey(TEST_ACTIVITI_NAMESPACE_CD, TEST_ACTIVITI_JOB_NAME);
        assertNotNull(jobDefinitionEntity);
        // Get the process definition id.
        String processDefinitionId = jobDefinitionEntity.getActivitiId();
        // Build the parameters map.
        Map<String, Object> parameters = new HashMap<>();
        parameters.put("counter", 0);
        // Start the job.
        ProcessInstance processInstance = activitiService.startProcessInstanceByProcessDefinitionId(processDefinitionId, parameters);
        assertNotNull(processInstance);
        // Get the process instance id for this job.
        String processInstanceId = processInstance.getProcessInstanceId();
        // Wait for all processes to become active - we expect to have the main process along with 800 sub-processes.
        waitUntilActiveProcessesThreshold(processDefinitionId, 801);
        // Get the job and validate that it is RUNNING.
        Job getJobResponse = jobService.getJob(processInstanceId, true);
        assertNotNull(getJobResponse);
        assertEquals(JobStatusEnum.RUNNING, getJobResponse.getStatus());
        // Suspend the job.
        jobService.updateJob(processInstanceId, new JobUpdateRequest(JobActionEnum.SUSPEND));
        // Get the job again and validate that it is now SUSPENDED.
        getJobResponse = jobService.getJob(processInstanceId, true);
        assertNotNull(getJobResponse);
        assertEquals(JobStatusEnum.SUSPENDED, getJobResponse.getStatus());
        // Delete the job in suspended state and validate the response.
        Job deleteJobResponse = jobService.deleteJob(processInstanceId, new JobDeleteRequest(ACTIVITI_JOB_DELETE_REASON));
        assertEquals(JobStatusEnum.COMPLETED, deleteJobResponse.getStatus());
        assertEquals(ACTIVITI_JOB_DELETE_REASON, deleteJobResponse.getDeleteReason());
        // Validate the historic process instance.
        HistoricProcessInstance historicProcessInstance = activitiHistoryService.createHistoricProcessInstanceQuery().processInstanceId(processInstanceId).singleResult();
        assertNotNull(historicProcessInstance);
        assertEquals(ACTIVITI_JOB_DELETE_REASON, historicProcessInstance.getDeleteReason());
    } finally {
        // Clean up the Herd database.
        executeJdbcTestHelper.cleanUpHerdDatabaseAfterExecuteJdbcWithReceiveTaskTest(TEST_ACTIVITI_NAMESPACE_CD, TEST_ACTIVITI_JOB_NAME);
        // Clean up the Activiti.
        deleteActivitiDeployments();
    }
}