Examples with WorkflowJobBean org.apache.oozie.WorkflowJobBean used on opensource projects

Example 76 with WorkflowJobBean

use of org.apache.oozie.WorkflowJobBean in project oozie by apache.

the class TestSLACalculatorMemory method testWorkflowJobSLAStatusOnRestart.

@Test
public void testWorkflowJobSLAStatusOnRestart() throws Exception {
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-mm-dd");
    SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
    slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
    SLARegistrationBean slaRegBean1 = _createSLARegistration("job-1-W", AppType.WORKFLOW_JOB);
    final String jobId1 = slaRegBean1.getId();
    slaRegBean1.setExpectedEnd(sdf.parse("2013-03-07"));
    slaRegBean1.setExpectedStart(sdf.parse("2012-03-07"));
    slaCalcMemory.addRegistration(jobId1, slaRegBean1);
    SLACalcStatus calc1 = slaCalcMemory.get(jobId1);
    calc1.setEventProcessed(1);
    calc1.setSLAStatus(SLAEvent.SLAStatus.IN_PROCESS);
    calc1.setJobStatus(WorkflowJob.Status.RUNNING.name());
    calc1.setLastModifiedTime(new Date());
    SLASummaryBean slaSummaryBean = new SLASummaryBean(calc1);
    SLASummaryQueryExecutor.getInstance().executeUpdate(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL, slaSummaryBean);
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1);
    // Simulate a lost success event
    WorkflowJobBean wjb = new WorkflowJobBean();
    wjb.setId(jobId1);
    wjb.setStatus(WorkflowJob.Status.SUCCEEDED);
    wjb.setStartTime(sdf.parse("2012-02-07"));
    wjb.setEndTime(sdf.parse("2013-02-07"));
    wjb.setLastModifiedTime(new Date());
    WorkflowJobQueryExecutor.getInstance().insert(wjb);
    slaCalcMemory = new SLACalculatorMemory();
    slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
    slaCalcMemory.updateAllSlaStatus();
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1);
    assertEquals("job-1-W", slaSummary.getId());
    assertEquals(8, slaSummary.getEventProcessed());
    assertEquals(AppType.WORKFLOW_JOB, slaSummary.getAppType());
    assertEquals("SUCCEEDED", slaSummary.getJobStatus());
    assertEquals(SLAEvent.SLAStatus.MET, slaSummary.getSLAStatus());
    assertEquals(sdf.parse("2012-02-07"), slaSummary.getActualStart());
    assertEquals(sdf.parse("2013-02-07"), slaSummary.getActualEnd());
    assertEquals(sdf.parse("2013-02-07").getTime() - sdf.parse("2012-02-07").getTime(), slaSummary.getActualDuration());
    // Simulate a lost failed event
    wjb.setStatus(WorkflowJob.Status.FAILED);
    wjb.setLastModifiedTime(new Date());
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wjb);
    // Reset the summary Bean
    calc1.setEventProcessed(1);
    calc1.setSLAStatus(SLAEvent.SLAStatus.IN_PROCESS);
    calc1.setJobStatus(WorkflowJob.Status.RUNNING.name());
    slaSummaryBean = new SLASummaryBean(calc1);
    SLASummaryQueryExecutor.getInstance().executeUpdate(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL, slaSummaryBean);
    slaCalcMemory = new SLACalculatorMemory();
    slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
    slaCalcMemory.updateAllSlaStatus();
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1);
    assertEquals("FAILED", slaSummary.getJobStatus());
    assertEquals(8, slaSummary.getEventProcessed());
    assertEquals(sdf.parse("2012-02-07"), slaSummary.getActualStart());
    assertEquals(sdf.parse("2013-02-07"), slaSummary.getActualEnd());
    assertEquals(SLAEvent.SLAStatus.MISS, slaSummary.getSLAStatus());
    // Simulate a lost RUNNING event
    wjb.setStatus(WorkflowJob.Status.RUNNING);
    wjb.setLastModifiedTime(new Date());
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wjb);
    // Reset the summary Bean
    calc1.setEventProcessed(0);
    calc1.setSLAStatus(SLAEvent.SLAStatus.NOT_STARTED);
    calc1.setJobStatus(null);
    slaSummaryBean = new SLASummaryBean(calc1);
    SLASummaryQueryExecutor.getInstance().executeUpdate(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL, slaSummaryBean);
    SLAService slaService = Services.get().get(SLAService.class);
    slaService.startSLAWorker();
    slaService.addStatusEvent(jobId1, "RUNNING", null, null, null);
    waitFor(60 * 1000, new Predicate() {

        public boolean evaluate() throws Exception {
            return SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1).getEventProcessed() == 7;
        }
    });
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, slaSummaryBean.getId());
    // since job is already running and it's a old job
    assertEquals(7, slaSummary.getEventProcessed());
    assertEquals("RUNNING", slaSummary.getJobStatus());
    assertEquals(sdf.parse("2012-02-07"), slaSummary.getActualStart());
    assertNull(slaSummary.getActualEnd());
    assertEquals(-1, slaSummary.getActualDuration());
    assertEquals(SLAEvent.SLAStatus.MISS, slaSummary.getSLAStatus());
}

Example 77 with WorkflowJobBean

use of org.apache.oozie.WorkflowJobBean in project oozie by apache.

the class TestSLACalculatorMemory method testSingleAddUpdateRemoveInstrumentedCorrectly.

public void testSingleAddUpdateRemoveInstrumentedCorrectly() throws Exception {
    SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
    slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
    WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
    SLARegistrationBean slaRegBean = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
    Date startTime = new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000);
    // 1 hour back
    slaRegBean.setExpectedStart(startTime);
    slaRegBean.setExpectedDuration(1000);
    slaRegBean.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000));
    String jobId = slaRegBean.getId();
    slaCalcMemory.addRegistration(slaRegBean.getId(), slaRegBean);
    long slaMapSize = instrumentation.getCounters().get(SLACalculatorMemory.INSTRUMENTATION_GROUP).get(SLACalculatorMemory.SLA_MAP).getValue();
    assertEquals("SLA map size after add should be 1", 1, slaMapSize);
    slaCalcMemory.updateJobSla(jobId);
    slaMapSize = instrumentation.getCounters().get(SLACalculatorMemory.INSTRUMENTATION_GROUP).get(SLACalculatorMemory.SLA_MAP).getValue();
    assertEquals("SLA map size after update should be 1", 1, slaMapSize);
    slaCalcMemory.removeRegistration(jobId);
    slaMapSize = instrumentation.getCounters().get(SLACalculatorMemory.INSTRUMENTATION_GROUP).get(SLACalculatorMemory.SLA_MAP).getValue();
    assertEquals("SLA map size after remove should be 0", 0, slaMapSize);
}

Example 78 with WorkflowJobBean

use of org.apache.oozie.WorkflowJobBean in project oozie by apache.

the class TestSLACalculatorMemory method testDuplicateEndMiss.

@Test
public void testDuplicateEndMiss() throws Exception {
    EventHandlerService ehs = Services.get().get(EventHandlerService.class);
    SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
    slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
    WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    SLARegistrationBean slaRegBean = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
    // 1 hour ahead
    Date startTime = new Date(System.currentTimeMillis() + 1 * 1 * 3600 * 1000);
    slaRegBean.setExpectedStart(startTime);
    slaRegBean.setExpectedDuration(3600 * 1000);
    // 1 hour back
    slaRegBean.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000));
    String jobId = slaRegBean.getId();
    slaCalcMemory.addRegistration(slaRegBean.getId(), slaRegBean);
    slaCalcMemory.updateJobSla(jobId);
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    slaRegBean = SLARegistrationQueryExecutor.getInstance().get(SLARegQuery.GET_SLA_REG_ALL, jobId);
    assertNotNull(slaRegBean.getCreatedTimestamp());
    assertEquals(slaRegBean.getCreatedTimestamp(), slaSummary.getCreatedTimestamp());
    // Only end sla should be processed (100)
    assertEquals(4, slaSummary.getEventProcessed());
    slaCalcMemory.updateJobSla(jobId);
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    assertEquals(4, slaSummary.getEventProcessed());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    job1.setId(job1.getId());
    job1.setStatus(WorkflowJob.Status.SUCCEEDED);
    job1.setStartTime(new Date(System.currentTimeMillis()));
    job1.setEndTime(new Date(System.currentTimeMillis() + 1 * 1 * 3600 * 1000));
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, job1);
    slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.SUCCEEDED.toString(), EventStatus.SUCCESS, new Date(System.currentTimeMillis()), new Date(System.currentTimeMillis() + 1 * 1 * 3600 * 1000));
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    // Only Duration sla should be processed as end is already processed
    // (110)
    assertEquals(8, slaSummary.getEventProcessed());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    // Recieve start event
    assertTrue(slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.RUNNING.toString(), EventStatus.STARTED, new Date(System.currentTimeMillis()), new Date(System.currentTimeMillis() + 1 * 1 * 3600 * 1000)));
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    // Start event received so all bits should be processed (111)
    assertEquals(8, slaSummary.getEventProcessed());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(0, slaCalcMemory.size());
    assertEquals(3, ehs.getEventQueue().size());
}

Example 79 with WorkflowJobBean

use of org.apache.oozie.WorkflowJobBean in project oozie by apache.

the class TestSLACalculatorMemory method testSLAEvents2.

@Test
public void testSLAEvents2() throws Exception {
    SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
    EventHandlerService ehs = Services.get().get(EventHandlerService.class);
    slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
    WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
    SLARegistrationBean slaRegBean = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
    slaRegBean.setExpectedStart(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000));
    slaRegBean.setExpectedEnd(new Date(System.currentTimeMillis() + 2 * 1 * 3600 * 1000));
    String jobId = slaRegBean.getId();
    slaCalcMemory.addRegistration(jobId, slaRegBean);
    assertEquals(1, slaCalcMemory.size());
    slaCalcMemory.updateJobSla(jobId);
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    // Duration bit should be processed as expected duration is not set
    assertEquals(3, slaSummary.getEventProcessed());
    // check only start event in queue
    assertEquals(1, ehs.getEventQueue().size());
    ehs.getEventQueue().clear();
    // set back to 1, to make duration event not processed
    slaSummary.setEventProcessed(1);
    SLASummaryQueryExecutor.getInstance().executeUpdate(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL, slaSummary);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd");
    job1.setStatus(WorkflowJob.Status.SUCCEEDED);
    job1.setStartTime(sdf.parse("2012-01-01"));
    job1.setEndTime(sdf.parse("2012-01-02"));
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, job1);
    slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.SUCCEEDED.toString(), EventStatus.SUCCESS, sdf.parse("2012-01-01"), sdf.parse("2012-01-02"));
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    // all should be processed
    assertEquals(8, slaSummary.getEventProcessed());
    // check only end event is in queue
    assertEquals(1, ehs.getEventQueue().size());
    ehs.getEventQueue().clear();
    slaSummary.setEventProcessed(1);
    SLASummaryQueryExecutor.getInstance().executeUpdate(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL, slaSummary);
    WorkflowJobBean job2 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
    slaRegBean = _createSLARegistration(job2.getId(), AppType.WORKFLOW_JOB);
    slaRegBean.setExpectedStart(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000));
    slaRegBean.setExpectedEnd(new Date(System.currentTimeMillis() + 2 * 1 * 3600 * 1000));
    jobId = slaRegBean.getId();
    slaCalcMemory.addRegistration(jobId, slaRegBean);
    assertEquals(1, slaCalcMemory.size());
    job2.setStatus(WorkflowJob.Status.KILLED);
    job2.setEndTime(sdf.parse("2012-01-02"));
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, job2);
    slaCalcMemory.addJobStatus(job2.getId(), WorkflowJob.Status.KILLED.toString(), EventStatus.FAILURE, null, sdf.parse("2012-01-02"));
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    // Actual start null, so all events processed
    assertEquals(8, slaSummary.getEventProcessed());
    assertEquals(1, ehs.getEventQueue().size());
    assertNull(slaSummary.getActualStart());
    assertEquals(sdf.parse("2012-01-02"), slaSummary.getActualEnd());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
}

Example 80 with WorkflowJobBean

use of org.apache.oozie.WorkflowJobBean in project oozie by apache.

the class TestSLAEventGeneration method testWorkflowJobSLARerun.

/**
 * Test for SLA Events generated through wf rerun
 *
 * @throws Exception
 */
@Test
public void testWorkflowJobSLARerun() throws Exception {
    SLAService slas = services.get(SLAService.class);
    String wfXml = IOUtils.getResourceAsString("wf-job-sla.xml", -1);
    Path appPath = getFsTestCaseDir();
    writeToFile(wfXml, appPath, "workflow.xml");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, appPath.toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    cal.setTime(new Date());
    // for start_miss
    cal.add(Calendar.MINUTE, -40);
    Date nominal = cal.getTime();
    String nominalTime = DateUtils.formatDateOozieTZ(nominal);
    conf.set("nominal_time", nominalTime);
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 10);
    String expectedStart = DateUtils.formatDateOozieTZ(cal.getTime());
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 30);
    String expectedEnd = DateUtils.formatDateOozieTZ(cal.getTime());
    // Call SubmitX
    SubmitXCommand sc = new SubmitXCommand(conf);
    String jobId = sc.call();
    SLACalcStatus slaEvent = slas.getSLACalculator().get(jobId);
    assertEquals(jobId, slaEvent.getId());
    assertEquals("test-wf-job-sla", slaEvent.getAppName());
    assertEquals(AppType.WORKFLOW_JOB, slaEvent.getAppType());
    assertEquals(nominalTime, DateUtils.formatDateOozieTZ(slaEvent.getNominalTime()));
    assertEquals(expectedStart, DateUtils.formatDateOozieTZ(slaEvent.getExpectedStart()));
    assertEquals(expectedEnd, DateUtils.formatDateOozieTZ(slaEvent.getExpectedEnd()));
    slas.runSLAWorker();
    slaEvent = (SLACalcStatus) ehs.getEventQueue().poll();
    assertEquals(SLAStatus.NOT_STARTED, slaEvent.getSLAStatus());
    assertEquals(EventStatus.START_MISS, slaEvent.getEventStatus());
    slas.getSLACalculator().clear();
    JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean wfBean = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    // set job status to succeeded, so rerun doesn't fail
    wfBean.setStatus(WorkflowJob.Status.SUCCEEDED);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfBean);
    // change conf for rerun
    cal.setTime(new Date());
    // for start_miss
    cal.add(Calendar.MINUTE, -20);
    nominalTime = DateUtils.formatDateOozieTZ(cal.getTime());
    conf.set("nominal_time", nominalTime);
    nominal = cal.getTime();
    // as per the sla xml
    cal.add(Calendar.MINUTE, 10);
    expectedStart = DateUtils.formatDateOozieTZ(cal.getTime());
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 30);
    expectedEnd = DateUtils.formatDateOozieTZ(cal.getTime());
    ReRunXCommand rerun = new ReRunXCommand(jobId, conf);
    rerun.call();
    slaEvent = slas.getSLACalculator().get(jobId);
    // assert for new conf
    assertNotNull(slaEvent);
    assertEquals(jobId, slaEvent.getId());
    assertEquals("test-wf-job-sla", slaEvent.getAppName());
    assertEquals(AppType.WORKFLOW_JOB, slaEvent.getAppType());
    // assert for new conf
    assertEquals(nominalTime, DateUtils.formatDateOozieTZ(slaEvent.getNominalTime()));
    assertEquals(expectedStart, DateUtils.formatDateOozieTZ(slaEvent.getExpectedStart()));
    assertEquals(expectedEnd, DateUtils.formatDateOozieTZ(slaEvent.getExpectedEnd()));
    // assert for values in summary bean to be reset
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    assertEquals(0, slaSummary.getEventProcessed());
    assertEquals(-1, slaSummary.getActualDuration());
    assertNull(slaSummary.getActualStart());
    assertNull(slaSummary.getActualEnd());
    assertEquals("PREP", slaSummary.getJobStatus());
    assertEquals(SLAStatus.NOT_STARTED, slaSummary.getSLAStatus());
    assertNull(slaEvent.getEventStatus());
    ehs.getEventQueue().clear();
    slas.runSLAWorker();
    slaEvent = (SLACalcStatus) ehs.getEventQueue().poll();
    assertEquals(SLAStatus.IN_PROCESS, slaEvent.getSLAStatus());
    assertEquals(EventStatus.START_MISS, slaEvent.getEventStatus());
}