Examples with SLAService org.apache.oozie.sla.service.SLAService used on opensource projects

Example 6 with SLAService

use of org.apache.oozie.sla.service.SLAService in project oozie by apache.

the class TestSLAEventGeneration method testCoordinatorActionCommandsSubmitAndStart.

/**
 * Test for SLA Events generated through Coordinator Action commands
 * CoordSubmitX and CoordStartX
 *
 * @throws Exception
 */
@Test
public void testCoordinatorActionCommandsSubmitAndStart() throws Exception {
    // reduce noise from WF Job events (also default) by setting it to only
    // coord action
    ehs.setAppTypes(new HashSet<String>(Arrays.asList(new String[] { "coordinator_action" })));
    ehs.getEventQueue().clear();
    SLAService slas = services.get(SLAService.class);
    String coordXml = IOUtils.getResourceAsString("coord-action-sla.xml", -1);
    Path appPath = getFsTestCaseDir();
    writeToFile(coordXml, appPath, "coordinator.xml");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPath.toString());
    String wfXml = IOUtils.getResourceAsString("wf-credentials.xml", -1);
    writeToFile(wfXml, appPath, "workflow.xml");
    conf.set("wfAppPath", appPath.toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    cal.setTime(new Date());
    // for start_miss
    cal.add(Calendar.MINUTE, -20);
    Date nominal = cal.getTime();
    String nominalTime = DateUtils.formatDateOozieTZ(nominal);
    conf.set("nominal_time", nominalTime);
    conf.set("start", "2009-01-02T08:01Z");
    conf.set("frequency", "coord:days(1)");
    conf.set("end", "2009-01-03T08:00Z");
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 10);
    String expectedStart = DateUtils.formatDateOozieTZ(cal.getTime());
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 30);
    String expectedEnd = DateUtils.formatDateOozieTZ(cal.getTime());
    String appName = "test-coord-sla";
    // testing creation of new sla registration via Submit + Materialize
    // command
    String jobId = new CoordSubmitXCommand(conf).call();
    // waiting for materialize command to run
    Thread.sleep(500);
    final CoordActionGetJPAExecutor getCmd = new CoordActionGetJPAExecutor(jobId + "@1");
    CoordinatorActionBean action = jpa.execute(getCmd);
    String actionId = action.getId();
    SLACalcStatus slaEvent = slas.getSLACalculator().get(actionId);
    assertEquals(actionId, slaEvent.getId());
    assertEquals(appName, slaEvent.getAppName());
    assertEquals(AppType.COORDINATOR_ACTION, slaEvent.getAppType());
    assertEquals(nominalTime, DateUtils.formatDateOozieTZ(slaEvent.getNominalTime()));
    assertEquals(expectedStart, DateUtils.formatDateOozieTZ(slaEvent.getExpectedStart()));
    assertEquals(expectedEnd, DateUtils.formatDateOozieTZ(slaEvent.getExpectedEnd()));
    assertEquals(30 * 60 * 1000, slaEvent.getExpectedDuration());
    assertEquals(alert_events, slaEvent.getAlertEvents());
    slas.runSLAWorker();
    slaEvent = skipToSLAEvent();
    assertTrue(SLAStatus.NOT_STARTED == slaEvent.getSLAStatus());
    assertEquals(EventStatus.START_MISS, slaEvent.getEventStatus());
    // test that sla processes the Job Event from Start command
    ehs.getEventQueue().clear();
    action.setStatus(CoordinatorAction.Status.SUBMITTED);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_STATUS_PENDING_TIME, action);
    new CoordActionStartXCommand(actionId, getTestUser(), appName, jobId).call();
    slaEvent = slas.getSLACalculator().get(actionId);
    // resetting for testing sla event
    slaEvent.setEventProcessed(0);
    SLASummaryBean suBean = new SLASummaryBean();
    suBean.setId(actionId);
    suBean.setEventProcessed(0);
    SLASummaryQueryExecutor.getInstance().executeUpdate(SLASummaryQuery.UPDATE_SLA_SUMMARY_EVENTPROCESSED, suBean);
    ehs.new EventWorker().run();
    slaEvent = skipToSLAEvent();
    assertEquals(actionId, slaEvent.getId());
    assertNotNull(slaEvent.getActualStart());
    assertEquals(SLAStatus.IN_PROCESS, slaEvent.getSLAStatus());
    assertEquals(CoordinatorAction.Status.RUNNING.name(), slaEvent.getJobStatus());
}

Example 7 with SLAService

use of org.apache.oozie.sla.service.SLAService in project oozie by apache.

the class TestSLAEventGeneration method testWorkflowJobSLARerun.

/**
 * Test for SLA Events generated through wf rerun
 *
 * @throws Exception
 */
@Test
public void testWorkflowJobSLARerun() throws Exception {
    SLAService slas = services.get(SLAService.class);
    String wfXml = IOUtils.getResourceAsString("wf-job-sla.xml", -1);
    Path appPath = getFsTestCaseDir();
    writeToFile(wfXml, appPath, "workflow.xml");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, appPath.toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    cal.setTime(new Date());
    // for start_miss
    cal.add(Calendar.MINUTE, -40);
    Date nominal = cal.getTime();
    String nominalTime = DateUtils.formatDateOozieTZ(nominal);
    conf.set("nominal_time", nominalTime);
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 10);
    String expectedStart = DateUtils.formatDateOozieTZ(cal.getTime());
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 30);
    String expectedEnd = DateUtils.formatDateOozieTZ(cal.getTime());
    // Call SubmitX
    SubmitXCommand sc = new SubmitXCommand(conf);
    String jobId = sc.call();
    SLACalcStatus slaEvent = slas.getSLACalculator().get(jobId);
    assertEquals(jobId, slaEvent.getId());
    assertEquals("test-wf-job-sla", slaEvent.getAppName());
    assertEquals(AppType.WORKFLOW_JOB, slaEvent.getAppType());
    assertEquals(nominalTime, DateUtils.formatDateOozieTZ(slaEvent.getNominalTime()));
    assertEquals(expectedStart, DateUtils.formatDateOozieTZ(slaEvent.getExpectedStart()));
    assertEquals(expectedEnd, DateUtils.formatDateOozieTZ(slaEvent.getExpectedEnd()));
    slas.runSLAWorker();
    slaEvent = (SLACalcStatus) ehs.getEventQueue().poll();
    assertEquals(SLAStatus.NOT_STARTED, slaEvent.getSLAStatus());
    assertEquals(EventStatus.START_MISS, slaEvent.getEventStatus());
    slas.getSLACalculator().clear();
    JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean wfBean = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    // set job status to succeeded, so rerun doesn't fail
    wfBean.setStatus(WorkflowJob.Status.SUCCEEDED);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfBean);
    // change conf for rerun
    cal.setTime(new Date());
    // for start_miss
    cal.add(Calendar.MINUTE, -20);
    nominalTime = DateUtils.formatDateOozieTZ(cal.getTime());
    conf.set("nominal_time", nominalTime);
    nominal = cal.getTime();
    // as per the sla xml
    cal.add(Calendar.MINUTE, 10);
    expectedStart = DateUtils.formatDateOozieTZ(cal.getTime());
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 30);
    expectedEnd = DateUtils.formatDateOozieTZ(cal.getTime());
    ReRunXCommand rerun = new ReRunXCommand(jobId, conf);
    rerun.call();
    slaEvent = slas.getSLACalculator().get(jobId);
    // assert for new conf
    assertNotNull(slaEvent);
    assertEquals(jobId, slaEvent.getId());
    assertEquals("test-wf-job-sla", slaEvent.getAppName());
    assertEquals(AppType.WORKFLOW_JOB, slaEvent.getAppType());
    // assert for new conf
    assertEquals(nominalTime, DateUtils.formatDateOozieTZ(slaEvent.getNominalTime()));
    assertEquals(expectedStart, DateUtils.formatDateOozieTZ(slaEvent.getExpectedStart()));
    assertEquals(expectedEnd, DateUtils.formatDateOozieTZ(slaEvent.getExpectedEnd()));
    // assert for values in summary bean to be reset
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    assertEquals(0, slaSummary.getEventProcessed());
    assertEquals(-1, slaSummary.getActualDuration());
    assertNull(slaSummary.getActualStart());
    assertNull(slaSummary.getActualEnd());
    assertEquals("PREP", slaSummary.getJobStatus());
    assertEquals(SLAStatus.NOT_STARTED, slaSummary.getSLAStatus());
    assertNull(slaEvent.getEventStatus());
    ehs.getEventQueue().clear();
    slas.runSLAWorker();
    slaEvent = (SLACalcStatus) ehs.getEventQueue().poll();
    assertEquals(SLAStatus.IN_PROCESS, slaEvent.getSLAStatus());
    assertEquals(EventStatus.START_MISS, slaEvent.getEventStatus());
}

Example 8 with SLAService

use of org.apache.oozie.sla.service.SLAService in project oozie by apache.

the class TestSLAEventGeneration method testWorkflowActionSLARerun.

/**
 * Test for SLA Events generated through wf action rerun
 *
 * @throws Exception
 */
@Test
public void testWorkflowActionSLARerun() throws Exception {
    SLAService slas = services.get(SLAService.class);
    String wfXml = IOUtils.getResourceAsString("wf-action-sla.xml", -1);
    Path appPath = getFsTestCaseDir();
    writeToFile(wfXml, appPath, "workflow.xml");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, appPath.toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    cal.setTime(new Date());
    // for start_miss
    cal.add(Calendar.MINUTE, -20);
    Date nominal = cal.getTime();
    String nominalTime = DateUtils.formatDateOozieTZ(nominal);
    conf.set("nominal_time", nominalTime);
    // Call SubmitX
    SubmitXCommand sc = new SubmitXCommand(conf);
    String jobId = sc.call();
    String actionId = jobId + "@grouper";
    slas.getSLACalculator().clear();
    JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean wfBean = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    // set job status to succeeded, so rerun doesn't fail
    wfBean.setStatus(WorkflowJob.Status.SUCCEEDED);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_MODTIME, wfBean);
    // change conf for rerun
    cal.setTime(new Date());
    nominalTime = DateUtils.formatDateOozieTZ(cal.getTime());
    conf.set("nominal_time", nominalTime);
    nominal = cal.getTime();
    // as per the sla xml
    cal.add(Calendar.MINUTE, 10);
    String expectedStart = DateUtils.formatDateOozieTZ(cal.getTime());
    cal.setTime(nominal);
    // as per the sla xml
    cal.add(Calendar.MINUTE, 30);
    String expectedEnd = DateUtils.formatDateOozieTZ(cal.getTime());
    ReRunXCommand rerun = new ReRunXCommand(jobId, conf);
    rerun.call();
    SLACalcStatus slaEvent = slas.getSLACalculator().get(actionId);
    assertNotNull(slaEvent);
    // assert for action configs
    assertEquals(actionId, slaEvent.getId());
    assertEquals("test-wf-action-sla", slaEvent.getAppName());
    assertEquals(AppType.WORKFLOW_ACTION, slaEvent.getAppType());
    // assert for new conf
    assertEquals(nominalTime, DateUtils.formatDateOozieTZ(slaEvent.getNominalTime()));
    assertEquals(expectedStart, DateUtils.formatDateOozieTZ(slaEvent.getExpectedStart()));
    assertEquals(expectedEnd, DateUtils.formatDateOozieTZ(slaEvent.getExpectedEnd()));
}

Example 9 with SLAService

use of org.apache.oozie.sla.service.SLAService in project oozie by apache.

the class TestSLAEventGeneration method testSLASchema1BackwardCompatibilityKill.

@Test
public void testSLASchema1BackwardCompatibilityKill() throws Exception {
    assertNotNull(ehs);
    SLAService slas = services.get(SLAService.class);
    assertNotNull(slas);
    Path appPath = getFsTestCaseDir();
    writeToFile(SLA_XML_1, appPath, "workflow.xml");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.APP_PATH, appPath.toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    cal.setTime(new Date());
    // for start_miss
    cal.add(Calendar.MINUTE, -20);
    Date nominal = cal.getTime();
    String nominalTime = DateUtils.formatDateOozieTZ(nominal);
    conf.set("nominal_time", nominalTime);
    _testWorkflowJobKillCommand(conf, slas);
}

Example 10 with SLAService

use of org.apache.oozie.sla.service.SLAService in project oozie by apache.

the class TestSLAService method testEndMissDBConfirm.

@Test
public void testEndMissDBConfirm() throws Exception {
    SLAService slas = Services.get().get(SLAService.class);
    EventHandlerService ehs = Services.get().get(EventHandlerService.class);
    JPAService jpaService = Services.get().get(JPAService.class);
    Date date = new Date();
    // CASE 1: positive test WF job
    WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
    SLARegistrationBean sla = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
    // half hour back
    sla.setExpectedEnd(new Date(date.getTime() - 1 * 1800 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 2: negative test WF job
    WorkflowJobBean job2 = addRecordToWfJobTable(WorkflowJob.Status.SUCCEEDED, WorkflowInstance.Status.SUCCEEDED);
    job2.setEndTime(new Date(date.getTime() - 1 * 1800 * 1000));
    job2.setStartTime(new Date(date.getTime() - 1 * 2000 * 1000));
    job2.setLastModifiedTime(new Date());
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, job2);
    sla = _createSLARegistration(job2.getId(), AppType.WORKFLOW_JOB);
    // in past but > actual end
    sla.setExpectedEnd(new Date(date.getTime() - 1 * 1500 * 1000));
    // unreasonable to cause MISS
    sla.setExpectedDuration(100);
    slas.addRegistrationEvent(sla);
    slas.runSLAWorker();
    // CASE 3: positive test Coord action
    CoordinatorActionBean action1 = addRecordToCoordActionTable("coord-action-C@1", 1, CoordinatorAction.Status.WAITING, "coord-action-get.xml", 0);
    action1.setExternalId(null);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_RERUN, action1);
    sla = _createSLARegistration(action1.getId(), AppType.COORDINATOR_ACTION);
    // past
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 2000 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 4: positive test coord action
    CoordinatorActionBean action2 = addRecordToCoordActionTable("coord-action-C@2", 1, CoordinatorAction.Status.FAILED, "coord-action-get.xml", 0);
    WorkflowJobBean extWf = new WorkflowJobBean();
    extWf.setId(action2.getExternalId());
    // actual end before expected. but action is failed
    extWf.setEndTime(new Date(System.currentTimeMillis() - 1 * 1800 * 1000));
    extWf.setStartTime(new Date(System.currentTimeMillis() - 1 * 2000 * 1000));
    jpaService.execute(new WorkflowJobInsertJPAExecutor(extWf));
    sla = _createSLARegistration(action2.getId(), AppType.COORDINATOR_ACTION);
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1500 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 5: negative test coord action
    CoordinatorActionBean action3 = addRecordToCoordActionTable("coord-action-C@3", 1, CoordinatorAction.Status.SUCCEEDED, "coord-action-get.xml", 0);
    extWf = new WorkflowJobBean();
    extWf.setId(action3.getExternalId());
    extWf.setStartTime(new Date(System.currentTimeMillis() - 1 * 2100 * 1000));
    extWf.setEndTime(new Date(System.currentTimeMillis() - 1 * 1800 * 1000));
    jpaService.execute(new WorkflowJobInsertJPAExecutor(extWf));
    sla = _createSLARegistration(action3.getId(), AppType.COORDINATOR_ACTION);
    // cause start_miss
    sla.setExpectedStart(new Date(System.currentTimeMillis() - 1 * 3600 * 1000));
    // in past but > actual end, end_met
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1500 * 1000));
    // cause duration miss
    sla.setExpectedDuration(0);
    slas.addRegistrationEvent(sla);
    slas.runSLAWorker();
    ehs.new EventWorker().run();
    int count = 0;
    for (int ptr = output.indexOf("END - MISS"); ptr < output.length() && ptr > 0; ptr = output.indexOf("END - MISS", ptr + 1)) {
        count++;
    }
    // only 3 out of the 5 are correct end_misses
    assertEquals(3, count);
    assertEventNoDuplicates(output.toString(), job1.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), action1.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), action2.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), job2.getId() + " Sla END - MET!!!");
    assertEventNoDuplicates(output.toString(), job2.getId() + " Sla DURATION - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla START - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla DURATION - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla END - MET!!!");
    // negative on MISS after DB check, updated with actual times
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, job2.getId());
    assertEquals(job2.getStartTime(), slaSummary.getActualStart());
    assertEquals(job2.getEndTime(), slaSummary.getActualEnd());
    assertEquals(job2.getEndTime().getTime() - job2.getStartTime().getTime(), slaSummary.getActualDuration());
    assertEquals(job2.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MET, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MET, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
    // removed from memory
    assertNull(slas.getSLACalculator().get(job2.getId()));
    // positives but also updated with actual times immediately after DB check
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action2.getId());
    extWf = jpaService.execute(new WorkflowJobGetJPAExecutor(action2.getExternalId()));
    assertEquals(extWf.getStartTime(), slaSummary.getActualStart());
    assertEquals(extWf.getEndTime(), slaSummary.getActualEnd());
    assertEquals(extWf.getEndTime().getTime() - extWf.getStartTime().getTime(), slaSummary.getActualDuration());
    assertEquals(action2.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
    // removed from memory
    assertNull(slas.getSLACalculator().get(action2.getId()));
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action1.getId());
    assertNull(slaSummary.getActualStart());
    assertNull(slaSummary.getActualEnd());
    assertEquals(action1.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(7, slaSummary.getEventProcessed());
    assertNotNull(slas.getSLACalculator().get(action1.getId()));
    // From waiting to TIMEOUT with wf jobid
    action1.setStatus(CoordinatorAction.Status.TIMEDOUT);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_RERUN, action1);
    slas.getSLACalculator().addJobStatus(action1.getId(), null, null, null, null);
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action1.getId());
    assertNull(slaSummary.getActualStart());
    assertNotNull(slaSummary.getActualEnd());
    assertEquals("TIMEDOUT", slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
}