Examples with EventHandlerService org.apache.oozie.service.EventHandlerService used on opensource projects

Example 6 with EventHandlerService

use of org.apache.oozie.service.EventHandlerService in project oozie by apache.

the class TestSLAService method testEndMissDBConfirm.

@Test
public void testEndMissDBConfirm() throws Exception {
    SLAService slas = Services.get().get(SLAService.class);
    EventHandlerService ehs = Services.get().get(EventHandlerService.class);
    JPAService jpaService = Services.get().get(JPAService.class);
    Date date = new Date();
    // CASE 1: positive test WF job
    WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
    SLARegistrationBean sla = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
    // half hour back
    sla.setExpectedEnd(new Date(date.getTime() - 1 * 1800 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 2: negative test WF job
    WorkflowJobBean job2 = addRecordToWfJobTable(WorkflowJob.Status.SUCCEEDED, WorkflowInstance.Status.SUCCEEDED);
    job2.setEndTime(new Date(date.getTime() - 1 * 1800 * 1000));
    job2.setStartTime(new Date(date.getTime() - 1 * 2000 * 1000));
    job2.setLastModifiedTime(new Date());
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, job2);
    sla = _createSLARegistration(job2.getId(), AppType.WORKFLOW_JOB);
    // in past but > actual end
    sla.setExpectedEnd(new Date(date.getTime() - 1 * 1500 * 1000));
    // unreasonable to cause MISS
    sla.setExpectedDuration(100);
    slas.addRegistrationEvent(sla);
    slas.runSLAWorker();
    // CASE 3: positive test Coord action
    CoordinatorActionBean action1 = addRecordToCoordActionTable("coord-action-C@1", 1, CoordinatorAction.Status.WAITING, "coord-action-get.xml", 0);
    action1.setExternalId(null);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_RERUN, action1);
    sla = _createSLARegistration(action1.getId(), AppType.COORDINATOR_ACTION);
    // past
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 2000 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 4: positive test coord action
    CoordinatorActionBean action2 = addRecordToCoordActionTable("coord-action-C@2", 1, CoordinatorAction.Status.FAILED, "coord-action-get.xml", 0);
    WorkflowJobBean extWf = new WorkflowJobBean();
    extWf.setId(action2.getExternalId());
    // actual end before expected. but action is failed
    extWf.setEndTime(new Date(System.currentTimeMillis() - 1 * 1800 * 1000));
    extWf.setStartTime(new Date(System.currentTimeMillis() - 1 * 2000 * 1000));
    jpaService.execute(new WorkflowJobInsertJPAExecutor(extWf));
    sla = _createSLARegistration(action2.getId(), AppType.COORDINATOR_ACTION);
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1500 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 5: negative test coord action
    CoordinatorActionBean action3 = addRecordToCoordActionTable("coord-action-C@3", 1, CoordinatorAction.Status.SUCCEEDED, "coord-action-get.xml", 0);
    extWf = new WorkflowJobBean();
    extWf.setId(action3.getExternalId());
    extWf.setStartTime(new Date(System.currentTimeMillis() - 1 * 2100 * 1000));
    extWf.setEndTime(new Date(System.currentTimeMillis() - 1 * 1800 * 1000));
    jpaService.execute(new WorkflowJobInsertJPAExecutor(extWf));
    sla = _createSLARegistration(action3.getId(), AppType.COORDINATOR_ACTION);
    // cause start_miss
    sla.setExpectedStart(new Date(System.currentTimeMillis() - 1 * 3600 * 1000));
    // in past but > actual end, end_met
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1500 * 1000));
    // cause duration miss
    sla.setExpectedDuration(0);
    slas.addRegistrationEvent(sla);
    slas.runSLAWorker();
    ehs.new EventWorker().run();
    int count = 0;
    for (int ptr = output.indexOf("END - MISS"); ptr < output.length() && ptr > 0; ptr = output.indexOf("END - MISS", ptr + 1)) {
        count++;
    }
    // only 3 out of the 5 are correct end_misses
    assertEquals(3, count);
    assertEventNoDuplicates(output.toString(), job1.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), action1.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), action2.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), job2.getId() + " Sla END - MET!!!");
    assertEventNoDuplicates(output.toString(), job2.getId() + " Sla DURATION - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla START - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla DURATION - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla END - MET!!!");
    // negative on MISS after DB check, updated with actual times
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, job2.getId());
    assertEquals(job2.getStartTime(), slaSummary.getActualStart());
    assertEquals(job2.getEndTime(), slaSummary.getActualEnd());
    assertEquals(job2.getEndTime().getTime() - job2.getStartTime().getTime(), slaSummary.getActualDuration());
    assertEquals(job2.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MET, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MET, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
    // removed from memory
    assertNull(slas.getSLACalculator().get(job2.getId()));
    // positives but also updated with actual times immediately after DB check
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action2.getId());
    extWf = jpaService.execute(new WorkflowJobGetJPAExecutor(action2.getExternalId()));
    assertEquals(extWf.getStartTime(), slaSummary.getActualStart());
    assertEquals(extWf.getEndTime(), slaSummary.getActualEnd());
    assertEquals(extWf.getEndTime().getTime() - extWf.getStartTime().getTime(), slaSummary.getActualDuration());
    assertEquals(action2.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
    // removed from memory
    assertNull(slas.getSLACalculator().get(action2.getId()));
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action1.getId());
    assertNull(slaSummary.getActualStart());
    assertNull(slaSummary.getActualEnd());
    assertEquals(action1.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(7, slaSummary.getEventProcessed());
    assertNotNull(slas.getSLACalculator().get(action1.getId()));
    // From waiting to TIMEOUT with wf jobid
    action1.setStatus(CoordinatorAction.Status.TIMEDOUT);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_RERUN, action1);
    slas.getSLACalculator().addJobStatus(action1.getId(), null, null, null, null);
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action1.getId());
    assertNull(slaSummary.getActualStart());
    assertNotNull(slaSummary.getActualEnd());
    assertEquals("TIMEDOUT", slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
}

Example 7 with EventHandlerService

use of org.apache.oozie.service.EventHandlerService in project oozie by apache.

the class TestEventQueue method testMemoryEventQueueBasic.

@Test
public void testMemoryEventQueueBasic() throws Exception {
    EventHandlerService ehs = Services.get().get(EventHandlerService.class);
    assertNotNull(ehs);
    EventQueue eventQ = ehs.getEventQueue();
    assertNotNull(eventQ);
    // default
    assertTrue(eventQ instanceof MemoryEventQueue);
}

Example 8 with EventHandlerService

use of org.apache.oozie.service.EventHandlerService in project oozie by apache.

the class TestEventGeneration method setUp.

@Override
@Before
protected void setUp() throws Exception {
    super.setUp();
    services = new Services();
    Configuration conf = services.getConf();
    // The EventHandlerService manipulates the queues in the background, so the actual test results depend on the
    // circumstances (like the speed of the machine, debugging etc).
    conf.setInt("oozie.service.EventHandlerService.worker.threads", 0);
    conf.set(Services.CONF_SERVICE_EXT_CLASSES, "org.apache.oozie.service.EventHandlerService");
    services.init();
    ehs = services.get(EventHandlerService.class);
    queue = ehs.getEventQueue();
    jpaService = services.get(JPAService.class);
}

Example 9 with EventHandlerService

use of org.apache.oozie.service.EventHandlerService in project oozie by apache.

the class TestEventQueue method testQueueOperations.

@Test
public void testQueueOperations() throws Exception {
    EventHandlerService ehs = Services.get().get(EventHandlerService.class);
    EventQueue eventQ = ehs.getEventQueue();
    assertEquals(eventQ.size(), 0);
    assertEquals(eventQ.getBatchSize(), 3);
    // create some events to enqueue
    WorkflowJobEvent wfEvent = new WorkflowJobEvent("1234-W", "1234-C", WorkflowJob.Status.RUNNING, getTestUser(), "myapp", null, null);
    for (int i = 0; i < 10; i++) {
        ehs.queueEvent(wfEvent);
    }
    assertEquals(eventQ.size(), 10);
    // test single threads polling from queue
    int numThreads = 1;
    Thread[] thread = new Thread[numThreads];
    for (int i = 0; i < numThreads; i++) {
        thread[i] = new Thread(ehs.new EventWorker());
        thread[i].run();
    }
    // n(events) - n(batch) i.e.
    assertEquals(eventQ.size(), 7);
    // restore events count to 10
    for (int i = 0; i < 3; i++) {
        ehs.queueEvent(wfEvent);
    }
    assertEquals(eventQ.size(), 10);
    // test two threads polling concurrently from queue
    numThreads = 2;
    thread = new Thread[numThreads];
    for (int i = 0; i < numThreads; i++) {
        thread[i] = new Thread(ehs.new EventWorker());
        thread[i].run();
    }
    // n(events) - n(batch)*n(threads)
    assertEquals(eventQ.size(), 4);
    // enqueue events again
    for (int i = 0; i < 6; i++) {
        ehs.queueEvent(wfEvent);
    }
    assertEquals(eventQ.size(), 10);
    // test the 2 threads draining repeatedly (mimicking SchedulerService)
    // from queue
    int repetition = 3;
    int r = 0;
    while (r < repetition) {
        if (eventQ.isEmpty()) {
            break;
        }
        for (int i = 0; i < numThreads; i++) {
            thread[i].run();
        }
        r++;
    }
    assertEquals(eventQ.size(), 0);
}

Example 10 with EventHandlerService

use of org.apache.oozie.service.EventHandlerService in project oozie by apache.

the class TestSLACalculatorMemory method testSLAHistorySet.

public void testSLAHistorySet() throws Exception {
    EventHandlerService ehs = Services.get().get(EventHandlerService.class);
    SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
    slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
    WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
    SLARegistrationBean slaRegBean = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
    Date startTime = new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000);
    // 1 hour back
    slaRegBean.setExpectedStart(startTime);
    slaRegBean.setExpectedDuration(1000);
    slaRegBean.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000));
    String jobId = slaRegBean.getId();
    slaCalcMemory.addRegistration(slaRegBean.getId(), slaRegBean);
    slaCalcMemory.updateJobSla(jobId);
    job1.setId(job1.getId());
    job1.setStatus(WorkflowJob.Status.RUNNING);
    job1.setStartTime(new Date(System.currentTimeMillis() - 3600 * 1000));
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, job1);
    slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.RUNNING.toString(), EventStatus.STARTED, new Date(System.currentTimeMillis() - 3600 * 1000), null);
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    // The actual end times are not stored, but sla's processed so (111)
    assertEquals(7, slaSummary.getEventProcessed());
    // Moved from map to history set
    assertEquals(0, slaCalcMemory.size());
    // Add terminal state event so actual end time is stored
    job1.setId(job1.getId());
    job1.setStatus(WorkflowJob.Status.SUCCEEDED);
    job1.setEndTime(new Date(System.currentTimeMillis() - 3600 * 1000));
    job1.setStartTime(new Date(System.currentTimeMillis()));
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, job1);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, job1);
    slaCalcMemory.addJobStatus(jobId, WorkflowJob.Status.SUCCEEDED.toString(), EventStatus.SUCCESS, new Date(System.currentTimeMillis() - 3600 * 1000), new Date(System.currentTimeMillis()));
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId);
    // The actual times are stored, so event processed(1000)
    assertEquals(8, slaSummary.getEventProcessed());
    assertEquals(3, ehs.getEventQueue().size());
}