Examples with WorkflowJobBean org.apache.oozie.WorkflowJobBean used on opensource projects

Example 86 with WorkflowJobBean

use of org.apache.oozie.WorkflowJobBean in project oozie by apache.

the class XDataTestCase method createWorkflow.

/**
 * Create workflow job bean
 *
 * @param app workflow app
 * @param conf workflow configuration
 * @param authToken auth token
 * @param jobStatus workflow job status
 * @param instanceStatus workflow instance status
 * @return workflow job bean
 * @throws Exception thrown if unable to create workflow job bean
 */
protected WorkflowJobBean createWorkflow(WorkflowApp app, Configuration conf, WorkflowJob.Status jobStatus, WorkflowInstance.Status instanceStatus) throws Exception {
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    Configuration protoActionConf = wps.createProtoActionConf(conf, true);
    WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
    WorkflowInstance wfInstance = workflowLib.createInstance(app, conf);
    ((LiteWorkflowInstance) wfInstance).setStatus(instanceStatus);
    WorkflowJobBean workflow = new WorkflowJobBean();
    workflow.setId(wfInstance.getId());
    workflow.setExternalId("extid");
    workflow.setAppName(app.getName());
    workflow.setAppPath(conf.get(OozieClient.APP_PATH));
    workflow.setConf(XmlUtils.prettyPrint(conf).toString());
    workflow.setProtoActionConf(XmlUtils.prettyPrint(protoActionConf).toString());
    workflow.setCreatedTime(new Date());
    workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
    workflow.setStatus(jobStatus);
    workflow.setRun(0);
    workflow.setUser(conf.get(OozieClient.USER_NAME));
    workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
    workflow.setWorkflowInstance(wfInstance);
    workflow.setSlaXml("<sla></sla>");
    return workflow;
}

Example 87 with WorkflowJobBean

use of org.apache.oozie.WorkflowJobBean in project oozie by apache.

the class TestSLAService method testEndMissDBConfirm.

@Test
public void testEndMissDBConfirm() throws Exception {
    SLAService slas = Services.get().get(SLAService.class);
    EventHandlerService ehs = Services.get().get(EventHandlerService.class);
    JPAService jpaService = Services.get().get(JPAService.class);
    Date date = new Date();
    // CASE 1: positive test WF job
    WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
    SLARegistrationBean sla = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
    // half hour back
    sla.setExpectedEnd(new Date(date.getTime() - 1 * 1800 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 2: negative test WF job
    WorkflowJobBean job2 = addRecordToWfJobTable(WorkflowJob.Status.SUCCEEDED, WorkflowInstance.Status.SUCCEEDED);
    job2.setEndTime(new Date(date.getTime() - 1 * 1800 * 1000));
    job2.setStartTime(new Date(date.getTime() - 1 * 2000 * 1000));
    job2.setLastModifiedTime(new Date());
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, job2);
    sla = _createSLARegistration(job2.getId(), AppType.WORKFLOW_JOB);
    // in past but > actual end
    sla.setExpectedEnd(new Date(date.getTime() - 1 * 1500 * 1000));
    // unreasonable to cause MISS
    sla.setExpectedDuration(100);
    slas.addRegistrationEvent(sla);
    slas.runSLAWorker();
    // CASE 3: positive test Coord action
    CoordinatorActionBean action1 = addRecordToCoordActionTable("coord-action-C@1", 1, CoordinatorAction.Status.WAITING, "coord-action-get.xml", 0);
    action1.setExternalId(null);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_RERUN, action1);
    sla = _createSLARegistration(action1.getId(), AppType.COORDINATOR_ACTION);
    // past
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 2000 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 4: positive test coord action
    CoordinatorActionBean action2 = addRecordToCoordActionTable("coord-action-C@2", 1, CoordinatorAction.Status.FAILED, "coord-action-get.xml", 0);
    WorkflowJobBean extWf = new WorkflowJobBean();
    extWf.setId(action2.getExternalId());
    // actual end before expected. but action is failed
    extWf.setEndTime(new Date(System.currentTimeMillis() - 1 * 1800 * 1000));
    extWf.setStartTime(new Date(System.currentTimeMillis() - 1 * 2000 * 1000));
    jpaService.execute(new WorkflowJobInsertJPAExecutor(extWf));
    sla = _createSLARegistration(action2.getId(), AppType.COORDINATOR_ACTION);
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1500 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 5: negative test coord action
    CoordinatorActionBean action3 = addRecordToCoordActionTable("coord-action-C@3", 1, CoordinatorAction.Status.SUCCEEDED, "coord-action-get.xml", 0);
    extWf = new WorkflowJobBean();
    extWf.setId(action3.getExternalId());
    extWf.setStartTime(new Date(System.currentTimeMillis() - 1 * 2100 * 1000));
    extWf.setEndTime(new Date(System.currentTimeMillis() - 1 * 1800 * 1000));
    jpaService.execute(new WorkflowJobInsertJPAExecutor(extWf));
    sla = _createSLARegistration(action3.getId(), AppType.COORDINATOR_ACTION);
    // cause start_miss
    sla.setExpectedStart(new Date(System.currentTimeMillis() - 1 * 3600 * 1000));
    // in past but > actual end, end_met
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1500 * 1000));
    // cause duration miss
    sla.setExpectedDuration(0);
    slas.addRegistrationEvent(sla);
    slas.runSLAWorker();
    ehs.new EventWorker().run();
    int count = 0;
    for (int ptr = output.indexOf("END - MISS"); ptr < output.length() && ptr > 0; ptr = output.indexOf("END - MISS", ptr + 1)) {
        count++;
    }
    // only 3 out of the 5 are correct end_misses
    assertEquals(3, count);
    assertEventNoDuplicates(output.toString(), job1.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), action1.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), action2.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), job2.getId() + " Sla END - MET!!!");
    assertEventNoDuplicates(output.toString(), job2.getId() + " Sla DURATION - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla START - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla DURATION - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla END - MET!!!");
    // negative on MISS after DB check, updated with actual times
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, job2.getId());
    assertEquals(job2.getStartTime(), slaSummary.getActualStart());
    assertEquals(job2.getEndTime(), slaSummary.getActualEnd());
    assertEquals(job2.getEndTime().getTime() - job2.getStartTime().getTime(), slaSummary.getActualDuration());
    assertEquals(job2.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MET, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MET, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
    // removed from memory
    assertNull(slas.getSLACalculator().get(job2.getId()));
    // positives but also updated with actual times immediately after DB check
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action2.getId());
    extWf = jpaService.execute(new WorkflowJobGetJPAExecutor(action2.getExternalId()));
    assertEquals(extWf.getStartTime(), slaSummary.getActualStart());
    assertEquals(extWf.getEndTime(), slaSummary.getActualEnd());
    assertEquals(extWf.getEndTime().getTime() - extWf.getStartTime().getTime(), slaSummary.getActualDuration());
    assertEquals(action2.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
    // removed from memory
    assertNull(slas.getSLACalculator().get(action2.getId()));
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action1.getId());
    assertNull(slaSummary.getActualStart());
    assertNull(slaSummary.getActualEnd());
    assertEquals(action1.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(7, slaSummary.getEventProcessed());
    assertNotNull(slas.getSLACalculator().get(action1.getId()));
    // From waiting to TIMEOUT with wf jobid
    action1.setStatus(CoordinatorAction.Status.TIMEDOUT);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_RERUN, action1);
    slas.getSLACalculator().addJobStatus(action1.getId(), null, null, null, null);
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action1.getId());
    assertNull(slaSummary.getActualStart());
    assertNotNull(slaSummary.getActualEnd());
    assertEquals("TIMEDOUT", slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
}

Example 88 with WorkflowJobBean

use of org.apache.oozie.WorkflowJobBean in project oozie by apache.

the class TestDBWorkflowStore method _testPurge.

private void _testPurge() throws Exception {
    store.beginTrx();
    wfBean1.setEndTime(new Date(System.currentTimeMillis() - (31 * 24 * 60 * 60 * 1000l)));
    wfBean2.setEndTime(new Date(System.currentTimeMillis() - (31 * 24 * 60 * 60 * 1000l)));
    WorkflowApp app = new LiteWorkflowApp("testApp", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    Configuration conf2 = new Configuration();
    conf2.set(OozieClient.APP_PATH, "testPath");
    conf2.set(OozieClient.LOG_TOKEN, "testToken");
    conf2.set(OozieClient.USER_NAME, getTestUser2());
    WorkflowJobBean wfBean3 = createWorkflow(app, conf2, "auth");
    store.insertWorkflow(wfBean3);
    store.updateWorkflow(wfBean2);
    store.updateWorkflow(wfBean1);
    store.commitTrx();
    WorkflowActionBean a31 = new WorkflowActionBean();
    StringBuilder str = new StringBuilder();
    str.append(System.currentTimeMillis());
    str.append("31");
    a31.setId(str.toString());
    a31.setJobId(wfBean3.getId());
    a31.setStatus(WorkflowAction.Status.PREP);
    store.beginTrx();
    store.insertAction(a31);
    store.commitTrx();
    store.beginTrx();
    store.purge(30, 10000);
    store.commitTrx();
/*
 * SqlStatement s = getCount(OozieTable.WF_JOBS); ResultSet rs =
 * s.prepareAndSetValues(conn).executeQuery(); rs.next(); assertEquals(3,
 * rs.getInt(1)); rs.close();
 *
 * s = getCount(OozieTable.WF_ACTIONS); rs =
 * s.prepareAndSetValues(conn).executeQuery(); rs.next(); assertEquals(4,
 * rs.getInt(1)); rs.close();
 *
 * store.purge(30); store.commit();
 *
 * s = getCount(OozieTable.WF_JOBS); rs =
 * s.prepareAndSetValues(conn).executeQuery(); rs.next(); assertEquals(1,
 * rs.getInt(1)); rs.close();
 *
 * WorkflowJobBean tmp = store.getWorkflow(wfBean3.getId(), false);
 * assertEquals(tmp.getId(), wfBean3.getId());
 *
 * s = getCount(OozieTable.WF_ACTIONS); rs =
 * s.prepareAndSetValues(conn).executeQuery(); rs.next(); assertEquals(1,
 * rs.getInt(1)); rs.close();
 *
 * WorkflowActionBean tmpa = store.getAction("31", false); assertEquals("31",
 * tmpa.getId());
 */
}

Example 89 with WorkflowJobBean

use of org.apache.oozie.WorkflowJobBean in project oozie by apache.

the class TestDBWorkflowStore method _testUpdateWF.

private void _testUpdateWF() throws StoreException {
    wfBean1.setStatus(WorkflowJob.Status.SUCCEEDED);
    WorkflowInstance wfInstance = wfBean1.getWorkflowInstance();
    wfInstance.setVar("test", "hello");
    wfBean1.setWorkflowInstance(wfInstance);
    wfBean1.setExternalId("testExtId");
    store.getWorkflow(wfBean1.getId(), false);
    store.updateWorkflow(wfBean1);
    WorkflowJobBean wfBean = store.getWorkflow(wfBean1.getId(), false);
    assertEquals("hello", wfBean.getWorkflowInstance().getVar("test"));
    assertEquals(wfBean.getStatus(), WorkflowJob.Status.SUCCEEDED);
    store.commitTrx();
}

Example 90 with WorkflowJobBean

use of org.apache.oozie.WorkflowJobBean in project oozie by apache.

the class TestDBWorkflowStore method _testGetStatusCount.

private void _testGetStatusCount() throws StoreException, InterruptedException {
    store.beginTrx();
    // assertEquals(1,
    // store.getWorkflowCountWithStatus(WorkflowJob.Status.PREP.name()));
    // assertEquals(1,
    // store.getWorkflowCountWithStatus(WorkflowJob.Status.SUCCEEDED.name()));
    // assertEquals(1,
    // store.getWorkflowCountWithStatusInLastNSeconds(WorkflowJob.Status.PREP.name(),
    // 5));
    // assertEquals(1,
    // store.getWorkflowCountWithStatusInLastNSeconds(WorkflowJob.Status.SUCCEEDED.name(),
    // 5));
    sleep(1000);
    long t1 = System.currentTimeMillis();
    WorkflowJobBean wfBean = store.getWorkflow(wfBean2.getId(), false);
    store.updateWorkflow(wfBean);
    long t2 = System.currentTimeMillis();
    int s = (int) ((t2 - t1) / 1000);
    if (s < 1) {
        s = 1;
    }
    // assertEquals(1,
    // store.getWorkflowCountWithStatusInLastNSeconds(WorkflowJob.Status.PREP.name(),
    // s));
    // assertEquals(0,
    // store.getWorkflowCountWithStatusInLastNSeconds(WorkflowJob.Status.SUCCEEDED.name(),
    // s));
    store.commitTrx();
}