Examples with WorkflowJobInsertJPAExecutor org.apache.oozie.executor.jpa.WorkflowJobInsertJPAExecutor used on opensource projects

Example 6 with WorkflowJobInsertJPAExecutor

use of org.apache.oozie.executor.jpa.WorkflowJobInsertJPAExecutor in project oozie by apache.

the class TestSLAService method testEndMissDBConfirm.

@Test
public void testEndMissDBConfirm() throws Exception {
    SLAService slas = Services.get().get(SLAService.class);
    EventHandlerService ehs = Services.get().get(EventHandlerService.class);
    JPAService jpaService = Services.get().get(JPAService.class);
    Date date = new Date();
    // CASE 1: positive test WF job
    WorkflowJobBean job1 = addRecordToWfJobTable(WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
    SLARegistrationBean sla = _createSLARegistration(job1.getId(), AppType.WORKFLOW_JOB);
    // half hour back
    sla.setExpectedEnd(new Date(date.getTime() - 1 * 1800 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 2: negative test WF job
    WorkflowJobBean job2 = addRecordToWfJobTable(WorkflowJob.Status.SUCCEEDED, WorkflowInstance.Status.SUCCEEDED);
    job2.setEndTime(new Date(date.getTime() - 1 * 1800 * 1000));
    job2.setStartTime(new Date(date.getTime() - 1 * 2000 * 1000));
    job2.setLastModifiedTime(new Date());
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, job2);
    sla = _createSLARegistration(job2.getId(), AppType.WORKFLOW_JOB);
    // in past but > actual end
    sla.setExpectedEnd(new Date(date.getTime() - 1 * 1500 * 1000));
    // unreasonable to cause MISS
    sla.setExpectedDuration(100);
    slas.addRegistrationEvent(sla);
    slas.runSLAWorker();
    // CASE 3: positive test Coord action
    CoordinatorActionBean action1 = addRecordToCoordActionTable("coord-action-C@1", 1, CoordinatorAction.Status.WAITING, "coord-action-get.xml", 0);
    action1.setExternalId(null);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_RERUN, action1);
    sla = _createSLARegistration(action1.getId(), AppType.COORDINATOR_ACTION);
    // past
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 2000 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 4: positive test coord action
    CoordinatorActionBean action2 = addRecordToCoordActionTable("coord-action-C@2", 1, CoordinatorAction.Status.FAILED, "coord-action-get.xml", 0);
    WorkflowJobBean extWf = new WorkflowJobBean();
    extWf.setId(action2.getExternalId());
    // actual end before expected. but action is failed
    extWf.setEndTime(new Date(System.currentTimeMillis() - 1 * 1800 * 1000));
    extWf.setStartTime(new Date(System.currentTimeMillis() - 1 * 2000 * 1000));
    jpaService.execute(new WorkflowJobInsertJPAExecutor(extWf));
    sla = _createSLARegistration(action2.getId(), AppType.COORDINATOR_ACTION);
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1500 * 1000));
    slas.addRegistrationEvent(sla);
    // CASE 5: negative test coord action
    CoordinatorActionBean action3 = addRecordToCoordActionTable("coord-action-C@3", 1, CoordinatorAction.Status.SUCCEEDED, "coord-action-get.xml", 0);
    extWf = new WorkflowJobBean();
    extWf.setId(action3.getExternalId());
    extWf.setStartTime(new Date(System.currentTimeMillis() - 1 * 2100 * 1000));
    extWf.setEndTime(new Date(System.currentTimeMillis() - 1 * 1800 * 1000));
    jpaService.execute(new WorkflowJobInsertJPAExecutor(extWf));
    sla = _createSLARegistration(action3.getId(), AppType.COORDINATOR_ACTION);
    // cause start_miss
    sla.setExpectedStart(new Date(System.currentTimeMillis() - 1 * 3600 * 1000));
    // in past but > actual end, end_met
    sla.setExpectedEnd(new Date(System.currentTimeMillis() - 1 * 1500 * 1000));
    // cause duration miss
    sla.setExpectedDuration(0);
    slas.addRegistrationEvent(sla);
    slas.runSLAWorker();
    ehs.new EventWorker().run();
    int count = 0;
    for (int ptr = output.indexOf("END - MISS"); ptr < output.length() && ptr > 0; ptr = output.indexOf("END - MISS", ptr + 1)) {
        count++;
    }
    // only 3 out of the 5 are correct end_misses
    assertEquals(3, count);
    assertEventNoDuplicates(output.toString(), job1.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), action1.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), action2.getId() + " Sla END - MISS!!!");
    assertEventNoDuplicates(output.toString(), job2.getId() + " Sla END - MET!!!");
    assertEventNoDuplicates(output.toString(), job2.getId() + " Sla DURATION - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla START - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla DURATION - MISS!!!");
    assertEventNoDuplicates(output.toString(), action3.getId() + " Sla END - MET!!!");
    // negative on MISS after DB check, updated with actual times
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, job2.getId());
    assertEquals(job2.getStartTime(), slaSummary.getActualStart());
    assertEquals(job2.getEndTime(), slaSummary.getActualEnd());
    assertEquals(job2.getEndTime().getTime() - job2.getStartTime().getTime(), slaSummary.getActualDuration());
    assertEquals(job2.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MET, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MET, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
    // removed from memory
    assertNull(slas.getSLACalculator().get(job2.getId()));
    // positives but also updated with actual times immediately after DB check
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action2.getId());
    extWf = jpaService.execute(new WorkflowJobGetJPAExecutor(action2.getExternalId()));
    assertEquals(extWf.getStartTime(), slaSummary.getActualStart());
    assertEquals(extWf.getEndTime(), slaSummary.getActualEnd());
    assertEquals(extWf.getEndTime().getTime() - extWf.getStartTime().getTime(), slaSummary.getActualDuration());
    assertEquals(action2.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
    // removed from memory
    assertNull(slas.getSLACalculator().get(action2.getId()));
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action1.getId());
    assertNull(slaSummary.getActualStart());
    assertNull(slaSummary.getActualEnd());
    assertEquals(action1.getStatusStr(), slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(7, slaSummary.getEventProcessed());
    assertNotNull(slas.getSLACalculator().get(action1.getId()));
    // From waiting to TIMEOUT with wf jobid
    action1.setStatus(CoordinatorAction.Status.TIMEDOUT);
    CoordActionQueryExecutor.getInstance().executeUpdate(CoordActionQuery.UPDATE_COORD_ACTION_RERUN, action1);
    slas.getSLACalculator().addJobStatus(action1.getId(), null, null, null, null);
    slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, action1.getId());
    assertNull(slaSummary.getActualStart());
    assertNotNull(slaSummary.getActualEnd());
    assertEquals("TIMEDOUT", slaSummary.getJobStatus());
    assertEquals(SLAEvent.EventStatus.END_MISS, slaSummary.getEventStatus());
    assertEquals(SLAStatus.MISS, slaSummary.getSLAStatus());
    assertEquals(8, slaSummary.getEventProcessed());
}

Example 7 with WorkflowJobInsertJPAExecutor

use of org.apache.oozie.executor.jpa.WorkflowJobInsertJPAExecutor in project oozie by apache.

the class TestEventGeneration method _createWorkflowJob.

private WorkflowJobBean _createWorkflowJob() throws Exception {
    LiteWorkflowApp app = new LiteWorkflowApp("my-app", "<workflow-app/>", new StartNodeDef(TestControlNodeHandler.class, "one")).addNode(new ActionNodeDef("one", "<java></java>", TestActionNodeHandler.class, "end", "end")).addNode(new EndNodeDef("end", TestControlNodeHandler.class));
    Configuration conf = new Configuration();
    Path appUri = new Path(getAppPath(), "workflow.xml");
    conf.set(OozieClient.APP_PATH, appUri.toString());
    conf.set(OozieClient.LOG_TOKEN, "testToken");
    conf.set(OozieClient.USER_NAME, getTestUser());
    WorkflowJobBean workflow = createWorkflow(app, conf, WorkflowJob.Status.PREP, WorkflowInstance.Status.PREP);
    String executionPath = "/";
    assertNotNull(jpaService);
    WorkflowJobInsertJPAExecutor wfInsertCmd = new WorkflowJobInsertJPAExecutor(workflow);
    jpaService.execute(wfInsertCmd);
    WorkflowActionBean wfAction = addRecordToWfActionTable(workflow.getId(), "one", WorkflowAction.Status.OK, executionPath, true);
    wfAction.setPending();
    wfAction.setSignalValue(WorkflowAction.Status.OK.name());
    WorkflowActionQueryExecutor.getInstance().executeUpdate(WorkflowActionQuery.UPDATE_ACTION, wfAction);
    return workflow;
}

Example 8 with WorkflowJobInsertJPAExecutor

use of org.apache.oozie.executor.jpa.WorkflowJobInsertJPAExecutor in project oozie by apache.

the class SubmitHttpXCommand method execute.

/* (non-Javadoc)
     * @see org.apache.oozie.command.XCommand#execute()
     */
@Override
protected String execute() throws CommandException {
    InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    try {
        XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
        String wfXml = getWorkflowXml(conf);
        LOG.debug("workflow xml created on the server side is :\n");
        LOG.debug(wfXml);
        WorkflowApp app = wps.parseDef(wfXml, conf);
        XConfiguration protoActionConf = wps.createProtoActionConf(conf, false);
        WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
        PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
        // Resolving all variables in the job properties.
        // This ensures the Hadoop Configuration semantics is preserved.
        XConfiguration resolvedVarsConf = new XConfiguration();
        for (Map.Entry<String, String> entry : conf) {
            resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
        }
        conf = resolvedVarsConf;
        WorkflowInstance wfInstance;
        try {
            wfInstance = workflowLib.createInstance(app, conf);
        } catch (WorkflowException e) {
            throw new StoreException(e);
        }
        Configuration conf = wfInstance.getConf();
        WorkflowJobBean workflow = new WorkflowJobBean();
        workflow.setId(wfInstance.getId());
        workflow.setAppName(app.getName());
        workflow.setAppPath(conf.get(OozieClient.APP_PATH));
        workflow.setConf(XmlUtils.prettyPrint(conf).toString());
        workflow.setProtoActionConf(protoActionConf.toXmlString());
        workflow.setCreatedTime(new Date());
        workflow.setLastModifiedTime(new Date());
        workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
        workflow.setStatus(WorkflowJob.Status.PREP);
        workflow.setRun(0);
        workflow.setUser(conf.get(OozieClient.USER_NAME));
        workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
        workflow.setWorkflowInstance(wfInstance);
        workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
        LogUtils.setLogInfo(workflow);
        JPAService jpaService = Services.get().get(JPAService.class);
        if (jpaService != null) {
            jpaService.execute(new WorkflowJobInsertJPAExecutor(workflow));
        } else {
            LOG.error(ErrorCode.E0610);
            return null;
        }
        return workflow.getId();
    } catch (WorkflowException ex) {
        throw new CommandException(ex);
    } catch (Exception ex) {
        throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
    }
}

Example 9 with WorkflowJobInsertJPAExecutor

use of org.apache.oozie.executor.jpa.WorkflowJobInsertJPAExecutor in project oozie by apache.

the class TestPurgeXCommand method addRecordToWfJobTableForNegCase.

protected WorkflowJobBean addRecordToWfJobTableForNegCase(WorkflowJob.Status jobStatus, WorkflowInstance.Status instanceStatus) throws Exception {
    WorkflowApp app = new LiteWorkflowApp("testApp", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    Configuration conf = new Configuration();
    Path appUri = new Path(getAppPath(), "workflow.xml");
    conf.set(OozieClient.APP_PATH, appUri.toString());
    conf.set(OozieClient.LOG_TOKEN, "testToken");
    conf.set(OozieClient.USER_NAME, getTestUser());
    WorkflowJobBean wfBean = createWorkflow(app, conf, jobStatus, instanceStatus);
    // Set start time to 100 days from now
    wfBean.setStartTime(new Date(System.currentTimeMillis() + (long) 100 * 24 * 60 * 60 * 1000));
    // Set end time to 100 days + 2 hours from now
    wfBean.setEndTime(new Date(System.currentTimeMillis() + (long) 100 * 24 * 60 * 60 * 1000 + (long) 2 * 60 * 60 * 1000));
    try {
        JPAService jpaService = Services.get().get(JPAService.class);
        assertNotNull(jpaService);
        WorkflowJobInsertJPAExecutor wfInsertCmd = new WorkflowJobInsertJPAExecutor(wfBean);
        jpaService.execute(wfInsertCmd);
    } catch (JPAExecutorException ce) {
        ce.printStackTrace();
        fail("Unable to insert the test wf job record to table");
        throw ce;
    }
    return wfBean;
}

Example 10 with WorkflowJobInsertJPAExecutor

use of org.apache.oozie.executor.jpa.WorkflowJobInsertJPAExecutor in project oozie by apache.

the class TestWorkflowIdGetForExternalIdJPAExecutor method addRecordToWfJobTable.

@Override
protected WorkflowJobBean addRecordToWfJobTable(WorkflowJob.Status jobStatus, WorkflowInstance.Status instanceStatus) throws Exception {
    WorkflowApp app = new LiteWorkflowApp("testApp", "<workflow-app/>", new StartNodeDef(LiteWorkflowStoreService.LiteControlNodeHandler.class, "end")).addNode(new EndNodeDef("end", LiteWorkflowStoreService.LiteControlNodeHandler.class));
    Configuration conf = new Configuration();
    conf.set(OozieClient.APP_PATH, "testPath");
    conf.set(OozieClient.LOG_TOKEN, "testToken");
    conf.set(OozieClient.USER_NAME, getTestUser());
    WorkflowJobBean wfBean = createWorkflow(app, conf, jobStatus, instanceStatus);
    wfBean.setExternalId("external-id");
    try {
        JPAService jpaService = Services.get().get(JPAService.class);
        assertNotNull(jpaService);
        WorkflowJobInsertJPAExecutor wfInsertCmd = new WorkflowJobInsertJPAExecutor(wfBean);
        jpaService.execute(wfInsertCmd);
    } catch (JPAExecutorException je) {
        je.printStackTrace();
        fail("Unable to insert the test wf job record to table");
        throw je;
    }
    return wfBean;
}