Examples with UpdateEntry org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry used on opensource projects

Example 11 with UpdateEntry

use of org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry in project oozie by apache.

the class ActionStartXCommand method execute.

@Override
protected ActionExecutorContext execute() throws CommandException {
    LOG.debug("STARTED ActionStartXCommand for wf actionId=" + actionId);
    Configuration conf = wfJob.getWorkflowInstance().getConf();
    int maxRetries = 0;
    long retryInterval = 0;
    boolean execSynchronous = false;
    if (!(executor instanceof ControlNodeActionExecutor)) {
        maxRetries = conf.getInt(OozieClient.ACTION_MAX_RETRIES, executor.getMaxRetries());
        retryInterval = conf.getLong(OozieClient.ACTION_RETRY_INTERVAL, executor.getRetryInterval());
    }
    executor.setMaxRetries(maxRetries);
    executor.setRetryInterval(retryInterval);
    try {
        boolean isRetry = false;
        if (wfAction.getStatus() == WorkflowActionBean.Status.START_RETRY || wfAction.getStatus() == WorkflowActionBean.Status.START_MANUAL) {
            isRetry = true;
            prepareForRetry(wfAction);
        }
        boolean isUserRetry = false;
        if (wfAction.getStatus() == WorkflowActionBean.Status.USER_RETRY) {
            isUserRetry = true;
            prepareForRetry(wfAction);
        }
        context = getContext(isRetry, isUserRetry);
        boolean caught = false;
        try {
            if (!(executor instanceof ControlNodeActionExecutor)) {
                String tmpActionConf = XmlUtils.removeComments(wfAction.getConf());
                String actionConf = context.getELEvaluator().evaluate(tmpActionConf, String.class);
                wfAction.setConf(actionConf);
                LOG.debug("Start, name [{0}] type [{1}] configuration{E}{E}{2}{E}", wfAction.getName(), wfAction.getType(), actionConf);
            }
        } catch (ELEvaluationException ex) {
            caught = true;
            throw new ActionExecutorException(ActionExecutorException.ErrorType.TRANSIENT, EL_EVAL_ERROR, ex.getMessage(), ex);
        } catch (ELException ex) {
            caught = true;
            context.setErrorInfo(EL_ERROR, ex.getMessage());
            LOG.warn("ELException in ActionStartXCommand ", ex.getMessage(), ex);
            handleError(context, wfJob, wfAction);
        } catch (org.jdom.JDOMException je) {
            caught = true;
            context.setErrorInfo("ParsingError", je.getMessage());
            LOG.warn("JDOMException in ActionStartXCommand ", je.getMessage(), je);
            handleError(context, wfJob, wfAction);
        } catch (Exception ex) {
            caught = true;
            context.setErrorInfo(EL_ERROR, ex.getMessage());
            LOG.warn("Exception in ActionStartXCommand ", ex.getMessage(), ex);
            handleError(context, wfJob, wfAction);
        }
        if (!caught) {
            wfAction.setErrorInfo(null, null);
            incrActionCounter(wfAction.getType(), 1);
            LOG.info("Start action [{0}] with user-retry state : userRetryCount [{1}], userRetryMax [{2}], userRetryInterval" + " [{3}]", wfAction.getId(), wfAction.getUserRetryCount(), wfAction.getUserRetryMax(), wfAction.getUserRetryInterval());
            Instrumentation.Cron cron = new Instrumentation.Cron();
            cron.start();
            // do not override starttime for retries
            if (wfAction.getStartTime() == null) {
                context.setStartTime();
            }
            context.setVar(JobUtils.getRetryKey(wfAction, JsonTags.WORKFLOW_ACTION_START_TIME), String.valueOf(new Date().getTime()));
            executor.start(context, wfAction);
            cron.stop();
            FaultInjection.activate("org.apache.oozie.command.SkipCommitFaultInjection");
            addActionCron(wfAction.getType(), cron);
            wfAction.setRetries(0);
            if (wfAction.isExecutionComplete()) {
                if (!context.isExecuted()) {
                    LOG.warn(XLog.OPS, "Action Completed, ActionExecutor [{0}] must call setExecutionData()", executor.getType());
                    wfAction.setErrorInfo(EXEC_DATA_MISSING, "Execution Complete, but Execution Data Missing from Action");
                    failJob(context);
                } else {
                    wfAction.setPending();
                    if (!(executor instanceof ControlNodeActionExecutor)) {
                        queue(new ActionEndXCommand(wfAction.getId(), wfAction.getType()));
                    } else {
                        execSynchronous = true;
                    }
                }
            } else {
                if (!context.isStarted()) {
                    LOG.warn(XLog.OPS, "Action Started, ActionExecutor [{0}] must call setStartData()", executor.getType());
                    wfAction.setErrorInfo(START_DATA_MISSING, "Execution Started, but Start Data Missing from Action");
                    failJob(context);
                } else {
                    queue(new WorkflowNotificationXCommand(wfJob, wfAction));
                }
            }
            LOG.info(XLog.STD, "[***" + wfAction.getId() + "***]" + "Action status=" + wfAction.getStatusStr());
            updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_START, wfAction));
            updateJobLastModified();
            // Add SLA status event (STARTED) for WF_ACTION
            SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(wfAction.getSlaXml(), wfAction.getId(), Status.STARTED, SlaAppType.WORKFLOW_ACTION);
            if (slaEvent != null) {
                insertList.add(slaEvent);
            }
            LOG.info(XLog.STD, "[***" + wfAction.getId() + "***]" + "Action updated in DB!");
        }
    } catch (ActionExecutorException ex) {
        LOG.warn("Error starting action [{0}]. ErrorType [{1}], ErrorCode [{2}], Message [{3}]", wfAction.getName(), ex.getErrorType(), ex.getErrorCode(), ex.getMessage(), ex);
        wfAction.setErrorInfo(ex.getErrorCode(), ex.getMessage());
        switch(ex.getErrorType()) {
            case TRANSIENT:
                if (!handleTransient(context, executor, WorkflowAction.Status.START_RETRY)) {
                    handleNonTransient(context, executor, WorkflowAction.Status.START_MANUAL);
                    wfAction.setPendingAge(new Date());
                    wfAction.setRetries(0);
                    wfAction.setStartTime(null);
                }
                break;
            case NON_TRANSIENT:
                handleNonTransient(context, executor, WorkflowAction.Status.START_MANUAL);
                break;
            case ERROR:
                handleError(context, executor, WorkflowAction.Status.ERROR.toString(), true, WorkflowAction.Status.DONE);
                break;
            case FAILED:
                try {
                    failJob(context);
                    endWF();
                    SLAEventBean slaEvent1 = SLADbXOperations.createStatusEvent(wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION);
                    if (slaEvent1 != null) {
                        insertList.add(slaEvent1);
                    }
                } catch (XException x) {
                    LOG.warn("ActionStartXCommand - case:FAILED ", x.getMessage());
                }
                break;
        }
        updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_START, wfAction));
        updateJobLastModified();
    } finally {
        try {
            BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null);
            if (!(executor instanceof ControlNodeActionExecutor) && EventHandlerService.isEnabled()) {
                generateEvent(wfAction, wfJob.getUser());
            }
            if (execSynchronous) {
                // Changing to synchronous call from asynchronous queuing to prevent
                // undue delay from ::start:: to action due to queuing
                callActionEnd();
            }
        } catch (JPAExecutorException e) {
            throw new CommandException(e);
        }
    }
    LOG.debug("ENDED ActionStartXCommand for wf actionId=" + actionId + ", jobId=" + jobId);
    return null;
}

Example 12 with UpdateEntry

use of org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry in project oozie by apache.

the class TransitionXCommand method generateEvents.

/**
 * This will be used to generate Job Notification events on status changes
 *
 * @param coordJob
 * @param startTime
 * @throws CommandException
 */
public void generateEvents(CoordinatorJobBean coordJob, Date startTime) throws CommandException {
    for (UpdateEntry entry : updateList) {
        JsonBean actionBean = entry.getBean();
        if (actionBean instanceof CoordinatorActionBean) {
            CoordinatorActionBean caBean = (CoordinatorActionBean) actionBean;
            caBean.setJobId(coordJob.getId());
            CoordinatorXCommand.generateEvent(caBean, coordJob.getUser(), coordJob.getAppName(), startTime);
        }
    // TODO generate Coord Job event
    }
}

Example 13 with UpdateEntry

use of org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry in project oozie by apache.

the class SignalXCommand method startForkedActions.

public void startForkedActions(List<WorkflowActionBean> workflowActionBeanListForForked) throws CommandException {
    List<CallableWrapper<ActionExecutorContext>> tasks = new ArrayList<CallableWrapper<ActionExecutorContext>>();
    List<UpdateEntry> updateList = new ArrayList<UpdateEntry>();
    List<JsonBean> insertList = new ArrayList<JsonBean>();
    boolean endWorkflow = false;
    boolean submitJobByQueuing = false;
    for (WorkflowActionBean workflowActionBean : workflowActionBeanListForForked) {
        LOG.debug("Starting forked actions parallely : " + workflowActionBean.getId());
        tasks.add(Services.get().get(CallableQueueService.class).new CallableWrapper<ActionExecutorContext>(new ForkedActionStartXCommand(wfJob, workflowActionBean.getId(), workflowActionBean.getType()), 0));
    }
    try {
        List<Future<ActionExecutorContext>> futures = Services.get().get(CallableQueueService.class).invokeAll(tasks);
        for (Future<ActionExecutorContext> result : futures) {
            if (result == null) {
                submitJobByQueuing = true;
                continue;
            }
            ActionExecutorContext context = result.get();
            Map<String, String> contextVariableMap = ((ForkedActionExecutorContext) context).getContextMap();
            LOG.debug("contextVariableMap size of action " + context.getAction().getId() + " is " + contextVariableMap.size());
            for (String key : contextVariableMap.keySet()) {
                context.setVarToWorkflow(key, contextVariableMap.get(key));
            }
            if (context.getJobStatus() != null && context.getJobStatus().equals(Job.Status.FAILED)) {
                LOG.warn("Action has failed, failing job" + context.getAction().getId());
                new ActionStartXCommand(context.getAction().getId(), null).failJob(context);
                updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_START, (WorkflowActionBean) context.getAction()));
                if (context.isShouldEndWF()) {
                    endWorkflow = true;
                }
            }
            if (context.getJobStatus() != null && context.getJobStatus().equals(Job.Status.SUSPENDED)) {
                LOG.warn("Action has failed, failing job" + context.getAction().getId());
                new ActionStartXCommand(context.getAction().getId(), null).handleNonTransient(context, null, WorkflowAction.Status.START_MANUAL);
                updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_START, (WorkflowActionBean) context.getAction()));
                if (context.isShouldEndWF()) {
                    endWorkflow = true;
                }
            }
        }
        if (endWorkflow) {
            endWF(insertList);
        }
    } catch (Exception e) {
        LOG.error("Error running forked jobs parallely", e);
        startForkedActionsByQueuing(workflowActionBeanListForForked);
        submitJobByQueuing = false;
    }
    if (submitJobByQueuing && !endWorkflow) {
        LOG.error("There is error in running forked jobs parallely");
        startForkedActionsByQueuing(workflowActionBeanListForForked);
    }
    wfJob.setLastModifiedTime(new Date());
    updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MODIFIED, wfJob));
    try {
        BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null);
    } catch (JPAExecutorException e) {
        throw new CommandException(e);
    }
    LOG.debug("forked actions submitted parallely");
}

Example 14 with UpdateEntry

use of org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry in project oozie by apache.

the class TestSLACalculationJPAExecutor method testInsertUpdate.

/**
 * Test insert + update
 *
 * @throws Exception
 */
@Test
public void testInsertUpdate() throws Exception {
    JPAService jpaService = Services.get().get(JPAService.class);
    assertNotNull(jpaService);
    String wfId = "workflow-1";
    // initial insert
    cal.setTime(new Date());
    cal.add(Calendar.DAY_OF_MONTH, -2);
    Date expStart = cal.getTime();
    cal.add(Calendar.DAY_OF_MONTH, -1);
    Date expEnd = cal.getTime();
    Date actStart = new Date();
    SLASummaryBean bean2 = _createSLASummaryBean(wfId, "RUNNING", EventStatus.START_MISS, expStart, expEnd, 1000, actStart, null, 2000, (byte) 0, actStart);
    List<JsonBean> insertList = new ArrayList<JsonBean>();
    insertList.add(bean2);
    BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, null, null);
    // update existing record
    Date newDate = new Date();
    bean2 = _createSLASummaryBean(wfId, "RUNNING", EventStatus.DURATION_MISS, expStart, expEnd, 1000, actStart, newDate, 2000, (byte) 1, newDate);
    bean2.setAppType(AppType.WORKFLOW_ACTION);
    List<UpdateEntry> updateList = new ArrayList<UpdateEntry>();
    SLASummaryQueryExecutor.getInstance().executeUpdate(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL, bean2);
    SLASummaryBean sBean = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, wfId);
    // check updated + original fields
    assertEquals(wfId, sBean.getId());
    assertEquals(EventStatus.DURATION_MISS, sBean.getEventStatus());
    assertEquals(expStart, sBean.getExpectedStart());
    assertEquals(expEnd, sBean.getExpectedEnd());
    assertEquals(1000, sBean.getExpectedDuration());
    assertEquals(actStart, sBean.getActualStart());
    assertEquals(newDate, sBean.getActualEnd());
    assertEquals(2000, sBean.getActualDuration());
    assertEquals(newDate, sBean.getLastModifiedTime());
}

Example 15 with UpdateEntry

use of org.apache.oozie.executor.jpa.BatchQueryExecutor.UpdateEntry in project oozie by apache.

the class TestSLACalculatorMemory method testLoadOnRestart.

@Test
public void testLoadOnRestart() throws Exception {
    SLACalculatorMemory slaCalcMemory = new SLACalculatorMemory();
    slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
    SLARegistrationBean slaRegBean1 = _createSLARegistration("job-1-W", AppType.WORKFLOW_JOB);
    String jobId1 = slaRegBean1.getId();
    SLARegistrationBean slaRegBean2 = _createSLARegistration("job-2-W", AppType.WORKFLOW_JOB);
    String jobId2 = slaRegBean2.getId();
    SLARegistrationBean slaRegBean3 = _createSLARegistration("job-3-W", AppType.WORKFLOW_JOB);
    String jobId3 = slaRegBean3.getId();
    List<String> idList = new ArrayList<String>();
    idList.add(slaRegBean1.getId());
    idList.add(slaRegBean2.getId());
    idList.add(slaRegBean3.getId());
    createWorkflow(idList);
    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-mm-dd");
    slaRegBean1.setAppName("app-name");
    slaRegBean1.setExpectedDuration(123);
    slaRegBean1.setExpectedEnd(sdf.parse("2012-02-07"));
    slaRegBean1.setExpectedStart(sdf.parse("2011-02-07"));
    slaRegBean1.setNominalTime(sdf.parse("2012-01-06"));
    slaRegBean1.setUser("user");
    slaRegBean1.setParentId("parentId");
    slaRegBean1.setUpstreamApps("upstreamApps");
    slaRegBean1.setNotificationMsg("notificationMsg");
    slaRegBean1.setAlertContact("a@abc.com");
    slaRegBean1.setAlertEvents("MISS");
    slaRegBean1.setJobData("jobData");
    // 1 hour back
    Date startTime = new Date(System.currentTimeMillis() - 1 * 1 * 3600 * 1000);
    // 1 hour back
    Date endTime = new Date(System.currentTimeMillis() + 2 * 1 * 3600 * 1000);
    slaRegBean3.setExpectedStart(startTime);
    slaRegBean3.setExpectedEnd(endTime);
    slaCalcMemory.addRegistration(jobId1, slaRegBean1);
    slaCalcMemory.addRegistration(jobId2, slaRegBean2);
    slaCalcMemory.addRegistration(jobId3, slaRegBean3);
    slaCalcMemory.updateAllSlaStatus();
    SLACalcStatus calc1 = slaCalcMemory.get(jobId1);
    SLACalcStatus calc2 = slaCalcMemory.get(jobId2);
    SLACalcStatus calc3 = slaCalcMemory.get(jobId3);
    calc1.setEventStatus(SLAEvent.EventStatus.END_MISS);
    calc1.setSLAStatus(SLAEvent.SLAStatus.MISS);
    calc1.setJobStatus(WorkflowJob.Status.FAILED.toString());
    // set last modified time 5 days back
    Date lastModifiedTime = new Date(System.currentTimeMillis() - 5 * 24 * 60 * 60 * 1000);
    calc1.setLastModifiedTime(lastModifiedTime);
    List<UpdateEntry> updateList = new ArrayList<UpdateEntry>();
    WorkflowJobBean wf1 = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId1);
    wf1.setId(jobId1);
    wf1.setStatus(WorkflowJob.Status.SUCCEEDED);
    wf1.setStartTime(sdf.parse("2011-03-09"));
    wf1.setEndTime(sdf.parse("2011-03-10"));
    wf1.setLastModifiedTime(new Date());
    WorkflowJobBean wf2 = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId2);
    wf2.setId(jobId2);
    wf2.setStatus(WorkflowJob.Status.RUNNING);
    wf2.setStartTime(sdf.parse("2011-03-09"));
    wf2.setEndTime(null);
    wf2.setLastModifiedTime(new Date());
    WorkflowJobBean wf3 = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId3);
    wf3.setId(jobId3);
    wf3.setStatus(WorkflowJob.Status.RUNNING);
    wf3.setStartTime(startTime);
    wf3.setEndTime(null);
    wf3.setLastModifiedTime(new Date());
    updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW, wf1));
    updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW, wf2));
    updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW, wf3));
    updateList.add(new UpdateEntry<SLASummaryQuery>(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL, new SLASummaryBean(calc2)));
    updateList.add(new UpdateEntry<SLASummaryQuery>(SLASummaryQuery.UPDATE_SLA_SUMMARY_ALL, new SLASummaryBean(calc3)));
    BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(null, updateList, null);
    slaCalcMemory = new SLACalculatorMemory();
    slaCalcMemory.init(Services.get().get(ConfigurationService.class).getConf());
    slaCalcMemory.updateAllSlaStatus();
    SLACalcStatus calc = new SLACalcStatus(SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId1), SLARegistrationQueryExecutor.getInstance().get(SLARegQuery.GET_SLA_REG_ON_RESTART, jobId1));
    assertEquals("job-1-W", calc.getId());
    assertEquals(AppType.WORKFLOW_JOB, calc.getAppType());
    assertEquals("app-name", calc.getAppName());
    assertEquals(123, calc.getExpectedDuration());
    assertEquals(sdf.parse("2012-02-07"), calc.getExpectedEnd());
    assertEquals(sdf.parse("2011-02-07"), calc.getExpectedStart());
    assertEquals(sdf.parse("2012-01-06"), calc.getNominalTime());
    assertEquals("user", calc.getUser());
    assertEquals("parentId", calc.getParentId());
    assertEquals("upstreamApps", calc.getUpstreamApps());
    assertEquals("notificationMsg", calc.getNotificationMsg());
    assertEquals("a@abc.com", calc.getAlertContact());
    assertEquals("MISS", calc.getAlertEvents());
    assertEquals("jobData", calc.getJobData());
    assertEquals(sdf.parse("2011-03-09"), calc.getActualStart());
    assertEquals(sdf.parse("2011-03-10"), calc.getActualEnd());
    assertEquals(SLAEvent.EventStatus.END_MISS, calc1.getEventStatus());
    assertEquals(SLAEvent.SLAStatus.MISS, calc1.getSLAStatus());
    assertEquals(WorkflowJob.Status.FAILED.toString(), calc1.getJobStatus());
    assertEquals(lastModifiedTime, calc1.getLastModifiedTime());
    calc2 = new SLACalcStatus(SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId2), SLARegistrationQueryExecutor.getInstance().get(SLARegQuery.GET_SLA_REG_ON_RESTART, jobId2));
    assertEquals(8, calc.getEventProcessed());
    assertEquals(7, calc2.getEventProcessed());
    // jobId2 should be in history set as eventprocessed is 7 (111)
    // job3 will be in slamap
    // 1 out of 3 jobs in map
    assertEquals(1, slaCalcMemory.size());
    WorkflowJobBean wf = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW, jobId3);
    wf.setId(jobId3);
    wf.setStatus(WorkflowJob.Status.SUCCEEDED);
    wf.setEndTime(endTime);
    wf.setStartTime(startTime);
    WorkflowJobQueryExecutor.getInstance().executeUpdate(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, wf);
    slaCalcMemory.addJobStatus(jobId3, WorkflowJob.Status.SUCCEEDED.toString(), EventStatus.SUCCESS, startTime, endTime);
    SLASummaryBean slaSummary = SLASummaryQueryExecutor.getInstance().get(SLASummaryQuery.GET_SLA_SUMMARY, jobId3);
    assertEquals(8, slaSummary.getEventProcessed());
    assertEquals(startTime, slaSummary.getActualStart());
    assertEquals(endTime, slaSummary.getActualEnd());
    assertEquals(WorkflowJob.Status.SUCCEEDED.toString(), slaSummary.getJobStatus());
}