Examples with ActionExecutorContext org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext used on opensource projects

Example 1 with ActionExecutorContext

use of org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext in project oozie by apache.

the class TestActionCheckXCommand method testActionCheckTransientDuringMRAction.

public void testActionCheckTransientDuringMRAction() throws Exception {
    // When using YARN, skip this test because it relies on shutting down the job tracker, which isn't used in YARN
    if (createJobConf().get("yarn.resourcemanager.address") != null) {
        return;
    }
    services.destroy();
    // Make the max number of retries lower so the test won't take as long
    final int maxRetries = 2;
    setSystemProperty("oozie.action.retries.max", Integer.toString(maxRetries));
    services = new Services();
    // Disable ActionCheckerService so it doesn't interfere by triggering any extra ActionCheckXCommands
    setClassesToBeExcluded(services.getConf(), new String[] { "org.apache.oozie.service.ActionCheckerService" });
    services.init();
    final JPAService jpaService = Services.get().get(JPAService.class);
    WorkflowJobBean job0 = this.addRecordToWfJobTable(WorkflowJob.Status.RUNNING, WorkflowInstance.Status.RUNNING);
    final String jobId = job0.getId();
    WorkflowActionBean action0 = this.addRecordToWfActionTable(jobId, "1", WorkflowAction.Status.PREP);
    final String actionId = action0.getId();
    final WorkflowActionGetJPAExecutor wfActionGetCmd = new WorkflowActionGetJPAExecutor(actionId);
    new ActionStartXCommand(actionId, "map-reduce").call();
    final WorkflowActionBean action1 = jpaService.execute(wfActionGetCmd);
    String originalLauncherId = action1.getExternalId();
    ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(job0, action1, false, false);
    MapReduceActionExecutor actionExecutor = new MapReduceActionExecutor();
    Configuration conf = actionExecutor.createBaseHadoopConf(context, XmlUtils.parseXml(action1.getConf()));
    String user = conf.get("user.name");
    JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, conf);
    final RunningJob launcherJob = jobClient.getJob(JobID.forName(originalLauncherId));
    waitFor(120 * 1000, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            return launcherJob.isComplete();
        }
    });
    assertTrue(launcherJob.isSuccessful());
    Map<String, String> actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherHelper.hasIdSwap(actionData));
    new ActionCheckXCommand(action1.getId()).call();
    WorkflowActionBean action2 = jpaService.execute(wfActionGetCmd);
    String originalMapperId = action2.getExternalChildIDs();
    assertFalse(originalLauncherId.equals(originalMapperId));
    // At this point, the launcher job has finished and the map-reduce action has started (but not finished)
    // Now, shutdown the job tracker to pretend it has gone down during the map-reduce job
    executeWhileJobTrackerIsShutdown(new ShutdownJobTrackerExecutable() {

        @Override
        public void execute() throws Exception {
            assertEquals(0, action1.getRetries());
            new ActionCheckXCommand(actionId).call();
            waitFor(30 * 1000, new Predicate() {

                @Override
                public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() > 0);
                }
            });
            waitFor(180 * 1000, new Predicate() {

                @Override
                public boolean evaluate() throws Exception {
                    WorkflowActionBean action1a = jpaService.execute(wfActionGetCmd);
                    return (action1a.getRetries() == 0);
                }
            });
            WorkflowActionBean action1b = jpaService.execute(wfActionGetCmd);
            assertEquals(0, action1b.getRetries());
            assertEquals("START_MANUAL", action1b.getStatusStr());
            WorkflowJobBean job1 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
            assertEquals("SUSPENDED", job1.getStatusStr());
        // At this point, the action has gotten a transient error, even after maxRetries tries so the workflow has been
        // SUSPENDED
        }
    });
    // Now, lets bring the job tracker back up and resume the workflow (which will restart the current action)
    // It should now continue and finish with SUCCEEDED
    new ResumeXCommand(jobId).call();
    WorkflowJobBean job2 = jpaService.execute(new WorkflowJobGetJPAExecutor(jobId));
    assertEquals("RUNNING", job2.getStatusStr());
    sleep(500);
    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action3 = jpaService.execute(wfActionGetCmd);
    String launcherId = action3.getExternalId();
    assertFalse(originalLauncherId.equals(launcherId));
    final RunningJob launcherJob2 = jobClient.getJob(JobID.forName(launcherId));
    waitFor(120 * 1000, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            return launcherJob2.isComplete();
        }
    });
    assertTrue(launcherJob2.isSuccessful());
    actionData = LauncherHelper.getActionData(getFileSystem(), context.getActionDir(), conf);
    assertTrue(LauncherHelper.hasIdSwap(actionData));
    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action4 = jpaService.execute(wfActionGetCmd);
    String mapperId = action4.getExternalChildIDs();
    assertFalse(originalMapperId.equals(mapperId));
    final RunningJob mrJob = jobClient.getJob(JobID.forName(mapperId));
    waitFor(120 * 1000, new Predicate() {

        @Override
        public boolean evaluate() throws Exception {
            return mrJob.isComplete();
        }
    });
    assertTrue(mrJob.isSuccessful());
    new ActionCheckXCommand(actionId).call();
    WorkflowActionBean action5 = jpaService.execute(wfActionGetCmd);
    assertEquals("SUCCEEDED", action5.getExternalStatus());
}

Example 2 with ActionExecutorContext

use of org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext in project oozie by apache.

the class SignalXCommand method execute.

@Override
protected Void execute() throws CommandException {
    LOG.debug("STARTED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
    WorkflowInstance workflowInstance = wfJob.getWorkflowInstance();
    workflowInstance.setTransientVar(WorkflowStoreService.WORKFLOW_BEAN, wfJob);
    WorkflowJob.Status prevStatus = wfJob.getStatus();
    boolean completed = false, skipAction = false;
    WorkflowActionBean syncAction = null;
    List<WorkflowActionBean> workflowActionBeanListForForked = new ArrayList<WorkflowActionBean>();
    if (wfAction == null) {
        if (wfJob.getStatus() == WorkflowJob.Status.PREP) {
            try {
                completed = workflowInstance.start();
            } catch (WorkflowException e) {
                throw new CommandException(e);
            }
            wfJob.setStatus(WorkflowJob.Status.RUNNING);
            wfJob.setStartTime(new Date());
            wfJob.setWorkflowInstance(workflowInstance);
            generateEvent = true;
            // 1. Add SLA status event for WF-JOB with status STARTED
            SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(wfJob.getSlaXml(), jobId, Status.STARTED, SlaAppType.WORKFLOW_JOB);
            if (slaEvent != null) {
                insertList.add(slaEvent);
            }
            // 2. Add SLA registration events for all WF_ACTIONS
            createSLARegistrationForAllActions(workflowInstance.getApp().getDefinition(), wfJob.getUser(), wfJob.getGroup(), wfJob.getConf());
            queue(new WorkflowNotificationXCommand(wfJob));
        } else {
            throw new CommandException(ErrorCode.E0801, wfJob.getId());
        }
    } else {
        WorkflowInstance.Status initialStatus = workflowInstance.getStatus();
        String skipVar = workflowInstance.getVar(wfAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunXCommand.TO_SKIP);
        if (skipVar != null) {
            skipAction = skipVar.equals("true");
        }
        try {
            completed = workflowInstance.signal(wfAction.getExecutionPath(), wfAction.getSignalValue());
        } catch (WorkflowException e) {
            LOG.error("Workflow action failed : " + e.getMessage(), e);
            wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString()));
            completed = true;
        }
        wfJob.setWorkflowInstance(workflowInstance);
        wfAction.resetPending();
        if (!skipAction) {
            wfAction.setTransition(workflowInstance.getTransition(wfAction.getName()));
            queue(new WorkflowNotificationXCommand(wfJob, wfAction));
        }
        updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS, wfAction));
        WorkflowInstance.Status endStatus = workflowInstance.getStatus();
        if (endStatus != initialStatus) {
            generateEvent = true;
        }
    }
    if (completed) {
        try {
            for (String actionToKillId : WorkflowStoreService.getActionsToKill(workflowInstance)) {
                WorkflowActionBean actionToKill;
                actionToKill = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION_ID_TYPE_LASTCHECK, actionToKillId);
                actionToKill.setPending();
                actionToKill.setStatus(WorkflowActionBean.Status.KILLED);
                updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, actionToKill));
                queue(new ActionKillXCommand(actionToKill.getId(), actionToKill.getType()));
            }
            for (String actionToFailId : WorkflowStoreService.getActionsToFail(workflowInstance)) {
                WorkflowActionBean actionToFail = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION_FAIL, actionToFailId);
                actionToFail.resetPending();
                actionToFail.setStatus(WorkflowActionBean.Status.FAILED);
                if (wfJobErrorCode != null) {
                    wfJobErrorCode = actionToFail.getErrorCode();
                    wfJobErrorMsg = actionToFail.getErrorMessage();
                }
                queue(new WorkflowNotificationXCommand(wfJob, actionToFail));
                SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION);
                if (slaEvent != null) {
                    insertList.add(slaEvent);
                }
                updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, actionToFail));
            }
        } catch (JPAExecutorException je) {
            throw new CommandException(je);
        }
        wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString()));
        wfJob.setEndTime(new Date());
        wfJob.setWorkflowInstance(workflowInstance);
        Status slaStatus = Status.SUCCEEDED;
        switch(wfJob.getStatus()) {
            case SUCCEEDED:
                slaStatus = Status.SUCCEEDED;
                break;
            case KILLED:
                slaStatus = Status.KILLED;
                break;
            case FAILED:
                slaStatus = Status.FAILED;
                break;
            default:
                // TODO SUSPENDED
                break;
        }
        SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(wfJob.getSlaXml(), jobId, slaStatus, SlaAppType.WORKFLOW_JOB);
        if (slaEvent != null) {
            insertList.add(slaEvent);
        }
        queue(new WorkflowNotificationXCommand(wfJob));
        if (wfJob.getStatus() == WorkflowJob.Status.SUCCEEDED) {
            InstrumentUtils.incrJobCounter(INSTR_SUCCEEDED_JOBS_COUNTER_NAME, 1, getInstrumentation());
        }
        // output message for Kill node
        if (wfAction != null) {
            // wfAction could be a no-op job
            NodeDef nodeDef = workflowInstance.getNodeDef(wfAction.getExecutionPath());
            if (nodeDef != null && nodeDef instanceof KillNodeDef) {
                boolean isRetry = false;
                boolean isUserRetry = false;
                ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(wfJob, wfAction, isRetry, isUserRetry);
                InstrumentUtils.incrJobCounter(INSTR_KILLED_JOBS_COUNTER_NAME, 1, getInstrumentation());
                try {
                    String tmpNodeConf = nodeDef.getConf();
                    String message = context.getELEvaluator().evaluate(tmpNodeConf, String.class);
                    LOG.debug("Try to resolve KillNode message for jobid [{0}], actionId [{1}], before resolve [{2}], " + "after resolve [{3}]", jobId, actionId, tmpNodeConf, message);
                    if (wfAction.getErrorCode() != null) {
                        wfAction.setErrorInfo(wfAction.getErrorCode(), message);
                    } else {
                        wfAction.setErrorInfo(ErrorCode.E0729.toString(), message);
                    }
                } catch (Exception ex) {
                    LOG.warn("Exception in SignalXCommand when processing Kill node message: {0}", ex.getMessage(), ex);
                    wfAction.setErrorInfo(ErrorCode.E0756.toString(), ErrorCode.E0756.format(ex.getMessage()));
                    wfAction.setStatus(WorkflowAction.Status.ERROR);
                }
                updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS_ERROR, wfAction));
            }
        }
    } else {
        for (WorkflowActionBean newAction : WorkflowStoreService.getActionsToStart(workflowInstance)) {
            boolean isOldWFAction = false;
            // old action. To avoid twice entry for same action, Checking in Db if the workflow action already exist.
            if (SubWorkflowActionExecutor.ACTION_TYPE.equals(newAction.getType())) {
                try {
                    WorkflowActionBean oldAction = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION_CHECK, newAction.getId());
                    newAction.setExternalId(oldAction.getExternalId());
                    newAction.setCreatedTime(oldAction.getCreatedTime());
                    isOldWFAction = true;
                } catch (JPAExecutorException e) {
                    if (e.getErrorCode() != ErrorCode.E0605) {
                        throw new CommandException(e);
                    }
                }
            }
            String skipVar = workflowInstance.getVar(newAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunXCommand.TO_SKIP);
            boolean skipNewAction = false, suspendNewAction = false;
            if (skipVar != null) {
                skipNewAction = skipVar.equals("true");
            }
            if (skipNewAction) {
                WorkflowActionBean oldAction = new WorkflowActionBean();
                oldAction.setId(newAction.getId());
                oldAction.setPending();
                oldAction.setExecutionPath(newAction.getExecutionPath());
                updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_PENDING, oldAction));
                queue(new SignalXCommand(jobId, oldAction.getId()));
            } else {
                if (!skipAction) {
                    try {
                        // Make sure that transition node for a forked action
                        // is inserted only once
                        WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION_ID_TYPE_LASTCHECK, newAction.getId());
                        continue;
                    } catch (JPAExecutorException jee) {
                    }
                }
                suspendNewAction = checkForSuspendNode(newAction);
                newAction.setPending();
                String actionSlaXml = getActionSLAXml(newAction.getName(), workflowInstance.getApp().getDefinition(), wfJob.getConf());
                newAction.setSlaXml(actionSlaXml);
                if (!isOldWFAction) {
                    newAction.setCreatedTime(new Date());
                    insertList.add(newAction);
                } else {
                    updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_START, newAction));
                }
                LOG.debug("SignalXCommand: Name: " + newAction.getName() + ", Id: " + newAction.getId() + ", Authcode:" + newAction.getCred());
                if (wfAction != null) {
                    // null during wf job submit
                    ActionService as = Services.get().get(ActionService.class);
                    ActionExecutor current = as.getExecutor(wfAction.getType());
                    LOG.trace("Current Action Type:" + current.getClass());
                    if (!suspendNewAction) {
                        if (current instanceof StartActionExecutor) {
                            // Excluding :start: here from executing first action synchronously since it
                            // blocks the consumer thread till the action is submitted to Hadoop,
                            // in turn reducing the number of new submissions the threads can accept.
                            // Would also be susceptible to longer delays in case Hadoop cluster is busy.
                            queue(new ActionStartXCommand(newAction.getId(), newAction.getType()));
                        } else if (current instanceof ForkActionExecutor) {
                            if (ConfigurationService.getBoolean(SignalXCommand.FORK_PARALLEL_JOBSUBMISSION)) {
                                workflowActionBeanListForForked.add(newAction);
                            } else {
                                queue(new ActionStartXCommand(newAction.getId(), newAction.getType()));
                            }
                        } else {
                            syncAction = newAction;
                        }
                    } else {
                        // will be ignored.
                        if (ConfigurationService.getBoolean(SignalXCommand.FORK_PARALLEL_JOBSUBMISSION)) {
                            workflowActionBeanListForForked.add(newAction);
                        }
                    }
                } else {
                    // first action after wf submit should always be sync
                    syncAction = newAction;
                }
            }
        }
    }
    try {
        wfJob.setLastModifiedTime(new Date());
        updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, wfJob));
        // call JPAExecutor to do the bulk writes
        BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null);
        if (prevStatus != wfJob.getStatus()) {
            LOG.debug("Updated the workflow status to " + wfJob.getId() + "  status =" + wfJob.getStatusStr());
        }
        if (generateEvent && EventHandlerService.isEnabled()) {
            generateEvent(wfJob, wfJobErrorCode, wfJobErrorMsg);
        }
    } catch (JPAExecutorException je) {
        throw new CommandException(je);
    }
    // undue delay from between end of previous and start of next action
    if (wfJob.getStatus() != WorkflowJob.Status.RUNNING && wfJob.getStatus() != WorkflowJob.Status.SUSPENDED) {
        // only for asynchronous actions, parent coord action's external id will
        // persisted and following update will succeed.
        updateParentIfNecessary(wfJob);
        // To delete the WF temp dir
        new WfEndXCommand(wfJob).call();
    } else if (syncAction != null) {
        new ActionStartXCommand(wfJob, syncAction.getId(), syncAction.getType()).call();
    } else if (!workflowActionBeanListForForked.isEmpty() && !checkForSuspendNode(workflowActionBeanListForForked)) {
        startForkedActions(workflowActionBeanListForForked);
    }
    LOG.debug("ENDED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
    return null;
}

Example 3 with ActionExecutorContext

use of org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext in project oozie by apache.

the class SubmitXCommand method execute.

@Override
protected String execute() throws CommandException {
    InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    try {
        XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
        String user = conf.get(OozieClient.USER_NAME);
        URI uri = new URI(conf.get(OozieClient.APP_PATH));
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        Configuration fsConf = has.createConfiguration(uri.getAuthority());
        FileSystem fs = has.createFileSystem(user, uri, fsConf);
        Path configDefault = null;
        Configuration defaultConf = null;
        // app path could be a directory
        Path path = new Path(uri.getPath());
        if (!fs.isFile(path)) {
            configDefault = new Path(path, CONFIG_DEFAULT);
        } else {
            configDefault = new Path(path.getParent(), CONFIG_DEFAULT);
        }
        if (fs.exists(configDefault)) {
            try {
                defaultConf = new XConfiguration(fs.open(configDefault));
                PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
                XConfiguration.injectDefaults(defaultConf, conf);
            } catch (IOException ex) {
                throw new IOException("default configuration file, " + ex.getMessage(), ex);
            }
        }
        if (defaultConf != null) {
            defaultConf = resolveDefaultConfVariables(defaultConf);
        }
        WorkflowApp app = wps.parseDef(conf, defaultConf);
        XConfiguration protoActionConf = wps.createProtoActionConf(conf, true);
        WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
        PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
        // Resolving all variables in the job properties.
        // This ensures the Hadoop Configuration semantics is preserved.
        XConfiguration resolvedVarsConf = new XConfiguration();
        for (Map.Entry<String, String> entry : conf) {
            resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
        }
        conf = resolvedVarsConf;
        WorkflowInstance wfInstance;
        try {
            wfInstance = workflowLib.createInstance(app, conf);
        } catch (WorkflowException e) {
            throw new StoreException(e);
        }
        Configuration conf = wfInstance.getConf();
        // System.out.println("WF INSTANCE CONF:");
        // System.out.println(XmlUtils.prettyPrint(conf).toString());
        WorkflowJobBean workflow = new WorkflowJobBean();
        workflow.setId(wfInstance.getId());
        workflow.setAppName(ELUtils.resolveAppName(app.getName(), conf));
        workflow.setAppPath(conf.get(OozieClient.APP_PATH));
        workflow.setConf(XmlUtils.prettyPrint(conf).toString());
        workflow.setProtoActionConf(protoActionConf.toXmlString());
        workflow.setCreatedTime(new Date());
        workflow.setLastModifiedTime(new Date());
        workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
        workflow.setStatus(WorkflowJob.Status.PREP);
        workflow.setRun(0);
        workflow.setUser(conf.get(OozieClient.USER_NAME));
        workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
        workflow.setWorkflowInstance(wfInstance);
        workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
        // Set parent id if it doesn't already have one (for subworkflows)
        if (workflow.getParentId() == null) {
            workflow.setParentId(conf.get(SubWorkflowActionExecutor.PARENT_ID));
        }
        // Set to coord action Id if workflow submitted through coordinator
        if (workflow.getParentId() == null) {
            workflow.setParentId(parentId);
        }
        LogUtils.setLogInfo(workflow);
        LOG.debug("Workflow record created, Status [{0}]", workflow.getStatus());
        Element wfElem = XmlUtils.parseXml(app.getDefinition());
        ELEvaluator evalSla = createELEvaluatorForGroup(conf, "wf-sla-submit");
        String jobSlaXml = verifySlaElements(wfElem, evalSla);
        if (!dryrun) {
            writeSLARegistration(wfElem, jobSlaXml, workflow.getId(), workflow.getParentId(), workflow.getUser(), workflow.getGroup(), workflow.getAppName(), LOG, evalSla);
            workflow.setSlaXml(jobSlaXml);
            // System.out.println("SlaXml :"+ slaXml);
            // store.insertWorkflow(workflow);
            insertList.add(workflow);
            JPAService jpaService = Services.get().get(JPAService.class);
            if (jpaService != null) {
                try {
                    BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, null, null);
                } catch (JPAExecutorException je) {
                    throw new CommandException(je);
                }
            } else {
                LOG.error(ErrorCode.E0610);
                return null;
            }
            return workflow.getId();
        } else {
            // Checking variable substitution for dryrun
            ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(workflow, null, false, false);
            Element workflowXml = XmlUtils.parseXml(app.getDefinition());
            removeSlaElements(workflowXml);
            String workflowXmlString = XmlUtils.removeComments(XmlUtils.prettyPrint(workflowXml).toString());
            workflowXmlString = context.getELEvaluator().evaluate(workflowXmlString, String.class);
            workflowXml = XmlUtils.parseXml(workflowXmlString);
            Iterator<Element> it = workflowXml.getDescendants(new ElementFilter("job-xml"));
            // Checking all variable substitutions in job-xml files
            while (it.hasNext()) {
                Element e = it.next();
                String jobXml = e.getTextTrim();
                Path xmlPath = new Path(workflow.getAppPath(), jobXml);
                Configuration jobXmlConf = new XConfiguration(fs.open(xmlPath));
                String jobXmlConfString = XmlUtils.prettyPrint(jobXmlConf).toString();
                jobXmlConfString = XmlUtils.removeComments(jobXmlConfString);
                context.getELEvaluator().evaluate(jobXmlConfString, String.class);
            }
            return "OK";
        }
    } catch (WorkflowException ex) {
        throw new CommandException(ex);
    } catch (HadoopAccessorException ex) {
        throw new CommandException(ex);
    } catch (Exception ex) {
        throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
    }
}

Example 4 with ActionExecutorContext

use of org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext in project oozie by apache.

the class ResumeXCommand method execute.

@Override
protected Void execute() throws CommandException {
    try {
        if (workflow.getStatus() == WorkflowJob.Status.SUSPENDED) {
            InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
            workflow.getWorkflowInstance().resume();
            WorkflowInstance wfInstance = workflow.getWorkflowInstance();
            ((LiteWorkflowInstance) wfInstance).setStatus(WorkflowInstance.Status.RUNNING);
            workflow.setWorkflowInstance(wfInstance);
            workflow.setStatus(WorkflowJob.Status.RUNNING);
            // for (WorkflowActionBean action : store.getActionsForWorkflow(id, false)) {
            for (WorkflowActionBean action : jpaService.execute(new WorkflowJobGetActionsJPAExecutor(id))) {
                // START_MANUAL or END_RETRY or END_MANUAL
                if (action.isRetryOrManual()) {
                    action.setPendingOnly();
                    updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, action));
                }
                if (action.isPending()) {
                    if (action.getStatus() == WorkflowActionBean.Status.PREP || action.getStatus() == WorkflowActionBean.Status.START_MANUAL) {
                        // a repeated transient error, we have to clean up the action dir
                        if (// The control actions have invalid
                        !action.getType().equals(StartActionExecutor.TYPE) && // action dir paths because they
                        !action.getType().equals(ForkActionExecutor.TYPE) && // contain ":" (colons)
                        !action.getType().equals(JoinActionExecutor.TYPE) && !action.getType().equals(KillActionExecutor.TYPE) && !action.getType().equals(EndActionExecutor.TYPE)) {
                            ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(workflow, action, false, false);
                            if (context.getAppFileSystem().exists(context.getActionDir())) {
                                context.getAppFileSystem().delete(context.getActionDir(), true);
                            }
                        }
                        queue(new ActionStartXCommand(action.getId(), action.getType()));
                    } else {
                        if (action.getStatus() == WorkflowActionBean.Status.START_RETRY) {
                            Date nextRunTime = action.getPendingAge();
                            queue(new ActionStartXCommand(action.getId(), action.getType()), nextRunTime.getTime() - System.currentTimeMillis());
                        } else {
                            if (action.getStatus() == WorkflowActionBean.Status.DONE || action.getStatus() == WorkflowActionBean.Status.END_MANUAL) {
                                queue(new ActionEndXCommand(action.getId(), action.getType()));
                            } else {
                                if (action.getStatus() == WorkflowActionBean.Status.END_RETRY) {
                                    Date nextRunTime = action.getPendingAge();
                                    queue(new ActionEndXCommand(action.getId(), action.getType()), nextRunTime.getTime() - System.currentTimeMillis());
                                }
                            }
                        }
                    }
                }
            }
            workflow.setLastModifiedTime(new Date());
            updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MODIFIED, workflow));
            BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(null, updateList, null);
            if (EventHandlerService.isEnabled()) {
                generateEvent(workflow);
            }
            queue(new WorkflowNotificationXCommand(workflow));
        }
        return null;
    } catch (WorkflowException ex) {
        throw new CommandException(ex);
    } catch (JPAExecutorException e) {
        throw new CommandException(e);
    } catch (HadoopAccessorException e) {
        throw new CommandException(e);
    } catch (IOException e) {
        throw new CommandException(ErrorCode.E0902, e.getMessage(), e);
    } catch (URISyntaxException e) {
        throw new CommandException(ErrorCode.E0902, e.getMessage(), e);
    } finally {
        updateParentIfNecessary(workflow);
    }
}

Example 5 with ActionExecutorContext

use of org.apache.oozie.command.wf.ActionXCommand.ActionExecutorContext in project oozie by apache.

the class SignalXCommand method startForkedActions.

public void startForkedActions(List<WorkflowActionBean> workflowActionBeanListForForked) throws CommandException {
    List<CallableWrapper<ActionExecutorContext>> tasks = new ArrayList<CallableWrapper<ActionExecutorContext>>();
    List<UpdateEntry> updateList = new ArrayList<UpdateEntry>();
    List<JsonBean> insertList = new ArrayList<JsonBean>();
    boolean endWorkflow = false;
    boolean submitJobByQueuing = false;
    for (WorkflowActionBean workflowActionBean : workflowActionBeanListForForked) {
        LOG.debug("Starting forked actions parallely : " + workflowActionBean.getId());
        tasks.add(Services.get().get(CallableQueueService.class).new CallableWrapper<ActionExecutorContext>(new ForkedActionStartXCommand(wfJob, workflowActionBean.getId(), workflowActionBean.getType()), 0));
    }
    try {
        List<Future<ActionExecutorContext>> futures = Services.get().get(CallableQueueService.class).invokeAll(tasks);
        for (Future<ActionExecutorContext> result : futures) {
            if (result == null) {
                submitJobByQueuing = true;
                continue;
            }
            ActionExecutorContext context = result.get();
            Map<String, String> contextVariableMap = ((ForkedActionExecutorContext) context).getContextMap();
            LOG.debug("contextVariableMap size of action " + context.getAction().getId() + " is " + contextVariableMap.size());
            for (String key : contextVariableMap.keySet()) {
                context.setVarToWorkflow(key, contextVariableMap.get(key));
            }
            if (context.getJobStatus() != null && context.getJobStatus().equals(Job.Status.FAILED)) {
                LOG.warn("Action has failed, failing job" + context.getAction().getId());
                new ActionStartXCommand(context.getAction().getId(), null).failJob(context);
                updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_START, (WorkflowActionBean) context.getAction()));
                if (context.isShouldEndWF()) {
                    endWorkflow = true;
                }
            }
            if (context.getJobStatus() != null && context.getJobStatus().equals(Job.Status.SUSPENDED)) {
                LOG.warn("Action has failed, failing job" + context.getAction().getId());
                new ActionStartXCommand(context.getAction().getId(), null).handleNonTransient(context, null, WorkflowAction.Status.START_MANUAL);
                updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_START, (WorkflowActionBean) context.getAction()));
                if (context.isShouldEndWF()) {
                    endWorkflow = true;
                }
            }
        }
        if (endWorkflow) {
            endWF(insertList);
        }
    } catch (Exception e) {
        LOG.error("Error running forked jobs parallely", e);
        startForkedActionsByQueuing(workflowActionBeanListForForked);
        submitJobByQueuing = false;
    }
    if (submitJobByQueuing && !endWorkflow) {
        LOG.error("There is error in running forked jobs parallely");
        startForkedActionsByQueuing(workflowActionBeanListForForked);
    }
    wfJob.setLastModifiedTime(new Date());
    updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MODIFIED, wfJob));
    try {
        BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null);
    } catch (JPAExecutorException e) {
        throw new CommandException(e);
    }
    LOG.debug("forked actions submitted parallely");
}