use of org.apache.oozie.workflow.WorkflowException in project oozie by apache.
the class KillXCommand method execute.
@Override
protected Void execute() throws CommandException {
LOG.info("STARTED WorkflowKillXCommand for jobId=" + wfId);
wfJob.setEndTime(new Date());
if (wfJob.getStatus() != WorkflowJob.Status.FAILED) {
InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
wfJob.setStatus(WorkflowJob.Status.KILLED);
SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(wfJob.getSlaXml(), wfJob.getId(), Status.KILLED, SlaAppType.WORKFLOW_JOB);
if (slaEvent != null) {
insertList.add(slaEvent);
}
try {
wfJob.getWorkflowInstance().kill();
} catch (WorkflowException e) {
throw new CommandException(ErrorCode.E0725, e.getMessage(), e);
}
WorkflowInstance wfInstance = wfJob.getWorkflowInstance();
((LiteWorkflowInstance) wfInstance).setStatus(WorkflowInstance.Status.KILLED);
wfJob.setWorkflowInstance(wfInstance);
}
try {
for (WorkflowActionBean action : actionList) {
if (action.getStatus() == WorkflowActionBean.Status.RUNNING || action.getStatus() == WorkflowActionBean.Status.DONE) {
if (!(actionService.getExecutor(action.getType()) instanceof ControlNodeActionExecutor)) {
action.setPending();
}
action.setStatus(WorkflowActionBean.Status.KILLED);
updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, action));
queue(new ActionKillXCommand(action.getId(), action.getType()));
} else if (action.getStatus() == WorkflowActionBean.Status.PREP || action.getStatus() == WorkflowActionBean.Status.START_RETRY || action.getStatus() == WorkflowActionBean.Status.START_MANUAL || action.getStatus() == WorkflowActionBean.Status.END_RETRY || action.getStatus() == WorkflowActionBean.Status.END_MANUAL || action.getStatus() == WorkflowActionBean.Status.USER_RETRY) {
action.setStatus(WorkflowActionBean.Status.KILLED);
action.resetPending();
SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(action.getSlaXml(), action.getId(), Status.KILLED, SlaAppType.WORKFLOW_ACTION);
if (slaEvent != null) {
insertList.add(slaEvent);
}
updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, action));
if (EventHandlerService.isEnabled() && !(actionService.getExecutor(action.getType()) instanceof ControlNodeActionExecutor)) {
generateEvent(action, wfJob.getUser());
}
}
}
wfJob.setLastModifiedTime(new Date());
updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_END, wfJob));
BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null);
if (EventHandlerService.isEnabled()) {
generateEvent(wfJob);
}
queue(new WorkflowNotificationXCommand(wfJob));
} catch (JPAExecutorException e) {
throw new CommandException(e);
} finally {
if (wfJob.getStatus() == WorkflowJob.Status.KILLED) {
// To delete the WF temp dir
new WfEndXCommand(wfJob).call();
}
updateParentIfNecessary(wfJob);
}
LOG.info("ENDED WorkflowKillXCommand for jobId=" + wfId);
return null;
}
use of org.apache.oozie.workflow.WorkflowException in project oozie by apache.
the class SignalXCommand method execute.
@Override
protected Void execute() throws CommandException {
LOG.debug("STARTED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
WorkflowInstance workflowInstance = wfJob.getWorkflowInstance();
workflowInstance.setTransientVar(WorkflowStoreService.WORKFLOW_BEAN, wfJob);
WorkflowJob.Status prevStatus = wfJob.getStatus();
boolean completed = false, skipAction = false;
WorkflowActionBean syncAction = null;
List<WorkflowActionBean> workflowActionBeanListForForked = new ArrayList<WorkflowActionBean>();
if (wfAction == null) {
if (wfJob.getStatus() == WorkflowJob.Status.PREP) {
try {
completed = workflowInstance.start();
} catch (WorkflowException e) {
throw new CommandException(e);
}
wfJob.setStatus(WorkflowJob.Status.RUNNING);
wfJob.setStartTime(new Date());
wfJob.setWorkflowInstance(workflowInstance);
generateEvent = true;
// 1. Add SLA status event for WF-JOB with status STARTED
SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(wfJob.getSlaXml(), jobId, Status.STARTED, SlaAppType.WORKFLOW_JOB);
if (slaEvent != null) {
insertList.add(slaEvent);
}
// 2. Add SLA registration events for all WF_ACTIONS
createSLARegistrationForAllActions(workflowInstance.getApp().getDefinition(), wfJob.getUser(), wfJob.getGroup(), wfJob.getConf());
queue(new WorkflowNotificationXCommand(wfJob));
} else {
throw new CommandException(ErrorCode.E0801, wfJob.getId());
}
} else {
WorkflowInstance.Status initialStatus = workflowInstance.getStatus();
String skipVar = workflowInstance.getVar(wfAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunXCommand.TO_SKIP);
if (skipVar != null) {
skipAction = skipVar.equals("true");
}
try {
completed = workflowInstance.signal(wfAction.getExecutionPath(), wfAction.getSignalValue());
} catch (WorkflowException e) {
LOG.error("Workflow action failed : " + e.getMessage(), e);
wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString()));
completed = true;
}
wfJob.setWorkflowInstance(workflowInstance);
wfAction.resetPending();
if (!skipAction) {
wfAction.setTransition(workflowInstance.getTransition(wfAction.getName()));
queue(new WorkflowNotificationXCommand(wfJob, wfAction));
}
updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS, wfAction));
WorkflowInstance.Status endStatus = workflowInstance.getStatus();
if (endStatus != initialStatus) {
generateEvent = true;
}
}
if (completed) {
try {
for (String actionToKillId : WorkflowStoreService.getActionsToKill(workflowInstance)) {
WorkflowActionBean actionToKill;
actionToKill = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION_ID_TYPE_LASTCHECK, actionToKillId);
actionToKill.setPending();
actionToKill.setStatus(WorkflowActionBean.Status.KILLED);
updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, actionToKill));
queue(new ActionKillXCommand(actionToKill.getId(), actionToKill.getType()));
}
for (String actionToFailId : WorkflowStoreService.getActionsToFail(workflowInstance)) {
WorkflowActionBean actionToFail = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION_FAIL, actionToFailId);
actionToFail.resetPending();
actionToFail.setStatus(WorkflowActionBean.Status.FAILED);
if (wfJobErrorCode != null) {
wfJobErrorCode = actionToFail.getErrorCode();
wfJobErrorMsg = actionToFail.getErrorMessage();
}
queue(new WorkflowNotificationXCommand(wfJob, actionToFail));
SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(wfAction.getSlaXml(), wfAction.getId(), Status.FAILED, SlaAppType.WORKFLOW_ACTION);
if (slaEvent != null) {
insertList.add(slaEvent);
}
updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_STATUS_PENDING, actionToFail));
}
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
wfJob.setStatus(WorkflowJob.Status.valueOf(workflowInstance.getStatus().toString()));
wfJob.setEndTime(new Date());
wfJob.setWorkflowInstance(workflowInstance);
Status slaStatus = Status.SUCCEEDED;
switch(wfJob.getStatus()) {
case SUCCEEDED:
slaStatus = Status.SUCCEEDED;
break;
case KILLED:
slaStatus = Status.KILLED;
break;
case FAILED:
slaStatus = Status.FAILED;
break;
default:
// TODO SUSPENDED
break;
}
SLAEventBean slaEvent = SLADbXOperations.createStatusEvent(wfJob.getSlaXml(), jobId, slaStatus, SlaAppType.WORKFLOW_JOB);
if (slaEvent != null) {
insertList.add(slaEvent);
}
queue(new WorkflowNotificationXCommand(wfJob));
if (wfJob.getStatus() == WorkflowJob.Status.SUCCEEDED) {
InstrumentUtils.incrJobCounter(INSTR_SUCCEEDED_JOBS_COUNTER_NAME, 1, getInstrumentation());
}
// output message for Kill node
if (wfAction != null) {
// wfAction could be a no-op job
NodeDef nodeDef = workflowInstance.getNodeDef(wfAction.getExecutionPath());
if (nodeDef != null && nodeDef instanceof KillNodeDef) {
boolean isRetry = false;
boolean isUserRetry = false;
ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(wfJob, wfAction, isRetry, isUserRetry);
InstrumentUtils.incrJobCounter(INSTR_KILLED_JOBS_COUNTER_NAME, 1, getInstrumentation());
try {
String tmpNodeConf = nodeDef.getConf();
String message = context.getELEvaluator().evaluate(tmpNodeConf, String.class);
LOG.debug("Try to resolve KillNode message for jobid [{0}], actionId [{1}], before resolve [{2}], " + "after resolve [{3}]", jobId, actionId, tmpNodeConf, message);
if (wfAction.getErrorCode() != null) {
wfAction.setErrorInfo(wfAction.getErrorCode(), message);
} else {
wfAction.setErrorInfo(ErrorCode.E0729.toString(), message);
}
} catch (Exception ex) {
LOG.warn("Exception in SignalXCommand when processing Kill node message: {0}", ex.getMessage(), ex);
wfAction.setErrorInfo(ErrorCode.E0756.toString(), ErrorCode.E0756.format(ex.getMessage()));
wfAction.setStatus(WorkflowAction.Status.ERROR);
}
updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_PENDING_TRANS_ERROR, wfAction));
}
}
} else {
for (WorkflowActionBean newAction : WorkflowStoreService.getActionsToStart(workflowInstance)) {
boolean isOldWFAction = false;
// old action. To avoid twice entry for same action, Checking in Db if the workflow action already exist.
if (SubWorkflowActionExecutor.ACTION_TYPE.equals(newAction.getType())) {
try {
WorkflowActionBean oldAction = WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION_CHECK, newAction.getId());
newAction.setExternalId(oldAction.getExternalId());
newAction.setCreatedTime(oldAction.getCreatedTime());
isOldWFAction = true;
} catch (JPAExecutorException e) {
if (e.getErrorCode() != ErrorCode.E0605) {
throw new CommandException(e);
}
}
}
String skipVar = workflowInstance.getVar(newAction.getName() + WorkflowInstance.NODE_VAR_SEPARATOR + ReRunXCommand.TO_SKIP);
boolean skipNewAction = false, suspendNewAction = false;
if (skipVar != null) {
skipNewAction = skipVar.equals("true");
}
if (skipNewAction) {
WorkflowActionBean oldAction = new WorkflowActionBean();
oldAction.setId(newAction.getId());
oldAction.setPending();
oldAction.setExecutionPath(newAction.getExecutionPath());
updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_PENDING, oldAction));
queue(new SignalXCommand(jobId, oldAction.getId()));
} else {
if (!skipAction) {
try {
// Make sure that transition node for a forked action
// is inserted only once
WorkflowActionQueryExecutor.getInstance().get(WorkflowActionQuery.GET_ACTION_ID_TYPE_LASTCHECK, newAction.getId());
continue;
} catch (JPAExecutorException jee) {
}
}
suspendNewAction = checkForSuspendNode(newAction);
newAction.setPending();
String actionSlaXml = getActionSLAXml(newAction.getName(), workflowInstance.getApp().getDefinition(), wfJob.getConf());
newAction.setSlaXml(actionSlaXml);
if (!isOldWFAction) {
newAction.setCreatedTime(new Date());
insertList.add(newAction);
} else {
updateList.add(new UpdateEntry<WorkflowActionQuery>(WorkflowActionQuery.UPDATE_ACTION_START, newAction));
}
LOG.debug("SignalXCommand: Name: " + newAction.getName() + ", Id: " + newAction.getId() + ", Authcode:" + newAction.getCred());
if (wfAction != null) {
// null during wf job submit
ActionService as = Services.get().get(ActionService.class);
ActionExecutor current = as.getExecutor(wfAction.getType());
LOG.trace("Current Action Type:" + current.getClass());
if (!suspendNewAction) {
if (current instanceof StartActionExecutor) {
// Excluding :start: here from executing first action synchronously since it
// blocks the consumer thread till the action is submitted to Hadoop,
// in turn reducing the number of new submissions the threads can accept.
// Would also be susceptible to longer delays in case Hadoop cluster is busy.
queue(new ActionStartXCommand(newAction.getId(), newAction.getType()));
} else if (current instanceof ForkActionExecutor) {
if (ConfigurationService.getBoolean(SignalXCommand.FORK_PARALLEL_JOBSUBMISSION)) {
workflowActionBeanListForForked.add(newAction);
} else {
queue(new ActionStartXCommand(newAction.getId(), newAction.getType()));
}
} else {
syncAction = newAction;
}
} else {
// will be ignored.
if (ConfigurationService.getBoolean(SignalXCommand.FORK_PARALLEL_JOBSUBMISSION)) {
workflowActionBeanListForForked.add(newAction);
}
}
} else {
// first action after wf submit should always be sync
syncAction = newAction;
}
}
}
}
try {
wfJob.setLastModifiedTime(new Date());
updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MOD_START_END, wfJob));
// call JPAExecutor to do the bulk writes
BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null);
if (prevStatus != wfJob.getStatus()) {
LOG.debug("Updated the workflow status to " + wfJob.getId() + " status =" + wfJob.getStatusStr());
}
if (generateEvent && EventHandlerService.isEnabled()) {
generateEvent(wfJob, wfJobErrorCode, wfJobErrorMsg);
}
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
// undue delay from between end of previous and start of next action
if (wfJob.getStatus() != WorkflowJob.Status.RUNNING && wfJob.getStatus() != WorkflowJob.Status.SUSPENDED) {
// only for asynchronous actions, parent coord action's external id will
// persisted and following update will succeed.
updateParentIfNecessary(wfJob);
// To delete the WF temp dir
new WfEndXCommand(wfJob).call();
} else if (syncAction != null) {
new ActionStartXCommand(wfJob, syncAction.getId(), syncAction.getType()).call();
} else if (!workflowActionBeanListForForked.isEmpty() && !checkForSuspendNode(workflowActionBeanListForForked)) {
startForkedActions(workflowActionBeanListForForked);
}
LOG.debug("ENDED SignalCommand for jobid=" + jobId + ", actionId=" + actionId);
return null;
}
use of org.apache.oozie.workflow.WorkflowException in project oozie by apache.
the class SuspendXCommand method execute.
@Override
protected Void execute() throws CommandException {
InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
try {
suspendJob(this.jpaService, this.wfJobBean, this.wfid, null, updateList);
this.wfJobBean.setLastModifiedTime(new Date());
updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW_STATUS_INSTANCE_MODIFIED, this.wfJobBean));
BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(null, updateList, null);
queue(new WorkflowNotificationXCommand(this.wfJobBean));
// Calling suspend recursively to handle parent workflow
suspendParentWorkFlow();
} catch (WorkflowException e) {
throw new CommandException(e);
} catch (JPAExecutorException je) {
throw new CommandException(je);
} finally {
updateParentIfNecessary(wfJobBean);
}
return null;
}
use of org.apache.oozie.workflow.WorkflowException in project oozie by apache.
the class SubmitXCommand method execute.
@Override
protected String execute() throws CommandException {
InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
try {
XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
String user = conf.get(OozieClient.USER_NAME);
URI uri = new URI(conf.get(OozieClient.APP_PATH));
HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
Configuration fsConf = has.createConfiguration(uri.getAuthority());
FileSystem fs = has.createFileSystem(user, uri, fsConf);
Path configDefault = null;
Configuration defaultConf = null;
// app path could be a directory
Path path = new Path(uri.getPath());
if (!fs.isFile(path)) {
configDefault = new Path(path, CONFIG_DEFAULT);
} else {
configDefault = new Path(path.getParent(), CONFIG_DEFAULT);
}
if (fs.exists(configDefault)) {
try {
defaultConf = new XConfiguration(fs.open(configDefault));
PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
XConfiguration.injectDefaults(defaultConf, conf);
} catch (IOException ex) {
throw new IOException("default configuration file, " + ex.getMessage(), ex);
}
}
if (defaultConf != null) {
defaultConf = resolveDefaultConfVariables(defaultConf);
}
WorkflowApp app = wps.parseDef(conf, defaultConf);
XConfiguration protoActionConf = wps.createProtoActionConf(conf, true);
WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
// Resolving all variables in the job properties.
// This ensures the Hadoop Configuration semantics is preserved.
XConfiguration resolvedVarsConf = new XConfiguration();
for (Map.Entry<String, String> entry : conf) {
resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
}
conf = resolvedVarsConf;
WorkflowInstance wfInstance;
try {
wfInstance = workflowLib.createInstance(app, conf);
} catch (WorkflowException e) {
throw new StoreException(e);
}
Configuration conf = wfInstance.getConf();
// System.out.println("WF INSTANCE CONF:");
// System.out.println(XmlUtils.prettyPrint(conf).toString());
WorkflowJobBean workflow = new WorkflowJobBean();
workflow.setId(wfInstance.getId());
workflow.setAppName(ELUtils.resolveAppName(app.getName(), conf));
workflow.setAppPath(conf.get(OozieClient.APP_PATH));
workflow.setConf(XmlUtils.prettyPrint(conf).toString());
workflow.setProtoActionConf(protoActionConf.toXmlString());
workflow.setCreatedTime(new Date());
workflow.setLastModifiedTime(new Date());
workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
workflow.setStatus(WorkflowJob.Status.PREP);
workflow.setRun(0);
workflow.setUser(conf.get(OozieClient.USER_NAME));
workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
workflow.setWorkflowInstance(wfInstance);
workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
// Set parent id if it doesn't already have one (for subworkflows)
if (workflow.getParentId() == null) {
workflow.setParentId(conf.get(SubWorkflowActionExecutor.PARENT_ID));
}
// Set to coord action Id if workflow submitted through coordinator
if (workflow.getParentId() == null) {
workflow.setParentId(parentId);
}
LogUtils.setLogInfo(workflow);
LOG.debug("Workflow record created, Status [{0}]", workflow.getStatus());
Element wfElem = XmlUtils.parseXml(app.getDefinition());
ELEvaluator evalSla = createELEvaluatorForGroup(conf, "wf-sla-submit");
String jobSlaXml = verifySlaElements(wfElem, evalSla);
if (!dryrun) {
writeSLARegistration(wfElem, jobSlaXml, workflow.getId(), workflow.getParentId(), workflow.getUser(), workflow.getGroup(), workflow.getAppName(), LOG, evalSla);
workflow.setSlaXml(jobSlaXml);
// System.out.println("SlaXml :"+ slaXml);
// store.insertWorkflow(workflow);
insertList.add(workflow);
JPAService jpaService = Services.get().get(JPAService.class);
if (jpaService != null) {
try {
BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, null, null);
} catch (JPAExecutorException je) {
throw new CommandException(je);
}
} else {
LOG.error(ErrorCode.E0610);
return null;
}
return workflow.getId();
} else {
// Checking variable substitution for dryrun
ActionExecutorContext context = new ActionXCommand.ActionExecutorContext(workflow, null, false, false);
Element workflowXml = XmlUtils.parseXml(app.getDefinition());
removeSlaElements(workflowXml);
String workflowXmlString = XmlUtils.removeComments(XmlUtils.prettyPrint(workflowXml).toString());
workflowXmlString = context.getELEvaluator().evaluate(workflowXmlString, String.class);
workflowXml = XmlUtils.parseXml(workflowXmlString);
Iterator<Element> it = workflowXml.getDescendants(new ElementFilter("job-xml"));
// Checking all variable substitutions in job-xml files
while (it.hasNext()) {
Element e = it.next();
String jobXml = e.getTextTrim();
Path xmlPath = new Path(workflow.getAppPath(), jobXml);
Configuration jobXmlConf = new XConfiguration(fs.open(xmlPath));
String jobXmlConfString = XmlUtils.prettyPrint(jobXmlConf).toString();
jobXmlConfString = XmlUtils.removeComments(jobXmlConfString);
context.getELEvaluator().evaluate(jobXmlConfString, String.class);
}
return "OK";
}
} catch (WorkflowException ex) {
throw new CommandException(ex);
} catch (HadoopAccessorException ex) {
throw new CommandException(ex);
} catch (Exception ex) {
throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
}
}
use of org.apache.oozie.workflow.WorkflowException in project oozie by apache.
the class LiteWorkflowApp method readFields.
@Override
public void readFields(DataInput dataInput) throws IOException {
name = dataInput.readUTF();
// read the full definition back
int definitionListFlag = dataInput.readInt();
if (definitionListFlag > NEW_SERIALIZATION_METHOD_FLAG) {
// negative number marking the usage of StringSerializationUtil
// positive number is the length of the array the String was broken into.
StringBuilder sb = new StringBuilder();
for (int i = 0; i < definitionListFlag; i++) {
sb.append(dataInput.readUTF());
}
definition = sb.toString();
} else {
definition = StringSerializationUtil.readString(dataInput);
}
int numNodes = dataInput.readInt();
for (int x = 0; x < numNodes; x++) {
try {
String nodeDefClass = dataInput.readUTF();
NodeDef node = (NodeDef) ReflectionUtils.newInstance(Class.forName(nodeDefClass), null);
node.readFields(dataInput);
addNode(node);
} catch (WorkflowException ex) {
throw new IOException(ex);
} catch (ClassNotFoundException e) {
throw new IOException(e);
}
}
}
Aggregations