Examples with DagEngine org.apache.oozie.DagEngine used on opensource projects

Example 16 with DagEngine

use of org.apache.oozie.DagEngine in project oozie by apache.

the class V1JobsServlet method bulkModifyJobs.

private JSONObject bulkModifyJobs(HttpServletRequest request, HttpServletResponse response) throws XServletException, IOException {
    String action = request.getParameter(RestConstants.ACTION_PARAM);
    String jobType = request.getParameter(RestConstants.JOBTYPE_PARAM);
    String filter = request.getParameter(RestConstants.JOBS_FILTER_PARAM);
    String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
    String lenStr = request.getParameter(RestConstants.LEN_PARAM);
    String timeZoneId = request.getParameter(RestConstants.TIME_ZONE_PARAM) == null ? "GMT" : request.getParameter(RestConstants.TIME_ZONE_PARAM);
    int start = (startStr != null) ? Integer.parseInt(startStr) : 1;
    start = (start < 1) ? 1 : start;
    int len = (lenStr != null) ? Integer.parseInt(lenStr) : 50;
    len = (len < 1) ? 50 : len;
    JSONObject json;
    List<String> ids = new ArrayList<String>();
    if (jobType.equals("wf")) {
        WorkflowsInfo jobs;
        DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(getUser(request));
        try {
            switch(action) {
                case RestConstants.JOB_ACTION_KILL:
                    jobs = dagEngine.killJobs(filter, start, len);
                    break;
                case RestConstants.JOB_ACTION_SUSPEND:
                    jobs = dagEngine.suspendJobs(filter, start, len);
                    break;
                case RestConstants.JOB_ACTION_RESUME:
                    jobs = dagEngine.resumeJobs(filter, start, len);
                    break;
                default:
                    throw new DagEngineException(ErrorCode.E0301, action);
            }
        } catch (DagEngineException ex) {
            throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
        }
        json = OozieJsonFactory.getWFJSONObject(jobs, timeZoneId);
    } else if (jobType.equals("bundle")) {
        BundleJobInfo jobs;
        BundleEngine bundleEngine = Services.get().get(BundleEngineService.class).getBundleEngine(getUser(request));
        try {
            switch(action) {
                case RestConstants.JOB_ACTION_KILL:
                    jobs = bundleEngine.killJobs(filter, start, len);
                    break;
                case RestConstants.JOB_ACTION_SUSPEND:
                    jobs = bundleEngine.suspendJobs(filter, start, len);
                    break;
                case RestConstants.JOB_ACTION_RESUME:
                    jobs = bundleEngine.resumeJobs(filter, start, len);
                    break;
                default:
                    throw new BundleEngineException(ErrorCode.E0301, action);
            }
        } catch (BundleEngineException ex) {
            throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
        }
        json = OozieJsonFactory.getBundleJSONObject(jobs, timeZoneId);
    } else {
        CoordinatorJobInfo jobs;
        CoordinatorEngine coordEngine = Services.get().get(CoordinatorEngineService.class).getCoordinatorEngine(getUser(request));
        try {
            switch(action) {
                case RestConstants.JOB_ACTION_KILL:
                    jobs = coordEngine.killJobs(filter, start, len);
                    break;
                case RestConstants.JOB_ACTION_SUSPEND:
                    jobs = coordEngine.suspendJobs(filter, start, len);
                    break;
                case RestConstants.JOB_ACTION_RESUME:
                    jobs = coordEngine.resumeJobs(filter, start, len);
                    break;
                default:
                    throw new CoordinatorEngineException(ErrorCode.E0301, action);
            }
        } catch (CoordinatorEngineException ex) {
            throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
        }
        json = OozieJsonFactory.getCoordJSONObject(jobs, timeZoneId);
    }
    json.put(JsonTags.JOB_IDS, toJSONArray(ids));
    return json;
}

Example 17 with DagEngine

use of org.apache.oozie.DagEngine in project oozie by apache.

the class V2JobServlet method getJMSTopicName.

@Override
protected String getJMSTopicName(HttpServletRequest request, HttpServletResponse response) throws XServletException, IOException {
    String topicName;
    String jobId = getResourceName(request);
    DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(getUser(request));
    try {
        topicName = dagEngine.getJMSTopicName(jobId);
    } catch (DagEngineException ex) {
        throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }
    return topicName;
}

Example 18 with DagEngine

use of org.apache.oozie.DagEngine in project oozie by apache.

the class CoordActionStartXCommand method execute.

@Override
protected Void execute() throws CommandException {
    boolean makeFail = true;
    String errCode = "";
    String errMsg = "";
    ParamChecker.notEmpty(user, "user");
    log.debug("actionid=" + actionId + ", status=" + coordAction.getStatus());
    if (coordAction.getStatus() == CoordinatorAction.Status.SUBMITTED) {
        // log.debug("getting.. job id: " + coordAction.getJobId());
        // create merged runConf to pass to WF Engine
        Configuration runConf = mergeConfig(coordAction);
        coordAction.setRunConf(XmlUtils.prettyPrint(runConf).toString());
        // log.debug("%%% merged runconf=" +
        // XmlUtils.prettyPrint(runConf).toString());
        DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user);
        try {
            Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf()));
            SLAEventBean slaEvent = SLADbOperations.createStatusEvent(coordAction.getSlaXml(), coordAction.getId(), Status.STARTED, SlaAppType.COORDINATOR_ACTION, log);
            if (slaEvent != null) {
                insertList.add(slaEvent);
            }
            if (OozieJobInfo.isJobInfoEnabled()) {
                conf.set(OozieJobInfo.COORD_ID, actionId);
                conf.set(OozieJobInfo.COORD_NAME, appName);
                conf.set(OozieJobInfo.COORD_NOMINAL_TIME, coordAction.getNominalTimestamp().toString());
            }
            // Normalize workflow appPath here;
            JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf);
            if (coordAction.getExternalId() != null) {
                conf.setBoolean(OozieClient.RERUN_FAIL_NODES, true);
                dagEngine.reRun(coordAction.getExternalId(), conf);
            } else {
                // Pushing the nominal time in conf to use for launcher tag search
                conf.set(OOZIE_COORD_ACTION_NOMINAL_TIME, String.valueOf(coordAction.getNominalTime().getTime()));
                String wfId = dagEngine.submitJobFromCoordinator(conf, actionId);
                coordAction.setExternalId(wfId);
            }
            coordAction.setStatus(CoordinatorAction.Status.RUNNING);
            coordAction.incrementAndGetPending();
            // store.updateCoordinatorAction(coordAction);
            JPAService jpaService = Services.get().get(JPAService.class);
            if (jpaService != null) {
                log.debug("Updating WF record for WFID :" + coordAction.getExternalId() + " with parent id: " + actionId);
                WorkflowJobBean wfJob = WorkflowJobQueryExecutor.getInstance().get(WorkflowJobQuery.GET_WORKFLOW_STARTTIME, coordAction.getExternalId());
                wfJob.setParentId(actionId);
                wfJob.setLastModifiedTime(new Date());
                BatchQueryExecutor executor = BatchQueryExecutor.getInstance();
                updateList.add(new UpdateEntry<WorkflowJobQuery>(WorkflowJobQuery.UPDATE_WORKFLOW_PARENT_MODIFIED, wfJob));
                updateList.add(new UpdateEntry<CoordActionQuery>(CoordActionQuery.UPDATE_COORD_ACTION_FOR_START, coordAction));
                try {
                    executor.executeBatchInsertUpdateDelete(insertList, updateList, null);
                    queue(new CoordActionNotificationXCommand(coordAction), 100);
                    if (EventHandlerService.isEnabled()) {
                        generateEvent(coordAction, user, appName, wfJob.getStartTime());
                    }
                } catch (JPAExecutorException je) {
                    throw new CommandException(je);
                }
            } else {
                log.error(ErrorCode.E0610);
            }
            makeFail = false;
        } catch (DagEngineException dee) {
            errMsg = dee.getMessage();
            errCode = dee.getErrorCode().toString();
            log.warn("can not create DagEngine for submitting jobs", dee);
        } catch (CommandException ce) {
            errMsg = ce.getMessage();
            errCode = ce.getErrorCode().toString();
            log.warn("command exception occurred ", ce);
        } catch (java.io.IOException ioe) {
            errMsg = ioe.getMessage();
            errCode = "E1005";
            log.warn("Configuration parse error. read from DB :" + coordAction.getRunConf(), ioe);
        } catch (Exception ex) {
            errMsg = ex.getMessage();
            errCode = "E1005";
            log.warn("can not create DagEngine for submitting jobs", ex);
        } finally {
            if (makeFail == true) {
                // No DB exception occurs
                log.error("Failing the action " + coordAction.getId() + ". Because " + errCode + " : " + errMsg);
                coordAction.setStatus(CoordinatorAction.Status.FAILED);
                if (errMsg.length() > 254) {
                    // Because table column size is 255
                    errMsg = errMsg.substring(0, 255);
                }
                coordAction.setErrorMessage(errMsg);
                coordAction.setErrorCode(errCode);
                updateList = new ArrayList<UpdateEntry>();
                updateList.add(new UpdateEntry<CoordActionQuery>(CoordActionQuery.UPDATE_COORD_ACTION_FOR_START, coordAction));
                insertList = new ArrayList<JsonBean>();
                SLAEventBean slaEvent = SLADbOperations.createStatusEvent(coordAction.getSlaXml(), coordAction.getId(), Status.FAILED, SlaAppType.COORDINATOR_ACTION, log);
                if (slaEvent != null) {
                    // Update SLA events
                    insertList.add(slaEvent);
                }
                try {
                    // call JPAExecutor to do the bulk writes
                    BatchQueryExecutor.getInstance().executeBatchInsertUpdateDelete(insertList, updateList, null);
                    if (EventHandlerService.isEnabled()) {
                        generateEvent(coordAction, user, appName, null);
                    }
                } catch (JPAExecutorException je) {
                    throw new CommandException(je);
                }
                queue(new CoordActionReadyXCommand(coordAction.getJobId()));
            }
        }
    }
    return null;
}

Example 19 with DagEngine

use of org.apache.oozie.DagEngine in project oozie by apache.

the class CallbackServlet method doPost.

/**
 * POST callback
 */
@Override
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    String queryString = request.getQueryString();
    CallbackService callbackService = Services.get().get(CallbackService.class);
    if (!callbackService.isValid(queryString)) {
        throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0402, queryString);
    }
    String actionId = callbackService.getActionId(queryString);
    if (actionId == null) {
        throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0402, queryString);
    }
    log = XLog.getLog(getClass());
    setLogInfo(actionId);
    log.debug("Received a CallbackServlet.doPost() with query string " + queryString);
    validateContentType(request, RestConstants.TEXT_CONTENT_TYPE);
    try {
        log.info(XLog.STD, "callback for action [{0}]", actionId);
        String data = IOUtils.getReaderAsString(request.getReader(), maxDataLen);
        Properties props = PropertiesUtils.stringToProperties(data);
        DagEngine dagEngine = Services.get().get(DagEngineService.class).getSystemDagEngine();
        dagEngine.processCallback(actionId, callbackService.getExternalStatus(queryString), props);
    } catch (IOException ex) {
        if (ex.getMessage().startsWith("stream exceeds limit")) {
            // TODO, WE MUST SET THE ACTION TO ERROR
            throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ErrorCode.E0403, "data too long");
        } else {
            throw ex;
        }
    } catch (DagEngineException ex) {
        throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }
}

Example 20 with DagEngine

use of org.apache.oozie.DagEngine in project oozie by apache.

the class V1JobsServlet method submitHttpJob.

/**
 * service implementation to submit a http job
 */
private JSONObject submitHttpJob(HttpServletRequest request, Configuration conf, String jobType) throws XServletException {
    JSONObject json = new JSONObject();
    try {
        String user = conf.get(OozieClient.USER_NAME);
        DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user);
        String id = dagEngine.submitHttpJob(conf, jobType);
        json.put(JsonTags.JOB_ID, id);
    } catch (DagEngineException ex) {
        throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
    }
    return json;
}