Examples with CoordActionsActiveCountJPAExecutor org.apache.oozie.executor.jpa.CoordActionsActiveCountJPAExecutor used on opensource projects

Example 1 with CoordActionsActiveCountJPAExecutor

use of org.apache.oozie.executor.jpa.CoordActionsActiveCountJPAExecutor in project oozie by apache.

the class CoordMaterializeTransitionXCommand method materializeActions.

/**
 * Create action instances starting from "startMatdTime" to "endMatdTime" and store them into coord action table.
 *
 * @param dryrun if this is a dry run
 * @throws Exception thrown if failed to materialize actions
 */
protected String materializeActions(boolean dryrun) throws Exception {
    Configuration jobConf = null;
    try {
        jobConf = new XConfiguration(new StringReader(coordJob.getConf()));
    } catch (IOException ioe) {
        LOG.warn("Configuration parse error. read from DB :" + coordJob.getConf(), ioe);
        throw new CommandException(ErrorCode.E1005, ioe.getMessage(), ioe);
    }
    String jobXml = coordJob.getJobXml();
    Element eJob = XmlUtils.parseXml(jobXml);
    TimeZone appTz = DateUtils.getTimeZone(coordJob.getTimeZone());
    String frequency = coordJob.getFrequency();
    TimeUnit freqTU = TimeUnit.valueOf(coordJob.getTimeUnitStr());
    TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration"));
    Calendar start = Calendar.getInstance(appTz);
    start.setTime(startMatdTime);
    DateUtils.moveToEnd(start, endOfFlag);
    Calendar end = Calendar.getInstance(appTz);
    end.setTime(endMatdTime);
    lastActionNumber = coordJob.getLastActionNumber();
    // Intentionally printing dates in their own timezone, not Oozie timezone
    LOG.info("materialize actions for tz=" + appTz.getDisplayName() + ",\n start=" + start.getTime() + ", end=" + end.getTime() + ",\n timeUnit " + freqTU.getCalendarUnit() + ",\n frequency :" + frequency + ":" + freqTU + ",\n lastActionNumber " + lastActionNumber);
    // Keep the actual start time
    Calendar origStart = Calendar.getInstance(appTz);
    origStart.setTime(coordJob.getStartTimestamp());
    // Move to the End of duration, if needed.
    DateUtils.moveToEnd(origStart, endOfFlag);
    StringBuilder actionStrings = new StringBuilder();
    Date jobPauseTime = coordJob.getPauseTime();
    Calendar pause = null;
    if (jobPauseTime != null) {
        pause = Calendar.getInstance(appTz);
        pause.setTime(DateUtils.convertDateToTimestamp(jobPauseTime));
    }
    String action = null;
    int numWaitingActions = dryrun ? 0 : jpaService.execute(new CoordActionsActiveCountJPAExecutor(coordJob.getId()));
    int maxActionToBeCreated = coordJob.getMatThrottling() - numWaitingActions;
    // If LAST_ONLY and all materialization is in the past, ignore maxActionsToBeCreated
    boolean ignoreMaxActions = (coordJob.getExecutionOrder().equals(CoordinatorJob.Execution.LAST_ONLY) || coordJob.getExecutionOrder().equals(CoordinatorJob.Execution.NONE)) && endMatdTime.before(new Date());
    LOG.debug("Coordinator job :" + coordJob.getId() + ", maxActionToBeCreated :" + maxActionToBeCreated + ", Mat_Throttle :" + coordJob.getMatThrottling() + ", numWaitingActions :" + numWaitingActions);
    boolean isCronFrequency = false;
    Calendar effStart = (Calendar) start.clone();
    try {
        int intFrequency = Integer.parseInt(coordJob.getFrequency());
        effStart = (Calendar) origStart.clone();
        effStart.add(freqTU.getCalendarUnit(), lastActionNumber * intFrequency);
    } catch (NumberFormatException e) {
        isCronFrequency = true;
    }
    boolean firstMater = true;
    end = new DaylightOffsetCalculator(startMatdTime, endMatdTime).calculate(appTz, end);
    while (effStart.compareTo(end) < 0 && (ignoreMaxActions || maxActionToBeCreated-- > 0)) {
        if (pause != null && effStart.compareTo(pause) >= 0) {
            break;
        }
        Date nextTime = effStart.getTime();
        if (isCronFrequency) {
            if (effStart.getTime().compareTo(startMatdTime) == 0 && firstMater) {
                effStart.add(Calendar.MINUTE, -1);
                firstMater = false;
            }
            nextTime = CoordCommandUtils.getNextValidActionTimeForCronFrequency(effStart.getTime(), coordJob);
            effStart.setTime(nextTime);
        }
        if (effStart.compareTo(end) < 0) {
            if (pause != null && effStart.compareTo(pause) >= 0) {
                break;
            }
            CoordinatorActionBean actionBean = new CoordinatorActionBean();
            lastActionNumber++;
            int timeout = coordJob.getTimeout();
            LOG.debug("Materializing action for time=" + DateUtils.formatDateOozieTZ(effStart.getTime()) + ", lastactionnumber=" + lastActionNumber + " timeout=" + timeout + " minutes");
            Date actualTime = new Date();
            action = CoordCommandUtils.materializeOneInstance(jobId, dryrun, (Element) eJob.clone(), nextTime, actualTime, lastActionNumber, jobConf, actionBean);
            actionBean.setTimeOut(timeout);
            if (!dryrun) {
                // Storing to table
                storeToDB(actionBean, action, jobConf);
            } else {
                actionStrings.append("action for new instance");
                actionStrings.append(action);
            }
        } else {
            break;
        }
        if (!isCronFrequency) {
            effStart = (Calendar) origStart.clone();
            effStart.add(freqTU.getCalendarUnit(), lastActionNumber * Integer.parseInt(coordJob.getFrequency()));
        }
    }
    if (isCronFrequency) {
        if (effStart.compareTo(end) < 0 && !(ignoreMaxActions || maxActionToBeCreated-- > 0)) {
            // to avoid creating duplicate actions
            if (!firstMater) {
                effStart.setTime(CoordCommandUtils.getNextValidActionTimeForCronFrequency(effStart.getTime(), coordJob));
            }
        }
    }
    endMatdTime = effStart.getTime();
    if (!dryrun) {
        return action;
    } else {
        return actionStrings.toString();
    }
}