Examples with TimeUnit org.apache.oozie.coord.TimeUnit used on opensource projects

Example 1 with TimeUnit

use of org.apache.oozie.coord.TimeUnit in project oozie by apache.

the class TestCoordMaterializeTransitionXCommand method testActionMaterEndOfWeeks.

public void testActionMaterEndOfWeeks() throws Exception {
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    String appXml = "<coordinator-app name=\"test\" frequency=\"${coord:endOfWeeks(1)}\" start=\"2016-02-03T01:00Z\" " + "end=\"2016-03-03T23:59Z\" timezone=\"UTC\" " + "xmlns=\"uri:oozie:coordinator:0.2\"> <controls> " + "<execution>LIFO</execution> </controls> <datasets> " + "<dataset name=\"a\" frequency=\"${coord:endOfWeeks(1)}\" initial-instance=\"2016-01-01T01:00Z\" " + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}") + "</uri-template>  " + "</dataset> " + "<dataset name=\"local_a\" frequency=\"${coord:endOfWeeks(1)}\" initial-instance=\"2016-01-01T01:00Z\" " + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}") + "</uri-template> " + " </dataset> " + "</datasets> <input-events> " + "<data-in name=\"A\" dataset=\"a\"> <instance>${coord:latest(0)}</instance> </data-in>  " + "</input-events> " + "<output-events> <data-out name=\"LOCAL_A\" dataset=\"local_a\"> " + "<instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> " + "<app-path>hdfs:///tmp/workflows/</app-path> " + "<configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> " + "<property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> " + "</property></configuration> </workflow> </action> </coordinator-app>";
    writeToFile(appXml, appPathFile);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    CoordinatorJobBean job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, jobId);
    assertEquals(job.getLastActionNumber(), 0);
    job.setMatThrottling(10);
    CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
    new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
    job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, job.getId());
    assertEquals(4, job.getLastActionNumber());
    String jobXml = job.getJobXml();
    Element eJob = XmlUtils.parseXml(jobXml);
    TimeZone appTz = DateUtils.getTimeZone(job.getTimeZone());
    TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration"));
    TimeUnit freqTU = TimeUnit.valueOf(job.getTimeUnitStr());
    Calendar origStart = Calendar.getInstance(appTz);
    origStart.setTime(job.getStartTimestamp());
    // Move to the End of duration, if needed.
    DateUtils.moveToEnd(origStart, endOfFlag);
    origStart.add(freqTU.getCalendarUnit(), 4 * Integer.parseInt(job.getFrequency()));
    assertEquals(origStart.getTime(), job.getNextMaterializedTime());
}

Example 2 with TimeUnit

use of org.apache.oozie.coord.TimeUnit in project oozie by apache.

the class CoordMaterializeTransitionXCommand method materializeActions.

/**
 * Create action instances starting from "startMatdTime" to "endMatdTime" and store them into coord action table.
 *
 * @param dryrun if this is a dry run
 * @throws Exception thrown if failed to materialize actions
 */
protected String materializeActions(boolean dryrun) throws Exception {
    Configuration jobConf = null;
    try {
        jobConf = new XConfiguration(new StringReader(coordJob.getConf()));
    } catch (IOException ioe) {
        LOG.warn("Configuration parse error. read from DB :" + coordJob.getConf(), ioe);
        throw new CommandException(ErrorCode.E1005, ioe.getMessage(), ioe);
    }
    String jobXml = coordJob.getJobXml();
    Element eJob = XmlUtils.parseXml(jobXml);
    TimeZone appTz = DateUtils.getTimeZone(coordJob.getTimeZone());
    String frequency = coordJob.getFrequency();
    TimeUnit freqTU = TimeUnit.valueOf(coordJob.getTimeUnitStr());
    TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration"));
    Calendar start = Calendar.getInstance(appTz);
    start.setTime(startMatdTime);
    DateUtils.moveToEnd(start, endOfFlag);
    Calendar end = Calendar.getInstance(appTz);
    end.setTime(endMatdTime);
    lastActionNumber = coordJob.getLastActionNumber();
    // Intentionally printing dates in their own timezone, not Oozie timezone
    LOG.info("materialize actions for tz=" + appTz.getDisplayName() + ",\n start=" + start.getTime() + ", end=" + end.getTime() + ",\n timeUnit " + freqTU.getCalendarUnit() + ",\n frequency :" + frequency + ":" + freqTU + ",\n lastActionNumber " + lastActionNumber);
    // Keep the actual start time
    Calendar origStart = Calendar.getInstance(appTz);
    origStart.setTime(coordJob.getStartTimestamp());
    // Move to the End of duration, if needed.
    DateUtils.moveToEnd(origStart, endOfFlag);
    StringBuilder actionStrings = new StringBuilder();
    Date jobPauseTime = coordJob.getPauseTime();
    Calendar pause = null;
    if (jobPauseTime != null) {
        pause = Calendar.getInstance(appTz);
        pause.setTime(DateUtils.convertDateToTimestamp(jobPauseTime));
    }
    String action = null;
    int numWaitingActions = dryrun ? 0 : jpaService.execute(new CoordActionsActiveCountJPAExecutor(coordJob.getId()));
    int maxActionToBeCreated = coordJob.getMatThrottling() - numWaitingActions;
    // If LAST_ONLY and all materialization is in the past, ignore maxActionsToBeCreated
    boolean ignoreMaxActions = (coordJob.getExecutionOrder().equals(CoordinatorJob.Execution.LAST_ONLY) || coordJob.getExecutionOrder().equals(CoordinatorJob.Execution.NONE)) && endMatdTime.before(new Date());
    LOG.debug("Coordinator job :" + coordJob.getId() + ", maxActionToBeCreated :" + maxActionToBeCreated + ", Mat_Throttle :" + coordJob.getMatThrottling() + ", numWaitingActions :" + numWaitingActions);
    boolean isCronFrequency = false;
    Calendar effStart = (Calendar) start.clone();
    try {
        int intFrequency = Integer.parseInt(coordJob.getFrequency());
        effStart = (Calendar) origStart.clone();
        effStart.add(freqTU.getCalendarUnit(), lastActionNumber * intFrequency);
    } catch (NumberFormatException e) {
        isCronFrequency = true;
    }
    boolean firstMater = true;
    end = new DaylightOffsetCalculator(startMatdTime, endMatdTime).calculate(appTz, end);
    while (effStart.compareTo(end) < 0 && (ignoreMaxActions || maxActionToBeCreated-- > 0)) {
        if (pause != null && effStart.compareTo(pause) >= 0) {
            break;
        }
        Date nextTime = effStart.getTime();
        if (isCronFrequency) {
            if (effStart.getTime().compareTo(startMatdTime) == 0 && firstMater) {
                effStart.add(Calendar.MINUTE, -1);
                firstMater = false;
            }
            nextTime = CoordCommandUtils.getNextValidActionTimeForCronFrequency(effStart.getTime(), coordJob);
            effStart.setTime(nextTime);
        }
        if (effStart.compareTo(end) < 0) {
            if (pause != null && effStart.compareTo(pause) >= 0) {
                break;
            }
            CoordinatorActionBean actionBean = new CoordinatorActionBean();
            lastActionNumber++;
            int timeout = coordJob.getTimeout();
            LOG.debug("Materializing action for time=" + DateUtils.formatDateOozieTZ(effStart.getTime()) + ", lastactionnumber=" + lastActionNumber + " timeout=" + timeout + " minutes");
            Date actualTime = new Date();
            action = CoordCommandUtils.materializeOneInstance(jobId, dryrun, (Element) eJob.clone(), nextTime, actualTime, lastActionNumber, jobConf, actionBean);
            actionBean.setTimeOut(timeout);
            if (!dryrun) {
                // Storing to table
                storeToDB(actionBean, action, jobConf);
            } else {
                actionStrings.append("action for new instance");
                actionStrings.append(action);
            }
        } else {
            break;
        }
        if (!isCronFrequency) {
            effStart = (Calendar) origStart.clone();
            effStart.add(freqTU.getCalendarUnit(), lastActionNumber * Integer.parseInt(coordJob.getFrequency()));
        }
    }
    if (isCronFrequency) {
        if (effStart.compareTo(end) < 0 && !(ignoreMaxActions || maxActionToBeCreated-- > 0)) {
            // to avoid creating duplicate actions
            if (!firstMater) {
                effStart.setTime(CoordCommandUtils.getNextValidActionTimeForCronFrequency(effStart.getTime(), coordJob));
            }
        }
    }
    endMatdTime = effStart.getTime();
    if (!dryrun) {
        return action;
    } else {
        return actionStrings.toString();
    }
}

Example 3 with TimeUnit

use of org.apache.oozie.coord.TimeUnit in project oozie by apache.

the class CoordCommandUtils method resolveInstanceRange.

/**
 * Resolve <start-instance> <end-insatnce> tag. Don't resolve any
 * latest()/future()
 *
 * @param event
 * @param instances
 * @param appInst
 * @param conf
 * @param eval ELEvalautor
 * @throws Exception
 */
public static void resolveInstanceRange(Element event, StringBuilder instances, SyncCoordAction appInst, Configuration conf, ELEvaluator eval) throws Exception {
    Element eStartInst = event.getChild("start-instance", event.getNamespace());
    Element eEndInst = event.getChild("end-instance", event.getNamespace());
    if (eStartInst != null && eEndInst != null) {
        String strStart = evaluateInstanceFunction(event, appInst, conf, eStartInst.getTextTrim());
        String strEnd = evaluateInstanceFunction(event, appInst, conf, eEndInst.getTextTrim());
        checkIfBothSameType(strStart, strEnd);
        // To store rest
        StringBuilder restArg = new StringBuilder();
        // arguments for
        // future
        // function
        int startIndex = getInstanceNumber(strStart, restArg);
        String startRestArg = restArg.toString();
        restArg.delete(0, restArg.length());
        int endIndex = getInstanceNumber(strEnd, restArg);
        String endRestArg = restArg.toString();
        int funcType = getFuncType(strStart);
        if (funcType == ABSOLUTE) {
            resolveAbsoluteRange(event, instances, appInst, conf, eval, strStart, endIndex, parseOneStringArg(strStart));
        } else if (funcType == ENDOFMONTHS) {
            resolveInstanceRangeEndOfDuration(TimeUnit.MONTH, event, instances, appInst, conf, eval, strStart, startIndex, endIndex);
        } else if (funcType == ENDOFWEEKS) {
            resolveInstanceRangeEndOfDuration(TimeUnit.WEEK, event, instances, appInst, conf, eval, strStart, startIndex, endIndex);
        } else if (funcType == ENDOFDAYS) {
            resolveInstanceRangeEndOfDuration(TimeUnit.DAY, event, instances, appInst, conf, eval, strStart, startIndex, endIndex);
        } else {
            if (funcType == OFFSET) {
                TimeUnit startU = TimeUnit.valueOf(startRestArg);
                TimeUnit endU = TimeUnit.valueOf(endRestArg);
                if (startU.getCalendarUnit() * startIndex > endU.getCalendarUnit() * endIndex) {
                    throw new CommandException(ErrorCode.E1010, " start-instance should be equal or earlier than the end-instance \n" + XmlUtils.prettyPrint(event));
                }
                Calendar startCal = CoordELFunctions.resolveOffsetRawTime(startIndex, startU, eval);
                Calendar endCal = CoordELFunctions.resolveOffsetRawTime(endIndex, endU, eval);
                if (startCal != null && endCal != null) {
                    List<Integer> expandedFreqs = CoordELFunctions.expandOffsetTimes(startCal, endCal, eval);
                    for (int i = expandedFreqs.size() - 1; i >= 0; i--) {
                        // we need to use DS timeout, bcz expandOffsetTimes will expand offset in Freqs in DS timeunit
                        String matInstance = materializeInstance(event, "${coord:offset(" + expandedFreqs.get(i) + ", \"" + CoordELFunctions.getDSTimeUnit(eval) + "\")}", appInst, conf, eval);
                        if (matInstance == null || matInstance.length() == 0) {
                            // Earlier than dataset's initial instance
                            break;
                        }
                        if (instances.length() > 0) {
                            instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
                        }
                        instances.append(matInstance);
                    }
                }
            } else {
                if (startIndex > endIndex) {
                    throw new CommandException(ErrorCode.E1010, " start-instance should be equal or earlier than the end-instance \n" + XmlUtils.prettyPrint(event));
                }
                if (funcType == CURRENT) {
                    // Everything could be resolved NOW. no latest() ELs
                    String matInstance = materializeInstance(event, "${coord:currentRange(" + startIndex + "," + endIndex + ")}", appInst, conf, eval);
                    if (matInstance != null && !matInstance.isEmpty()) {
                        if (instances.length() > 0) {
                            instances.append(CoordELFunctions.INSTANCE_SEPARATOR);
                        }
                        instances.append(matInstance);
                    }
                } else {
                    // latest(n)/future() EL is present
                    if (funcType == LATEST) {
                        instances.append("${coord:latestRange(").append(startIndex).append(",").append(endIndex).append(")}");
                    } else if (funcType == FUTURE) {
                        instances.append("${coord:futureRange(").append(startIndex).append(",").append(endIndex).append(",'").append(endRestArg).append("')}");
                    }
                }
            }
        }
        // Remove start-instance and end-instances
        event.removeChild("start-instance", event.getNamespace());
        event.removeChild("end-instance", event.getNamespace());
    }
}

Example 4 with TimeUnit

use of org.apache.oozie.coord.TimeUnit in project oozie by apache.

the class CoordCommandUtils method computeNextNominalTime.

/**
 * Computes the nominal time of the next action.
 * Based on CoordMaterializeTransitionXCommand#materializeActions
 *
 * The Coordinator Job needs to have the frequency, time unit, time zone, start time, end time, and job xml.
 * The Coordinator Action needs to have the nominal time and action number.
 *
 * @param coordJob The Coordinator Job
 * @param coordAction The Coordinator Action
 * @return the nominal time of the next action
 * @throws ParseException
 * @throws JDOMException
 */
public static Date computeNextNominalTime(CoordinatorJobBean coordJob, CoordinatorActionBean coordAction) throws ParseException, JDOMException {
    Date nextNominalTime;
    boolean isCronFrequency = false;
    int freq = -1;
    try {
        freq = Integer.parseInt(coordJob.getFrequency());
    } catch (NumberFormatException e) {
        isCronFrequency = true;
    }
    if (isCronFrequency) {
        nextNominalTime = CoordCommandUtils.getNextValidActionTimeForCronFrequency(coordAction.getNominalTime(), coordJob);
    } else {
        TimeZone appTz = DateUtils.getTimeZone(coordJob.getTimeZone());
        Calendar nextNominalTimeCal = Calendar.getInstance(appTz);
        nextNominalTimeCal.setTime(coordJob.getStartTimestamp());
        TimeUnit freqTU = TimeUnit.valueOf(coordJob.getTimeUnitStr());
        // Action Number is indexed by 1, so no need to +1 here
        nextNominalTimeCal.add(freqTU.getCalendarUnit(), coordAction.getActionNumber() * freq);
        String jobXml = coordJob.getJobXml();
        Element eJob = XmlUtils.parseXml(jobXml);
        TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration"));
        // Move to the End of duration, if needed.
        DateUtils.moveToEnd(nextNominalTimeCal, endOfFlag);
        nextNominalTime = nextNominalTimeCal.getTime();
    }
    // If the next nominal time is after the job's end time, then this is the last action, so return null
    if (nextNominalTime.after(coordJob.getEndTime())) {
        nextNominalTime = null;
    }
    return nextNominalTime;
}

Example 5 with TimeUnit

use of org.apache.oozie.coord.TimeUnit in project oozie by apache.

the class CoordMaterializeTransitionXCommand method getMaterializationTimeForCatchUp.

/**
 * Get materialization for window for catch-up jobs. for current jobs,it reruns currentMatdate, For catch-up, end
 * Mataterilized Time = startMatdTime + MatThrottling * frequency; unless LAST_ONLY execution order is set, in which
 * case it returns now (to materialize all actions in the past)
 *
 * @param currentMatTime
 * @return Date returns materialization for window for catch-up jobs
 * @throws CommandException
 * @throws JDOMException
 */
private Date getMaterializationTimeForCatchUp(Date currentMatTime) throws CommandException {
    if (currentMatTime.after(new Date())) {
        return currentMatTime;
    }
    if (coordJob.getExecutionOrder().equals(CoordinatorJob.Execution.LAST_ONLY) || coordJob.getExecutionOrder().equals(CoordinatorJob.Execution.NONE)) {
        return new Date();
    }
    final int frequency;
    try {
        frequency = Integer.parseInt(coordJob.getFrequency());
    } catch (final NumberFormatException e) {
        // bounded also by the throttle parameter, aka the number of coordinator actions to materialize
        return coordJob.getEndTime();
    }
    TimeZone appTz = DateUtils.getTimeZone(coordJob.getTimeZone());
    TimeUnit freqTU = TimeUnit.valueOf(coordJob.getTimeUnitStr());
    Calendar startInstance = Calendar.getInstance(appTz);
    startInstance.setTime(startMatdTime);
    Calendar endMatInstance = null;
    Calendar previousInstance = startInstance;
    for (int i = 1; i <= coordJob.getMatThrottling(); i++) {
        endMatInstance = (Calendar) startInstance.clone();
        endMatInstance.add(freqTU.getCalendarUnit(), i * frequency);
        if (endMatInstance.getTime().compareTo(new Date()) >= 0) {
            if (previousInstance.getTime().after(currentMatTime)) {
                return previousInstance.getTime();
            } else {
                return currentMatTime;
            }
        }
        previousInstance = endMatInstance;
    }
    if (endMatInstance == null) {
        return currentMatTime;
    } else {
        return endMatInstance.getTime();
    }
}