Examples with TimeUnit org.apache.oozie.coord.TimeUnit used on opensource projects

Example 6 with TimeUnit

use of org.apache.oozie.coord.TimeUnit in project oozie by apache.

the class CoordSubmitXCommand method resolveDataSets.

/**
 * Resolve datasets using job configuration.
 *
 * @param dsElems : Data set XML element.
 * @throws CoordinatorJobException thrown if failed to resolve datasets
 */
private void resolveDataSets(List<Element> dsElems) throws CoordinatorJobException {
    for (Element dsElem : dsElems) {
        // Setting up default TimeUnit and EndOFDuraion
        evalFreq.setVariable("timeunit", TimeUnit.MINUTE);
        evalFreq.setVariable("endOfDuration", TimeUnit.NONE);
        String val = resolveAttribute("frequency", dsElem, evalFreq);
        int ival = ParamChecker.checkInteger(val, "frequency");
        ParamChecker.checkGTZero(ival, "frequency");
        addAnAttribute("freq_timeunit", dsElem, evalFreq.getVariable("timeunit") == null ? TimeUnit.MINUTE.toString() : ((TimeUnit) evalFreq.getVariable("timeunit")).toString());
        addAnAttribute("end_of_duration", dsElem, evalFreq.getVariable("endOfDuration") == null ? TimeUnit.NONE.toString() : ((TimeUnit) evalFreq.getVariable("endOfDuration")).toString());
        val = resolveAttribute("initial-instance", dsElem, evalInitialInstance);
        ParamChecker.checkDateOozieTZ(val, "initial-instance");
        checkInitialInstance(val);
        val = resolveAttribute("timezone", dsElem, evalNofuncs);
        ParamChecker.checkTimeZone(val, "timezone");
        resolveTagContents("uri-template", dsElem, evalNofuncs);
        resolveTagContents("done-flag", dsElem, evalNofuncs);
    }
}

Example 7 with TimeUnit

use of org.apache.oozie.coord.TimeUnit in project oozie by apache.

the class CoordSubmitXCommand method resolveInitial.

/**
 * Resolve basic entities using job Configuration.
 *
 * @param conf :Job configuration
 * @param appXml : Original job XML
 * @param coordJob : Coordinator job bean to be populated.
 * @return Resolved job XML element.
 * @throws CoordinatorJobException thrown if failed to resolve basic entities
 * @throws Exception thrown if failed to resolve basic entities
 */
@SuppressWarnings("unchecked")
protected Element resolveInitial(Configuration conf, String appXml, CoordinatorJobBean coordJob) throws CoordinatorJobException, Exception {
    Element eAppXml = XmlUtils.parseXml(appXml);
    // job's main attributes
    // frequency
    String val = resolveAttribute("frequency", eAppXml, evalFreq);
    int ival = 0;
    val = ParamChecker.checkFrequency(val);
    coordJob.setFrequency(val);
    TimeUnit tmp = (evalFreq.getVariable("timeunit") == null) ? TimeUnit.MINUTE : ((TimeUnit) evalFreq.getVariable("timeunit"));
    try {
        Integer.parseInt(val);
    } catch (NumberFormatException ex) {
        tmp = TimeUnit.CRON;
    }
    addAnAttribute("freq_timeunit", eAppXml, tmp.toString());
    // TimeUnit
    coordJob.setTimeUnit(CoordinatorJob.Timeunit.valueOf(tmp.toString()));
    // End Of Duration
    tmp = evalFreq.getVariable("endOfDuration") == null ? TimeUnit.NONE : ((TimeUnit) evalFreq.getVariable("endOfDuration"));
    addAnAttribute("end_of_duration", eAppXml, tmp.toString());
    // Application name
    if (this.coordName == null) {
        String name = ELUtils.resolveAppName(eAppXml.getAttribute("name").getValue(), conf);
        coordJob.setAppName(name);
    } else {
        // this coord job is created from bundle
        coordJob.setAppName(this.coordName);
    }
    // start time
    val = resolveAttribute("start", eAppXml, evalNofuncs);
    ParamChecker.checkDateOozieTZ(val, "start");
    coordJob.setStartTime(DateUtils.parseDateOozieTZ(val));
    // end time
    val = resolveAttribute("end", eAppXml, evalNofuncs);
    ParamChecker.checkDateOozieTZ(val, "end");
    coordJob.setEndTime(DateUtils.parseDateOozieTZ(val));
    // Time zone
    val = resolveAttribute("timezone", eAppXml, evalNofuncs);
    ParamChecker.checkTimeZone(val, "timezone");
    coordJob.setTimeZone(val);
    // controls
    val = resolveTagContents("timeout", eAppXml.getChild("controls", eAppXml.getNamespace()), evalTimeout);
    if (val != null && val != "") {
        int t = Integer.parseInt(val);
        tmp = (evalTimeout.getVariable("timeunit") == null) ? TimeUnit.MINUTE : ((TimeUnit) evalTimeout.getVariable("timeunit"));
        switch(tmp) {
            case HOUR:
                val = String.valueOf(t * 60);
                break;
            case DAY:
                val = String.valueOf(t * 60 * 24);
                break;
            case MONTH:
                val = String.valueOf(t * 60 * 24 * 30);
                break;
            default:
                break;
        }
    } else {
        val = ConfigurationService.get(CONF_DEFAULT_TIMEOUT_NORMAL);
    }
    ival = ParamChecker.checkInteger(val, "timeout");
    if (ival < 0 || ival > ConfigurationService.getInt(CONF_DEFAULT_MAX_TIMEOUT)) {
        ival = ConfigurationService.getInt(CONF_DEFAULT_MAX_TIMEOUT);
    }
    coordJob.setTimeout(ival);
    val = resolveTagContents("concurrency", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
    if (val == null || val.isEmpty()) {
        val = ConfigurationService.get(CONF_DEFAULT_CONCURRENCY);
    }
    ival = ParamChecker.checkInteger(val, "concurrency");
    coordJob.setConcurrency(ival);
    val = resolveTagContents("throttle", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
    if (val == null || val.isEmpty()) {
        int defaultThrottle = ConfigurationService.getInt(CONF_DEFAULT_THROTTLE);
        ival = defaultThrottle;
    } else {
        ival = ParamChecker.checkInteger(val, "throttle");
    }
    int maxQueue = ConfigurationService.getInt(CONF_QUEUE_SIZE);
    float factor = ConfigurationService.getFloat(CONF_MAT_THROTTLING_FACTOR);
    int maxThrottle = (int) (maxQueue * factor);
    if (ival > maxThrottle || ival < 1) {
        ival = maxThrottle;
    }
    LOG.debug("max throttle " + ival);
    coordJob.setMatThrottling(ival);
    val = resolveTagContents("execution", eAppXml.getChild("controls", eAppXml.getNamespace()), evalNofuncs);
    if (val == "") {
        val = Execution.FIFO.toString();
    }
    coordJob.setExecutionOrder(Execution.valueOf(val));
    String[] acceptedVals = { Execution.LIFO.toString(), Execution.FIFO.toString(), Execution.LAST_ONLY.toString(), Execution.NONE.toString() };
    ParamChecker.isMember(val, acceptedVals, "execution");
    // datasets
    resolveTagContents("include", eAppXml.getChild("datasets", eAppXml.getNamespace()), evalNofuncs);
    // for each data set
    resolveDataSets(eAppXml);
    HashMap<String, String> dataNameList = new HashMap<String, String>();
    resolveIODataset(eAppXml);
    resolveIOEvents(eAppXml, dataNameList);
    if (CoordUtils.isInputLogicSpecified(eAppXml)) {
        resolveInputLogic(eAppXml.getChild(CoordInputLogicEvaluator.INPUT_LOGIC, eAppXml.getNamespace()), evalInst, dataNameList);
    }
    resolveTagContents("app-path", eAppXml.getChild("action", eAppXml.getNamespace()).getChild("workflow", eAppXml.getNamespace()), evalNofuncs);
    // TODO: If action or workflow tag is missing, NullPointerException will
    // occur
    Element configElem = eAppXml.getChild("action", eAppXml.getNamespace()).getChild("workflow", eAppXml.getNamespace()).getChild("configuration", eAppXml.getNamespace());
    evalData = CoordELEvaluator.createELEvaluatorForDataEcho(conf, "coord-job-submit-data", dataNameList);
    if (configElem != null) {
        for (Element propElem : (List<Element>) configElem.getChildren("property", configElem.getNamespace())) {
            resolveTagContents("name", propElem, evalData);
            // Want to check the data-integrity but don't want to modify the
            // XML
            // for properties only
            Element tmpProp = (Element) propElem.clone();
            resolveTagContents("value", tmpProp, evalData);
        }
    }
    evalSla = CoordELEvaluator.createELEvaluatorForDataAndConf(conf, "coord-sla-submit", dataNameList);
    resolveSLA(eAppXml, coordJob);
    return eAppXml;
}

Example 8 with TimeUnit

use of org.apache.oozie.coord.TimeUnit in project oozie by apache.

the class TestCoordMaterializeTransitionXCommand method testMaterizationEndOfMonths.

public void testMaterizationEndOfMonths() throws Exception {
    Configuration conf = new XConfiguration();
    File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
    String appXml = "<coordinator-app name=\"test\" frequency=\"${coord:endOfMonths(1)}\" start=\"2009-02-01T01:00Z\" " + "end=\"2009-02-03T23:59Z\" timezone=\"UTC\" " + "xmlns=\"uri:oozie:coordinator:0.2\"> <controls> " + "<execution>LIFO</execution> </controls> <datasets> " + "<dataset name=\"a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\" " + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}") + "</uri-template>  " + "</dataset> " + "<dataset name=\"local_a\" frequency=\"${coord:days(7)}\" initial-instance=\"2009-02-01T01:00Z\" " + "timezone=\"UTC\"> <uri-template>" + getTestCaseFileUri("coord/workflows/${YEAR}/${DAY}") + "</uri-template> " + " </dataset> " + "</datasets> <input-events> " + "<data-in name=\"A\" dataset=\"a\"> <instance>${coord:latest(0)}</instance> </data-in>  " + "</input-events> " + "<output-events> <data-out name=\"LOCAL_A\" dataset=\"local_a\"> " + "<instance>${coord:current(-1)}</instance> </data-out> </output-events> <action> <workflow> " + "<app-path>hdfs:///tmp/workflows/</app-path> " + "<configuration> <property> <name>inputA</name> <value>${coord:dataIn('A')}</value> </property> " + "<property> <name>inputB</name> <value>${coord:dataOut('LOCAL_A')}</value> " + "</property></configuration> </workflow> </action> </coordinator-app>";
    writeToFile(appXml, appPathFile);
    conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
    String jobId = sc.call();
    Calendar cal = Calendar.getInstance(DateUtils.getOozieProcessingTimeZone());
    cal.add(Calendar.MONTH, -3);
    Date startTime = cal.getTime();
    cal = Calendar.getInstance(DateUtils.getOozieProcessingTimeZone());
    cal.add(Calendar.MONTH, 3);
    Date endTime = cal.getTime();
    CoordinatorJobBean job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, jobId);
    assertEquals(job.getLastActionNumber(), 0);
    job.setStartTime(startTime);
    job.setEndTime(endTime);
    job.setMatThrottling(10);
    CoordJobQueryExecutor.getInstance().executeUpdate(CoordJobQuery.UPDATE_COORD_JOB, job);
    new CoordMaterializeTransitionXCommand(job.getId(), hoursToSeconds(1)).call();
    job = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, job.getId());
    assertEquals(job.getLastActionNumber(), 3);
    String jobXml = job.getJobXml();
    Element eJob = XmlUtils.parseXml(jobXml);
    TimeZone appTz = DateUtils.getTimeZone(job.getTimeZone());
    TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration"));
    TimeUnit freqTU = TimeUnit.valueOf(job.getTimeUnitStr());
    Calendar origStart = Calendar.getInstance(appTz);
    origStart.setTime(job.getStartTimestamp());
    // Move to the End of duration, if needed.
    DateUtils.moveToEnd(origStart, endOfFlag);
    origStart.add(freqTU.getCalendarUnit(), 3 * Integer.parseInt(job.getFrequency()));
    assertEquals(job.getNextMaterializedTime(), origStart.getTime());
}