Search in sources :

Example 86 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class XDataTestCase method createBundleJobNegative.

/**
 * Create bundle job that contains bad coordinator jobs
 *
 * @param jobStatus
 * @return bundle job bean
 * @throws Exception
 */
protected BundleJobBean createBundleJobNegative(Job.Status jobStatus) throws Exception {
    Path coordPath1 = new Path(getFsTestCaseDir(), "coord1");
    Path coordPath2 = new Path(getFsTestCaseDir(), "coord2");
    writeCoordXml(coordPath1, "coord-job-bundle-negative.xml");
    writeCoordXml(coordPath2, "coord-job-bundle-negative.xml");
    Path bundleAppPath = new Path(getFsTestCaseDir(), "bundle");
    String bundleAppXml = getBundleXml("bundle-submit-job.xml");
    bundleAppXml = bundleAppXml.replaceAll("#app_path1", new Path(coordPath1.toString(), "coordinator.xml").toString());
    bundleAppXml = bundleAppXml.replaceAll("#app_path2", new Path(coordPath1.toString(), "coordinator.xml").toString());
    writeToFile(bundleAppXml, bundleAppPath, "bundle.xml");
    Configuration conf = new XConfiguration();
    conf.set(OozieClient.BUNDLE_APP_PATH, bundleAppPath.toString());
    conf.set(OozieClient.USER_NAME, getTestUser());
    conf.set("jobTracker", getJobTrackerUri());
    conf.set("nameNode", getNameNodeUri());
    conf.set("coordName1", "coord1");
    conf.set("coordName2", "coord2");
    conf.set("coord1.starttime", "2009-02-01T00:00Z");
    conf.set("isEnabled", "true");
    BundleJobBean bundle = new BundleJobBean();
    bundle.setId(Services.get().get(UUIDService.class).generateId(ApplicationType.BUNDLE));
    bundle.setAppName("BUNDLE-TEST");
    bundle.setAppPath(bundleAppPath.toString());
    bundle.setConf(XmlUtils.prettyPrint(conf).toString());
    bundle.setConsoleUrl("consoleUrl");
    bundle.setCreatedTime(new Date());
    bundle.setJobXml(bundleAppXml);
    bundle.setLastModifiedTime(new Date());
    bundle.setOrigJobXml(bundleAppXml);
    bundle.resetPending();
    bundle.setStatus(jobStatus);
    bundle.setUser(conf.get(OozieClient.USER_NAME));
    bundle.setGroup(conf.get(OozieClient.GROUP_NAME));
    return bundle;
}
Also used : Path(org.apache.hadoop.fs.Path) XConfiguration(org.apache.oozie.util.XConfiguration) Configuration(org.apache.hadoop.conf.Configuration) XConfiguration(org.apache.oozie.util.XConfiguration) BundleJobBean(org.apache.oozie.BundleJobBean) Date(java.util.Date)

Example 87 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class XDataTestCase method getCoordConf.

/**
 * Get coordinator configuration
 *
 * @param appPath application path
 * @return coordinator configuration
 * @throws IOException thrown if unable to get coord conf
 */
protected Configuration getCoordConf(Path appPath) throws IOException {
    Path wfAppPath = new Path(getFsTestCaseDir(), "coord");
    Configuration jobConf = new XConfiguration();
    jobConf.set(OozieClient.COORDINATOR_APP_PATH, appPath.toString());
    jobConf.set(OozieClient.USER_NAME, getTestUser());
    jobConf.set("jobTracker", getJobTrackerUri());
    jobConf.set("nameNode", getNameNodeUri());
    jobConf.set("wfAppPath", wfAppPath.toString());
    String content = "<workflow-app xmlns='uri:oozie:workflow:0.1'  xmlns:sla='uri:oozie:sla:0.1' name='no-op-wf'>";
    content += "<start to='end' />";
    content += "<end name='end' /></workflow-app>";
    writeToFile(content, wfAppPath, "workflow.xml");
    return jobConf;
}
Also used : Path(org.apache.hadoop.fs.Path) XConfiguration(org.apache.oozie.util.XConfiguration) Configuration(org.apache.hadoop.conf.Configuration) XConfiguration(org.apache.oozie.util.XConfiguration)

Example 88 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class CoordMaterializeTransitionXCommand method materializeActions.

/**
 * Create action instances starting from "startMatdTime" to "endMatdTime" and store them into coord action table.
 *
 * @param dryrun if this is a dry run
 * @throws Exception thrown if failed to materialize actions
 */
protected String materializeActions(boolean dryrun) throws Exception {
    Configuration jobConf = null;
    try {
        jobConf = new XConfiguration(new StringReader(coordJob.getConf()));
    } catch (IOException ioe) {
        LOG.warn("Configuration parse error. read from DB :" + coordJob.getConf(), ioe);
        throw new CommandException(ErrorCode.E1005, ioe.getMessage(), ioe);
    }
    String jobXml = coordJob.getJobXml();
    Element eJob = XmlUtils.parseXml(jobXml);
    TimeZone appTz = DateUtils.getTimeZone(coordJob.getTimeZone());
    String frequency = coordJob.getFrequency();
    TimeUnit freqTU = TimeUnit.valueOf(coordJob.getTimeUnitStr());
    TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration"));
    Calendar start = Calendar.getInstance(appTz);
    start.setTime(startMatdTime);
    DateUtils.moveToEnd(start, endOfFlag);
    Calendar end = Calendar.getInstance(appTz);
    end.setTime(endMatdTime);
    lastActionNumber = coordJob.getLastActionNumber();
    // Intentionally printing dates in their own timezone, not Oozie timezone
    LOG.info("materialize actions for tz=" + appTz.getDisplayName() + ",\n start=" + start.getTime() + ", end=" + end.getTime() + ",\n timeUnit " + freqTU.getCalendarUnit() + ",\n frequency :" + frequency + ":" + freqTU + ",\n lastActionNumber " + lastActionNumber);
    // Keep the actual start time
    Calendar origStart = Calendar.getInstance(appTz);
    origStart.setTime(coordJob.getStartTimestamp());
    // Move to the End of duration, if needed.
    DateUtils.moveToEnd(origStart, endOfFlag);
    StringBuilder actionStrings = new StringBuilder();
    Date jobPauseTime = coordJob.getPauseTime();
    Calendar pause = null;
    if (jobPauseTime != null) {
        pause = Calendar.getInstance(appTz);
        pause.setTime(DateUtils.convertDateToTimestamp(jobPauseTime));
    }
    String action = null;
    int numWaitingActions = dryrun ? 0 : jpaService.execute(new CoordActionsActiveCountJPAExecutor(coordJob.getId()));
    int maxActionToBeCreated = coordJob.getMatThrottling() - numWaitingActions;
    // If LAST_ONLY and all materialization is in the past, ignore maxActionsToBeCreated
    boolean ignoreMaxActions = (coordJob.getExecutionOrder().equals(CoordinatorJob.Execution.LAST_ONLY) || coordJob.getExecutionOrder().equals(CoordinatorJob.Execution.NONE)) && endMatdTime.before(new Date());
    LOG.debug("Coordinator job :" + coordJob.getId() + ", maxActionToBeCreated :" + maxActionToBeCreated + ", Mat_Throttle :" + coordJob.getMatThrottling() + ", numWaitingActions :" + numWaitingActions);
    boolean isCronFrequency = false;
    Calendar effStart = (Calendar) start.clone();
    try {
        int intFrequency = Integer.parseInt(coordJob.getFrequency());
        effStart = (Calendar) origStart.clone();
        effStart.add(freqTU.getCalendarUnit(), lastActionNumber * intFrequency);
    } catch (NumberFormatException e) {
        isCronFrequency = true;
    }
    boolean firstMater = true;
    end = new DaylightOffsetCalculator(startMatdTime, endMatdTime).calculate(appTz, end);
    while (effStart.compareTo(end) < 0 && (ignoreMaxActions || maxActionToBeCreated-- > 0)) {
        if (pause != null && effStart.compareTo(pause) >= 0) {
            break;
        }
        Date nextTime = effStart.getTime();
        if (isCronFrequency) {
            if (effStart.getTime().compareTo(startMatdTime) == 0 && firstMater) {
                effStart.add(Calendar.MINUTE, -1);
                firstMater = false;
            }
            nextTime = CoordCommandUtils.getNextValidActionTimeForCronFrequency(effStart.getTime(), coordJob);
            effStart.setTime(nextTime);
        }
        if (effStart.compareTo(end) < 0) {
            if (pause != null && effStart.compareTo(pause) >= 0) {
                break;
            }
            CoordinatorActionBean actionBean = new CoordinatorActionBean();
            lastActionNumber++;
            int timeout = coordJob.getTimeout();
            LOG.debug("Materializing action for time=" + DateUtils.formatDateOozieTZ(effStart.getTime()) + ", lastactionnumber=" + lastActionNumber + " timeout=" + timeout + " minutes");
            Date actualTime = new Date();
            action = CoordCommandUtils.materializeOneInstance(jobId, dryrun, (Element) eJob.clone(), nextTime, actualTime, lastActionNumber, jobConf, actionBean);
            actionBean.setTimeOut(timeout);
            if (!dryrun) {
                // Storing to table
                storeToDB(actionBean, action, jobConf);
            } else {
                actionStrings.append("action for new instance");
                actionStrings.append(action);
            }
        } else {
            break;
        }
        if (!isCronFrequency) {
            effStart = (Calendar) origStart.clone();
            effStart.add(freqTU.getCalendarUnit(), lastActionNumber * Integer.parseInt(coordJob.getFrequency()));
        }
    }
    if (isCronFrequency) {
        if (effStart.compareTo(end) < 0 && !(ignoreMaxActions || maxActionToBeCreated-- > 0)) {
            // to avoid creating duplicate actions
            if (!firstMater) {
                effStart.setTime(CoordCommandUtils.getNextValidActionTimeForCronFrequency(effStart.getTime(), coordJob));
            }
        }
    }
    endMatdTime = effStart.getTime();
    if (!dryrun) {
        return action;
    } else {
        return actionStrings.toString();
    }
}
Also used : XConfiguration(org.apache.oozie.util.XConfiguration) Configuration(org.apache.hadoop.conf.Configuration) CoordinatorActionBean(org.apache.oozie.CoordinatorActionBean) Element(org.jdom.Element) Calendar(java.util.Calendar) IOException(java.io.IOException) CommandException(org.apache.oozie.command.CommandException) Date(java.util.Date) CoordActionsActiveCountJPAExecutor(org.apache.oozie.executor.jpa.CoordActionsActiveCountJPAExecutor) XConfiguration(org.apache.oozie.util.XConfiguration) TimeZone(java.util.TimeZone) StringReader(java.io.StringReader) TimeUnit(org.apache.oozie.coord.TimeUnit)

Example 89 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class CoordSLAAlertsEnableXCommand method updateJob.

@Override
protected void updateJob() throws CommandException {
    XConfiguration conf = new XConfiguration();
    if (isJobRequest()) {
        conf.set(OozieClient.SLA_DISABLE_ALERT, "");
        LOG.debug("Updating job property " + OozieClient.SLA_DISABLE_ALERT + " = ");
    } else {
        conf.set(OozieClient.SLA_ENABLE_ALERT, getActionDateListAsString());
        LOG.debug("Updating job property " + OozieClient.SLA_DISABLE_ALERT + " = " + getActionDateListAsString());
    }
    updateJobConf(conf);
}
Also used : XConfiguration(org.apache.oozie.util.XConfiguration)

Example 90 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class CoordUpdateXCommand method loadState.

@Override
protected void loadState() throws CommandException {
    super.loadState();
    jpaService = Services.get().get(JPAService.class);
    if (jpaService == null) {
        throw new CommandException(ErrorCode.E0610);
    }
    coordJob = new CoordinatorJobBean();
    try {
        oldCoordJob = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, jobId);
    } catch (JPAExecutorException e) {
        throw new CommandException(e);
    }
    LogUtils.setLogInfo(oldCoordJob);
    if (!isConfChange || StringUtils.isEmpty(conf.get(OozieClient.COORDINATOR_APP_PATH))) {
        try {
            XConfiguration jobConf = new XConfiguration(new StringReader(oldCoordJob.getConf()));
            if (!isConfChange) {
                conf = jobConf;
            } else {
                for (String bundleConfKey : bundleConfList) {
                    if (jobConf.get(bundleConfKey) != null) {
                        conf.set(bundleConfKey, jobConf.get(bundleConfKey));
                    }
                }
                if (StringUtils.isEmpty(conf.get(OozieClient.COORDINATOR_APP_PATH))) {
                    conf.set(OozieClient.COORDINATOR_APP_PATH, jobConf.get(OozieClient.COORDINATOR_APP_PATH));
                }
            }
        } catch (Exception e) {
            throw new CommandException(ErrorCode.E1023, e.getMessage(), e);
        }
    }
    coordJob.setConf(XmlUtils.prettyPrint(conf).toString());
    setJob(coordJob);
    LogUtils.setLogInfo(coordJob);
}
Also used : CoordinatorJobBean(org.apache.oozie.CoordinatorJobBean) JPAExecutorException(org.apache.oozie.executor.jpa.JPAExecutorException) XConfiguration(org.apache.oozie.util.XConfiguration) StringReader(java.io.StringReader) CommandException(org.apache.oozie.command.CommandException) JPAService(org.apache.oozie.service.JPAService) JPAExecutorException(org.apache.oozie.executor.jpa.JPAExecutorException) IOException(java.io.IOException) CommandException(org.apache.oozie.command.CommandException)

Aggregations

XConfiguration (org.apache.oozie.util.XConfiguration)373 Configuration (org.apache.hadoop.conf.Configuration)241 Path (org.apache.hadoop.fs.Path)106 StringReader (java.io.StringReader)97 File (java.io.File)92 IOException (java.io.IOException)78 WorkflowJobBean (org.apache.oozie.WorkflowJobBean)75 WorkflowActionBean (org.apache.oozie.WorkflowActionBean)70 CommandException (org.apache.oozie.command.CommandException)68 Element (org.jdom.Element)66 Writer (java.io.Writer)58 Date (java.util.Date)50 FileSystem (org.apache.hadoop.fs.FileSystem)48 FileWriter (java.io.FileWriter)45 Reader (java.io.Reader)43 CoordinatorActionBean (org.apache.oozie.CoordinatorActionBean)37 CoordinatorJobBean (org.apache.oozie.CoordinatorJobBean)36 JPAExecutorException (org.apache.oozie.executor.jpa.JPAExecutorException)28 OutputStream (java.io.OutputStream)27 FileOutputStream (java.io.FileOutputStream)25