Examples with Context org.apache.oozie.dependency.URIHandler.Context used on opensource projects

Example 1 with Context

use of org.apache.oozie.dependency.URIHandler.Context in project oozie by apache.

the class CoordRerunXCommand method rerunChildren.

@Override
public void rerunChildren() throws CommandException {
    boolean isError = false;
    try {
        CoordinatorActionInfo coordInfo = null;
        InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
        List<CoordinatorActionBean> coordActions = CoordUtils.getCoordActions(rerunType, jobId, scope, false);
        if (checkAllActionsRunnable(coordActions)) {
            Map<String, Context> uriHandlerContextMap = new HashMap<String, Context>();
            Configuration coordJobConf = null;
            try {
                coordJobConf = new XConfiguration(new StringReader(coordJob.getConf()));
            } catch (IOException e) {
                throw new CommandException(ErrorCode.E0907, "failed to read coord job conf to clean up output data");
            }
            try {
                for (CoordinatorActionBean coordAction : coordActions) {
                    String actionXml = coordAction.getActionXml();
                    // Cleanup activity should not run when failed option has been provided
                    if (!noCleanup && !failed) {
                        Element eAction = XmlUtils.parseXml(actionXml);
                        cleanupOutputEvents(eAction, coordJobConf, uriHandlerContextMap);
                    }
                    if (refresh) {
                        refreshAction(coordJob, coordAction);
                    }
                    updateAction(coordJob, coordAction);
                    if (SLAService.isEnabled()) {
                        SLAOperations.updateRegistrationEvent(coordAction.getId());
                    }
                    queue(new CoordActionNotificationXCommand(coordAction), 100);
                    queue(new CoordActionInputCheckXCommand(coordAction.getId(), coordAction.getJobId()), 100);
                    if (coordAction.getPushMissingDependencies() != null) {
                        queue(new CoordPushDependencyCheckXCommand(coordAction.getId(), true), 100);
                    }
                }
            } finally {
                Iterator<Entry<String, Context>> itr = uriHandlerContextMap.entrySet().iterator();
                while (itr.hasNext()) {
                    Entry<String, Context> entry = itr.next();
                    entry.getValue().destroy();
                    itr.remove();
                }
            }
        } else {
            isError = true;
            throw new CommandException(ErrorCode.E1018, "part or all actions are not eligible to rerun!");
        }
        coordInfo = new CoordinatorActionInfo(coordActions);
        ret = coordInfo;
    } catch (XException xex) {
        isError = true;
        throw new CommandException(xex);
    } catch (JDOMException jex) {
        isError = true;
        throw new CommandException(ErrorCode.E0700, jex.getMessage(), jex);
    } catch (Exception ex) {
        isError = true;
        throw new CommandException(ErrorCode.E1018, ex.getMessage(), ex);
    } finally {
        if (isError) {
            transitToPrevious();
        }
    }
}

Example 2 with Context

use of org.apache.oozie.dependency.URIHandler.Context in project oozie by apache.

the class CoordELFunctions method coord_latestRange_sync.

private static String coord_latestRange_sync(int startOffset, int endOffset) throws Exception {
    final XLog LOG = XLog.getLog(CoordELFunctions.class);
    final Thread currentThread = Thread.currentThread();
    ELEvaluator eval = ELEvaluator.getCurrent();
    String retVal = "";
    // in minutes
    int datasetFrequency = (int) getDSFrequency();
    TimeUnit dsTimeUnit = getDSTimeUnit();
    int[] instCount = new int[1];
    boolean useCurrentTime = Services.get().getConf().getBoolean(LATEST_EL_USE_CURRENT_TIME, false);
    Calendar nominalInstanceCal;
    if (useCurrentTime) {
        nominalInstanceCal = getCurrentInstance(new Date(), instCount);
    } else {
        nominalInstanceCal = getCurrentInstance(getActualTime(), instCount);
    }
    StringBuilder resolvedInstances = new StringBuilder();
    StringBuilder resolvedURIPaths = new StringBuilder();
    if (nominalInstanceCal != null) {
        Calendar initInstance = getInitialInstanceCal();
        SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
        if (ds == null) {
            throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
        }
        String uriTemplate = ds.getUriTemplate();
        Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
        if (conf == null) {
            throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
        }
        int available = 0;
        boolean resolved = false;
        String user = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
        String doneFlag = ds.getDoneFlag();
        URIHandlerService uriService = Services.get().get(URIHandlerService.class);
        URIHandler uriHandler = null;
        Context uriContext = null;
        try {
            while (nominalInstanceCal.compareTo(initInstance) >= 0 && !currentThread.isInterrupted()) {
                ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
                String uriPath = uriEval.evaluate(uriTemplate, String.class);
                if (uriHandler == null) {
                    URI uri = new URI(uriPath);
                    uriHandler = uriService.getURIHandler(uri);
                    uriContext = uriHandler.getContext(uri, conf, user, true);
                }
                String uriWithDoneFlag = uriHandler.getURIWithDoneFlag(uriPath, doneFlag);
                if (uriHandler.exists(new URI(uriWithDoneFlag), uriContext)) {
                    XLog.getLog(CoordELFunctions.class).debug("Found latest(" + available + "): " + uriWithDoneFlag);
                    if (available == startOffset) {
                        LOG.debug("Matched latest(" + available + "): " + uriWithDoneFlag);
                        resolved = true;
                        resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal));
                        resolvedURIPaths.append(uriPath);
                        retVal = resolvedInstances.toString();
                        eval.setVariable(CoordELConstants.RESOLVED_PATH, resolvedURIPaths.toString());
                        break;
                    } else if (available <= endOffset) {
                        LOG.debug("Matched latest(" + available + "): " + uriWithDoneFlag);
                        resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal)).append(INSTANCE_SEPARATOR);
                        resolvedURIPaths.append(uriPath).append(INSTANCE_SEPARATOR);
                    }
                    available--;
                }
                // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), -datasetFrequency);
                nominalInstanceCal = (Calendar) initInstance.clone();
                instCount[0]--;
                nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
            // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
            }
            if (!StringUtils.isEmpty(resolvedURIPaths.toString()) && eval.getVariable(CoordELConstants.RESOLVED_PATH) == null) {
                eval.setVariable(CoordELConstants.RESOLVED_PATH, resolvedURIPaths.toString());
            }
        } finally {
            if (uriContext != null) {
                uriContext.destroy();
            }
        }
        if (!resolved) {
            // return unchanged latest function with variable 'is_resolved'
            // to 'false'
            eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.FALSE);
            if (startOffset == endOffset) {
                retVal = "${coord:latest(" + startOffset + ")}";
            } else {
                retVal = "${coord:latestRange(" + startOffset + "," + endOffset + ")}";
            }
        } else {
            eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.TRUE);
        }
    } else {
        // No feasible nominal time
        eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.FALSE);
    }
    return retVal;
}

Example 3 with Context

use of org.apache.oozie.dependency.URIHandler.Context in project oozie by apache.

the class CoordRerunXCommand method cleanupOutputEvents.

/**
 * Cleanup output-events directories
 *
 * @param eAction coordinator action xml
 */
@SuppressWarnings("unchecked")
private void cleanupOutputEvents(Element eAction, Configuration coordJobConf, Map<String, Context> uriHandlerContextMap) throws CommandException {
    Element outputList = eAction.getChild("output-events", eAction.getNamespace());
    if (outputList != null) {
        for (Element data : (List<Element>) outputList.getChildren("data-out", eAction.getNamespace())) {
            String nocleanup = data.getAttributeValue("nocleanup");
            if (data.getChild("uris", data.getNamespace()) != null && (nocleanup == null || !nocleanup.equals("true"))) {
                String uris = data.getChild("uris", data.getNamespace()).getTextTrim();
                if (uris != null) {
                    String[] uriArr = uris.split(CoordELFunctions.INSTANCE_SEPARATOR);
                    try {
                        for (String uriStr : uriArr) {
                            URI uri = new URI(uriStr);
                            URIHandler handler = Services.get().get(URIHandlerService.class).getURIHandler(uri);
                            String schemeWithAuthority = uri.getScheme() + "://" + uri.getAuthority();
                            if (!uriHandlerContextMap.containsKey(schemeWithAuthority)) {
                                Context context = handler.getContext(uri, coordJobConf, coordJob.getUser(), false);
                                uriHandlerContextMap.put(schemeWithAuthority, context);
                            }
                            handler.delete(uri, uriHandlerContextMap.get(schemeWithAuthority));
                            LOG.info("Cleanup the output data " + uri.toString());
                        }
                    } catch (URISyntaxException e) {
                        throw new CommandException(ErrorCode.E0907, e.getMessage());
                    } catch (URIHandlerException e) {
                        throw new CommandException(ErrorCode.E0907, e.getMessage());
                    }
                }
            }
        }
    } else {
        LOG.info("No output-events defined in coordinator xml. Therefore nothing to cleanup");
    }
}

Example 4 with Context

use of org.apache.oozie.dependency.URIHandler.Context in project oozie by apache.

the class CoordELFunctions method coord_futureRange_sync.

private static String coord_futureRange_sync(int startOffset, int endOffset, int instance) throws Exception {
    final XLog LOG = XLog.getLog(CoordELFunctions.class);
    final Thread currentThread = Thread.currentThread();
    ELEvaluator eval = ELEvaluator.getCurrent();
    String retVal = "";
    // in minutes
    int datasetFrequency = (int) getDSFrequency();
    TimeUnit dsTimeUnit = getDSTimeUnit();
    int[] instCount = new int[1];
    Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
    StringBuilder resolvedInstances = new StringBuilder();
    StringBuilder resolvedURIPaths = new StringBuilder();
    if (nominalInstanceCal != null) {
        Calendar initInstance = getInitialInstanceCal();
        nominalInstanceCal = (Calendar) initInstance.clone();
        nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
        SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
        if (ds == null) {
            throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
        }
        String uriTemplate = ds.getUriTemplate();
        Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
        if (conf == null) {
            throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
        }
        int available = 0, checkedInstance = 0;
        boolean resolved = false;
        String user = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
        String doneFlag = ds.getDoneFlag();
        URIHandlerService uriService = Services.get().get(URIHandlerService.class);
        URIHandler uriHandler = null;
        Context uriContext = null;
        try {
            while (instance >= checkedInstance && !currentThread.isInterrupted()) {
                ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
                String uriPath = uriEval.evaluate(uriTemplate, String.class);
                if (uriHandler == null) {
                    URI uri = new URI(uriPath);
                    uriHandler = uriService.getURIHandler(uri);
                    uriContext = uriHandler.getContext(uri, conf, user, true);
                }
                String uriWithDoneFlag = uriHandler.getURIWithDoneFlag(uriPath, doneFlag);
                if (uriHandler.exists(new URI(uriWithDoneFlag), uriContext)) {
                    if (available == endOffset) {
                        LOG.debug("Matched future(" + available + "): " + uriWithDoneFlag);
                        resolved = true;
                        resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal));
                        resolvedURIPaths.append(uriPath);
                        retVal = resolvedInstances.toString();
                        eval.setVariable(CoordELConstants.RESOLVED_PATH, resolvedURIPaths.toString());
                        break;
                    } else if (available >= startOffset) {
                        LOG.debug("Matched future(" + available + "): " + uriWithDoneFlag);
                        resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal)).append(INSTANCE_SEPARATOR);
                        resolvedURIPaths.append(uriPath).append(INSTANCE_SEPARATOR);
                    }
                    available++;
                }
                // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency);
                nominalInstanceCal = (Calendar) initInstance.clone();
                instCount[0]++;
                nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
                checkedInstance++;
            // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
            }
            if (!StringUtils.isEmpty(resolvedURIPaths.toString()) && eval.getVariable(CoordELConstants.RESOLVED_PATH) == null) {
                eval.setVariable(CoordELConstants.RESOLVED_PATH, resolvedURIPaths.toString());
            }
        } finally {
            if (uriContext != null) {
                uriContext.destroy();
            }
        }
        if (!resolved) {
            // return unchanged future function with variable 'is_resolved'
            // to 'false'
            eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.FALSE);
            if (startOffset == endOffset) {
                retVal = "${coord:future(" + startOffset + ", " + instance + ")}";
            } else {
                retVal = "${coord:futureRange(" + startOffset + ", " + endOffset + ", " + instance + ")}";
            }
        } else {
            eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.TRUE);
        }
    } else {
        // No feasible nominal time
        eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.TRUE);
        retVal = "";
    }
    return retVal;
}