Examples with ActionDependency org.apache.oozie.dependency.ActionDependency used on opensource projects

Example 1 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class CoordCommandUtils method dryRunCoord.

/**
 * @param eAction the actionXml related element
 * @param actionBean the coordinator action bean
 * @return actionXml returns actionXml as String
 * @throws Exception
 */
static String dryRunCoord(Element eAction, CoordinatorActionBean actionBean) throws Exception {
    String action = XmlUtils.prettyPrint(eAction).toString();
    StringBuilder actionXml = new StringBuilder(action);
    Configuration actionConf = new XConfiguration(new StringReader(actionBean.getRunConf()));
    actionBean.setActionXml(action);
    if (CoordUtils.isInputLogicSpecified(eAction)) {
        new CoordInputLogicEvaluatorUtil(actionBean).validateInputLogic();
    }
    boolean isPushDepAvailable = true;
    String pushMissingDependencies = actionBean.getPushInputDependencies().getMissingDependencies();
    if (pushMissingDependencies != null) {
        ActionDependency actionDependencies = DependencyChecker.checkForAvailability(pushMissingDependencies, actionConf, true);
        if (actionDependencies.getMissingDependencies().size() != 0) {
            isPushDepAvailable = false;
        }
    }
    boolean isPullDepAvailable = true;
    CoordActionInputCheckXCommand coordActionInput = new CoordActionInputCheckXCommand(actionBean.getId(), actionBean.getJobId());
    if (actionBean.getMissingDependencies() != null) {
        StringBuilder existList = new StringBuilder();
        StringBuilder nonExistList = new StringBuilder();
        StringBuilder nonResolvedList = new StringBuilder();
        getResolvedList(actionBean.getPullInputDependencies().getMissingDependencies(), nonExistList, nonResolvedList);
        isPullDepAvailable = actionBean.getPullInputDependencies().checkPullMissingDependencies(actionBean, existList, nonExistList);
    }
    if (isPullDepAvailable && isPushDepAvailable) {
        // Check for latest/future
        boolean isLatestFutureDepAvailable = coordActionInput.checkUnResolvedInput(actionBean, actionXml, actionConf);
        if (isLatestFutureDepAvailable) {
            String newActionXml = CoordActionInputCheckXCommand.resolveCoordConfiguration(actionXml, actionConf, actionBean.getId());
            actionXml.replace(0, actionXml.length(), newActionXml);
        }
    }
    return actionXml.toString();
}

Example 2 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class CoordOldInputDependency method getUnResolvedDependency.

private ActionDependency getUnResolvedDependency(CoordinatorActionBean coordAction, Element event) throws JDOMException, URIHandlerException {
    String tmpUnresolved = event.getChildTextTrim(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG, event.getNamespace());
    StringBuilder nonResolvedList = new StringBuilder();
    CoordCommandUtils.getResolvedList(getMissingDependencies(), new StringBuilder(), nonResolvedList);
    ActionDependency dependency = new ActionDependency();
    if (nonResolvedList.length() > 0) {
        Element eDataset = event.getChild("dataset", event.getNamespace());
        dependency.setUriTemplate(eDataset.getChild("uri-template", eDataset.getNamespace()).getTextTrim());
        dependency.getMissingDependencies().add(tmpUnresolved);
    }
    return dependency;
}

Example 3 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class CoordOldInputDependency method getDependency.

@SuppressWarnings("unchecked")
private Map<String, ActionDependency> getDependency(CoordinatorActionBean coordAction) throws JDOMException, URIHandlerException {
    Map<String, ActionDependency> dependenciesMap = new HashMap<String, ActionDependency>();
    URIHandlerService uriService = Services.get().get(URIHandlerService.class);
    Element eAction = XmlUtils.parseXml(coordAction.getActionXml());
    Element inputList = eAction.getChild("input-events", eAction.getNamespace());
    List<Element> eDataEvents = inputList.getChildren("data-in", eAction.getNamespace());
    for (Element event : eDataEvents) {
        Element uri = event.getChild("uris", event.getNamespace());
        ActionDependency dependency = new ActionDependency();
        if (uri != null) {
            Element doneFlagElement = event.getChild("dataset", event.getNamespace()).getChild("done-flag", event.getNamespace());
            String[] dataSets = uri.getText().split(CoordELFunctions.INSTANCE_SEPARATOR);
            String doneFlag = CoordUtils.getDoneFlag(doneFlagElement);
            for (String dataSet : dataSets) {
                URIHandler uriHandler;
                uriHandler = uriService.getURIHandler(dataSet);
                dependency.getMissingDependencies().add(uriHandler.getURIWithDoneFlag(dataSet, doneFlag));
            }
        }
        if (event.getChildTextTrim(CoordCommandUtils.UNRESOLVED_INSTANCES_TAG, event.getNamespace()) != null) {
            ActionDependency unResolvedDependency = getUnResolvedDependency(coordAction, event);
            dependency.getMissingDependencies().addAll(unResolvedDependency.getMissingDependencies());
            dependency.setUriTemplate(unResolvedDependency.getUriTemplate());
        }
        dependenciesMap.put(event.getAttributeValue("name"), dependency);
    }
    return dependenciesMap;
}

Example 4 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class CoordPullInputDependency method getMissingDependencies.

@Override
public Map<String, ActionDependency> getMissingDependencies(CoordinatorActionBean coordAction) throws CommandException, IOException, JDOMException {
    Map<String, ActionDependency> missingDependenciesMap = new HashMap<String, ActionDependency>();
    missingDependenciesMap.putAll(super.getMissingDependencies(coordAction));
    for (String key : unResolvedList.keySet()) {
        if (!unResolvedList.get(key).isResolved()) {
            missingDependenciesMap.put(key, new ActionDependency(unResolvedList.get(key).getDependencies(), new ArrayList<String>()));
        }
    }
    return missingDependenciesMap;
}

Example 5 with ActionDependency

use of org.apache.oozie.dependency.ActionDependency in project oozie by apache.

the class CoordPushDependencyCheckXCommand method execute.

@Override
protected Void execute() throws CommandException {
    // this action should only get processed if current time > nominal time;
    // otherwise, requeue this action for delay execution;
    Date nominalTime = coordAction.getNominalTime();
    Date currentTime = new Date();
    if (nominalTime.compareTo(currentTime) > 0) {
        queue(new CoordPushDependencyCheckXCommand(coordAction.getId(), true), nominalTime.getTime() - currentTime.getTime());
        updateCoordAction(coordAction, false);
        LOG.info("[" + actionId + "]::CoordPushDependency:: nominal Time is newer than current time, so requeue and wait. Current=" + DateUtils.formatDateOozieTZ(currentTime) + ", nominal=" + DateUtils.formatDateOozieTZ(nominalTime));
        return null;
    }
    CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies();
    CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies();
    if (coordPushInputDependency.getMissingDependenciesAsList().size() == 0) {
        LOG.info("Nothing to check. Empty push missing dependency");
    } else {
        List<String> missingDependenciesArray = coordPushInputDependency.getMissingDependenciesAsList();
        LOG.info("First Push missing dependency is [{0}] ", missingDependenciesArray.get(0));
        LOG.trace("Push missing dependencies are [{0}] ", missingDependenciesArray);
        if (registerForNotification) {
            LOG.debug("Register for notifications is true");
        }
        try {
            Configuration actionConf = null;
            try {
                actionConf = new XConfiguration(new StringReader(coordAction.getRunConf()));
            } catch (IOException e) {
                throw new CommandException(ErrorCode.E1307, e.getMessage(), e);
            }
            boolean isChangeInDependency = true;
            boolean timeout = false;
            ActionDependency actionDependency = coordPushInputDependency.checkPushMissingDependencies(coordAction, registerForNotification);
            // CoordActionInputCheckXCommand for efficiency. listPartitions is costly.
            if (actionDependency.getMissingDependencies().size() == missingDependenciesArray.size()) {
                isChangeInDependency = false;
            } else {
                String stillMissingDeps = DependencyChecker.dependenciesAsString(actionDependency.getMissingDependencies());
                coordPushInputDependency.setMissingDependencies(stillMissingDeps);
            }
            if (coordPushInputDependency.isDependencyMet()) {
                // All push-based dependencies are available
                onAllPushDependenciesAvailable(coordPullInputDependency.isDependencyMet());
            } else {
                // Checking for timeout
                timeout = isTimeout();
                if (timeout) {
                    queue(new CoordActionTimeOutXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
                } else {
                    queue(new CoordPushDependencyCheckXCommand(coordAction.getId()), getCoordPushCheckRequeueInterval());
                }
            }
            updateCoordAction(coordAction, isChangeInDependency || coordPushInputDependency.isDependencyMet());
            if (registerForNotification) {
                registerForNotification(coordPushInputDependency.getMissingDependenciesAsList(), actionConf);
            }
            if (removeAvailDependencies) {
                unregisterAvailableDependencies(actionDependency.getAvailableDependencies());
            }
            if (timeout) {
                unregisterMissingDependencies(coordPushInputDependency.getMissingDependenciesAsList(), actionId);
            }
        } catch (Exception e) {
            final CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
            if (isTimeout()) {
                LOG.debug("Queueing timeout command");
                // XCommand.queue() will not work when there is a Exception
                callableQueueService.queue(new CoordActionTimeOutXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
                unregisterMissingDependencies(missingDependenciesArray, actionId);
            } else if (coordPullInputDependency.getMissingDependenciesAsList().size() > 0) {
                // Queue again on exception as RecoveryService will not queue this again with
                // the action being updated regularly by CoordActionInputCheckXCommand
                callableQueueService.queue(new CoordPushDependencyCheckXCommand(coordAction.getId(), registerForNotification, removeAvailDependencies), Services.get().getConf().getInt(RecoveryService.CONF_COORD_OLDER_THAN, 600) * 1000);
            }
            throw new CommandException(ErrorCode.E1021, e.getMessage(), e);
        }
    }
    return null;
}