Examples with CoordInputDependency org.apache.oozie.coord.input.dependency.CoordInputDependency used on opensource projects

Example 1 with CoordInputDependency

use of org.apache.oozie.coord.input.dependency.CoordInputDependency in project oozie by apache.

the class CoordCommandUtils method getFirstMissingDependency.

public static String getFirstMissingDependency(CoordinatorActionBean coordAction) {
    CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies();
    CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies();
    String firstMissingDependencies = coordPullInputDependency.getFirstMissingDependency();
    if (StringUtils.isEmpty(firstMissingDependencies) || firstMissingDependencies.trim().startsWith("${coord:")) {
        firstMissingDependencies = coordPushInputDependency.getFirstMissingDependency();
    }
    return firstMissingDependencies;
}

Example 2 with CoordInputDependency

use of org.apache.oozie.coord.input.dependency.CoordInputDependency in project oozie by apache.

the class CoordPushDependencyCheckXCommand method execute.

@Override
protected Void execute() throws CommandException {
    // this action should only get processed if current time > nominal time;
    // otherwise, requeue this action for delay execution;
    Date nominalTime = coordAction.getNominalTime();
    Date currentTime = new Date();
    if (nominalTime.compareTo(currentTime) > 0) {
        queue(new CoordPushDependencyCheckXCommand(coordAction.getId(), true), nominalTime.getTime() - currentTime.getTime());
        updateCoordAction(coordAction, false);
        LOG.info("[" + actionId + "]::CoordPushDependency:: nominal Time is newer than current time, so requeue and wait. Current=" + DateUtils.formatDateOozieTZ(currentTime) + ", nominal=" + DateUtils.formatDateOozieTZ(nominalTime));
        return null;
    }
    CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies();
    CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies();
    if (coordPushInputDependency.getMissingDependenciesAsList().size() == 0) {
        LOG.info("Nothing to check. Empty push missing dependency");
    } else {
        List<String> missingDependenciesArray = coordPushInputDependency.getMissingDependenciesAsList();
        LOG.info("First Push missing dependency is [{0}] ", missingDependenciesArray.get(0));
        LOG.trace("Push missing dependencies are [{0}] ", missingDependenciesArray);
        if (registerForNotification) {
            LOG.debug("Register for notifications is true");
        }
        try {
            Configuration actionConf = null;
            try {
                actionConf = new XConfiguration(new StringReader(coordAction.getRunConf()));
            } catch (IOException e) {
                throw new CommandException(ErrorCode.E1307, e.getMessage(), e);
            }
            boolean isChangeInDependency = true;
            boolean timeout = false;
            ActionDependency actionDependency = coordPushInputDependency.checkPushMissingDependencies(coordAction, registerForNotification);
            // CoordActionInputCheckXCommand for efficiency. listPartitions is costly.
            if (actionDependency.getMissingDependencies().size() == missingDependenciesArray.size()) {
                isChangeInDependency = false;
            } else {
                String stillMissingDeps = DependencyChecker.dependenciesAsString(actionDependency.getMissingDependencies());
                coordPushInputDependency.setMissingDependencies(stillMissingDeps);
            }
            if (coordPushInputDependency.isDependencyMet()) {
                // All push-based dependencies are available
                onAllPushDependenciesAvailable(coordPullInputDependency.isDependencyMet());
            } else {
                // Checking for timeout
                timeout = isTimeout();
                if (timeout) {
                    queue(new CoordActionTimeOutXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
                } else {
                    queue(new CoordPushDependencyCheckXCommand(coordAction.getId()), getCoordPushCheckRequeueInterval());
                }
            }
            updateCoordAction(coordAction, isChangeInDependency || coordPushInputDependency.isDependencyMet());
            if (registerForNotification) {
                registerForNotification(coordPushInputDependency.getMissingDependenciesAsList(), actionConf);
            }
            if (removeAvailDependencies) {
                unregisterAvailableDependencies(actionDependency.getAvailableDependencies());
            }
            if (timeout) {
                unregisterMissingDependencies(coordPushInputDependency.getMissingDependenciesAsList(), actionId);
            }
        } catch (Exception e) {
            final CallableQueueService callableQueueService = Services.get().get(CallableQueueService.class);
            if (isTimeout()) {
                LOG.debug("Queueing timeout command");
                // XCommand.queue() will not work when there is a Exception
                callableQueueService.queue(new CoordActionTimeOutXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
                unregisterMissingDependencies(missingDependenciesArray, actionId);
            } else if (coordPullInputDependency.getMissingDependenciesAsList().size() > 0) {
                // Queue again on exception as RecoveryService will not queue this again with
                // the action being updated regularly by CoordActionInputCheckXCommand
                callableQueueService.queue(new CoordPushDependencyCheckXCommand(coordAction.getId(), registerForNotification, removeAvailDependencies), Services.get().getConf().getInt(RecoveryService.CONF_COORD_OLDER_THAN, 600) * 1000);
            }
            throw new CommandException(ErrorCode.E1021, e.getMessage(), e);
        }
    }
    return null;
}

Example 3 with CoordInputDependency

use of org.apache.oozie.coord.input.dependency.CoordInputDependency in project oozie by apache.

the class CoordActionInputCheckXCommand method execute.

@Override
protected Void execute() throws CommandException {
    LOG.debug("[" + actionId + "]::ActionInputCheck:: Action is in WAITING state.");
    // this action should only get processed if current time > nominal time;
    // otherwise, requeue this action for delay execution;
    Date nominalTime = coordAction.getNominalTime();
    Date currentTime = new Date();
    if (nominalTime.compareTo(currentTime) > 0) {
        queue(new CoordActionInputCheckXCommand(coordAction.getId(), coordAction.getJobId()), nominalTime.getTime() - currentTime.getTime());
        updateCoordAction(coordAction, false);
        LOG.info("[" + actionId + "]::ActionInputCheck:: nominal Time is newer than current time, so requeue and wait. Current=" + DateUtils.formatDateOozieTZ(currentTime) + ", nominal=" + DateUtils.formatDateOozieTZ(nominalTime));
        return null;
    }
    StringBuilder actionXml = new StringBuilder(coordAction.getActionXml());
    boolean isChangeInDependency = false;
    try {
        Configuration actionConf = new XConfiguration(new StringReader(coordAction.getRunConf()));
        Date now = new Date();
        if (coordJob.getExecutionOrder().equals(CoordinatorJobBean.Execution.LAST_ONLY)) {
            Date nextNominalTime = CoordCommandUtils.computeNextNominalTime(coordJob, coordAction);
            if (nextNominalTime != null) {
                // should be started; so set it to SKIPPED
                if (now.after(nextNominalTime)) {
                    LOG.info("LAST_ONLY execution: Preparing to skip action [{0}] because the current time [{1}] is later than " + "the nominal time [{2}] of the next action]", coordAction.getId(), DateUtils.formatDateOozieTZ(now), DateUtils.formatDateOozieTZ(nextNominalTime));
                    queue(new CoordActionSkipXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
                    return null;
                } else {
                    LOG.debug("LAST_ONLY execution: Not skipping action [{0}] because the current time [{1}] is earlier than " + "the nominal time [{2}] of the next action]", coordAction.getId(), DateUtils.formatDateOozieTZ(now), DateUtils.formatDateOozieTZ(nextNominalTime));
                }
            }
        } else if (coordJob.getExecutionOrder().equals(CoordinatorJobBean.Execution.NONE)) {
            // If the current time is after the nominal time of this action plus some tolerance,
            // then we've passed the window where this action should be started; so set it to SKIPPED
            Calendar cal = Calendar.getInstance(DateUtils.getTimeZone(coordJob.getTimeZone()));
            cal.setTime(nominalTime);
            int tolerance = ConfigurationService.getInt(COORD_EXECUTION_NONE_TOLERANCE);
            cal.add(Calendar.MINUTE, tolerance);
            if (now.after(cal.getTime())) {
                LOG.info("NONE execution: Preparing to skip action [{0}] because the current time [{1}] is more than [{2}]" + " minutes later than the nominal time [{3}] of the current action]", coordAction.getId(), DateUtils.formatDateOozieTZ(now), tolerance, DateUtils.formatDateOozieTZ(nominalTime));
                queue(new CoordActionSkipXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
                return null;
            } else {
                LOG.debug("NONE execution: Not skipping action [{0}] because the current time [{1}] is earlier than [{2}]" + " minutes later than the nominal time [{3}] of the current action]", coordAction.getId(), DateUtils.formatDateOozieTZ(now), tolerance, DateUtils.formatDateOozieTZ(coordAction.getNominalTime()));
            }
        }
        StringBuilder existList = new StringBuilder();
        StringBuilder nonExistList = new StringBuilder();
        CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies();
        CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies();
        String missingDependencies = coordPullInputDependency.getMissingDependencies();
        StringBuilder nonResolvedList = new StringBuilder();
        CoordCommandUtils.getResolvedList(missingDependencies, nonExistList, nonResolvedList);
        String firstMissingDependency = "";
        // instead of printing entire list, some of which, may be available
        if (nonExistList.length() > 0) {
            firstMissingDependency = nonExistList.toString().split(CoordELFunctions.INSTANCE_SEPARATOR)[0];
        }
        LOG.info("[" + actionId + "]::CoordActionInputCheck:: Missing deps:" + firstMissingDependency + " " + nonResolvedList.toString());
        boolean status = checkResolvedInput(actionXml, existList, nonExistList, actionConf);
        boolean isPushDependenciesMet = coordPushInputDependency.isDependencyMet();
        if (status && nonResolvedList.length() > 0) {
            status = (isPushDependenciesMet) ? checkUnResolvedInput(actionXml, actionConf) : false;
        }
        coordAction.setLastModifiedTime(currentTime);
        coordAction.setActionXml(actionXml.toString());
        isChangeInDependency = isChangeInDependency(nonExistList, missingDependencies, nonResolvedList, status);
        if (status && isPushDependenciesMet) {
            moveCoordActionToReady(actionXml, actionConf, coordPullInputDependency, coordPushInputDependency);
        } else if (!isTimeout(currentTime)) {
            if (!status) {
                long addtionalDelay = isChangeInDependency ? 0 : ConfigurationService.getInt(CONF_COORD_INPUT_CHECK_REQUEUE_INTERVAL_ADDITIONAL_DELAY) * 1000L;
                queue(new CoordActionInputCheckXCommand(coordAction.getId(), coordAction.getJobId()), addtionalDelay + getCoordInputCheckRequeueInterval());
            }
            updateCoordAction(coordAction, isChangeInDependency);
        } else {
            if (isPushDependenciesMet) {
                queue(new CoordActionTimeOutXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
            } else {
                // Let CoordPushDependencyCheckXCommand queue the timeout
                queue(new CoordPushDependencyCheckXCommand(coordAction.getId()));
            }
            updateCoordAction(coordAction, isChangeInDependency);
        }
    } catch (AccessControlException e) {
        LOG.error("Permission error in ActionInputCheck", e);
        if (isTimeout(currentTime)) {
            LOG.debug("Queueing timeout command");
            Services.get().get(CallableQueueService.class).queue(new CoordActionTimeOutXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
        } else {
            // Requeue InputCheckCommand for permission denied error with longer interval
            Services.get().get(CallableQueueService.class).queue(new CoordActionInputCheckXCommand(coordAction.getId(), coordAction.getJobId()), 2 * getCoordInputCheckRequeueInterval());
        }
        updateCoordAction(coordAction, isChangeInDependency);
    } catch (Exception e) {
        if (isTimeout(currentTime)) {
            LOG.debug("Queueing timeout command");
            // XCommand.queue() will not work when there is a Exception
            Services.get().get(CallableQueueService.class).queue(new CoordActionTimeOutXCommand(coordAction, coordJob.getUser(), coordJob.getAppName()));
        }
        updateCoordAction(coordAction, isChangeInDependency);
        throw new CommandException(ErrorCode.E1021, e.getMessage(), e);
    }
    return null;
}

Example 4 with CoordInputDependency

use of org.apache.oozie.coord.input.dependency.CoordInputDependency in project oozie by apache.

the class CoordCommandUtils method materializeInputDataEvents.

public static void materializeInputDataEvents(List<Element> events, SyncCoordAction appInst, Configuration conf, CoordinatorActionBean actionBean, boolean isInputLogicSpecified) throws Exception {
    if (events == null) {
        return;
    }
    CoordInputDependency coordPullInputDependency = CoordInputDependencyFactory.createPullInputDependencies(isInputLogicSpecified);
    CoordInputDependency coordPushInputDependency = CoordInputDependencyFactory.createPushInputDependencies(isInputLogicSpecified);
    List<Pair<String, String>> unresolvedList = new ArrayList<Pair<String, String>>();
    URIHandlerService uriService = Services.get().get(URIHandlerService.class);
    for (Element event : events) {
        StringBuilder instances = new StringBuilder();
        ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf);
        // Handle list of instance tag
        resolveInstances(event, instances, appInst, conf, eval);
        // Handle start-instance and end-instance
        resolveInstanceRange(event, instances, appInst, conf, eval);
        // Separate out the unresolved instances
        String resolvedList = separateResolvedAndUnresolved(event, instances);
        String name = event.getAttribute("name").getValue();
        if (!resolvedList.isEmpty()) {
            Element uri = event.getChild("dataset", event.getNamespace()).getChild("uri-template", event.getNamespace());
            String uriTemplate = uri.getText();
            URI baseURI = uriService.getAuthorityWithScheme(uriTemplate);
            URIHandler handler = uriService.getURIHandler(baseURI);
            List<CoordInputInstance> inputInstanceList = new ArrayList<CoordInputInstance>();
            for (String inputInstance : resolvedList.split("#")) {
                inputInstanceList.add(new CoordInputInstance(inputInstance, false));
            }
            if (handler.getDependencyType(baseURI).equals(DependencyType.PULL)) {
                coordPullInputDependency.addInputInstanceList(name, inputInstanceList);
            } else {
                coordPushInputDependency.addInputInstanceList(name, inputInstanceList);
            }
        }
        String tmpUnresolved = event.getChildTextTrim(UNRESOLVED_INSTANCES_TAG, event.getNamespace());
        if (tmpUnresolved != null) {
            unresolvedList.add(new Pair<String, String>(name, tmpUnresolved));
        }
    }
    for (Pair<String, String> unresolvedDataset : unresolvedList) {
        coordPullInputDependency.addUnResolvedList(unresolvedDataset.getFirst(), unresolvedDataset.getSecond());
    }
    actionBean.setPullInputDependencies(coordPullInputDependency);
    actionBean.setPushInputDependencies(coordPushInputDependency);
    actionBean.setMissingDependencies(coordPullInputDependency.serialize());
    actionBean.setPushMissingDependencies(coordPushInputDependency.serialize());
}

Example 5 with CoordInputDependency

use of org.apache.oozie.coord.input.dependency.CoordInputDependency in project oozie by apache.

the class CoordActionMissingDependenciesXCommand method execute.

@Override
protected List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> execute() throws CommandException {
    List<Pair<CoordinatorActionBean, Map<String, ActionDependency>>> inputDependenciesListPair = new ArrayList<Pair<CoordinatorActionBean, Map<String, ActionDependency>>>();
    try {
        for (CoordinatorActionBean coordAction : coordActions) {
            CoordInputDependency coordPullInputDependency = coordAction.getPullInputDependencies();
            CoordInputDependency coordPushInputDependency = coordAction.getPushInputDependencies();
            Map<String, ActionDependency> dependencyMap = new HashMap<String, ActionDependency>();
            dependencyMap.putAll(coordPullInputDependency.getMissingDependencies(coordAction));
            dependencyMap.putAll(coordPushInputDependency.getMissingDependencies(coordAction));
            inputDependenciesListPair.add(new Pair<CoordinatorActionBean, Map<String, ActionDependency>>(coordAction, dependencyMap));
        }
    } catch (Exception e) {
        throw new CommandException(ErrorCode.E1028, e.getMessage(), e);
    }
    return inputDependenciesListPair;
}