Examples with CoordInputInstance org.apache.oozie.coord.input.dependency.CoordInputInstance used on opensource projects

Example 1 with CoordInputInstance

use of org.apache.oozie.coord.input.dependency.CoordInputInstance in project oozie by apache.

the class CoordInputLogicEvaluatorPhaseOne method combine.

public CoordInputLogicEvaluatorResult combine(AbstractCoordInputDependency coordInputDependency, String[] inputSets, int min, int wait) {
    List<String> availableList = new ArrayList<String>();
    if (coordInputDependency.getDependencyMap().get(inputSets[0]) == null) {
        return new CoordInputLogicEvaluatorResult(CoordInputLogicEvaluatorResult.STATUS.TIMED_WAITING);
    }
    try {
        Configuration jobConf = new XConfiguration(new StringReader(coordAction.getRunConf()));
        String firstInputSet = inputSets[0];
        List<CoordInputInstance> firstInputSetList = coordInputDependency.getDependencyMap().get(firstInputSet);
        for (int i = 0; i < firstInputSetList.size(); i++) {
            CoordInputInstance coordInputInstance = firstInputSetList.get(i);
            boolean found = false;
            if (!coordInputInstance.isAvailable()) {
                if (!pathExists(coordInputInstance.getInputDataInstance(), jobConf)) {
                    log.debug(MessageFormat.format("{0} is not found. Looking from other datasets.", coordInputInstance.getInputDataInstance()));
                    for (int j = 1; j < inputSets.length; j++) {
                        if (!coordInputDependency.getDependencyMap().get(inputSets[j]).get(i).isAvailable()) {
                            if (pathExists(coordInputDependency.getDependencyMap().get(inputSets[j]).get(i).getInputDataInstance(), jobConf)) {
                                coordInputDependency.addToAvailableDependencies(inputSets[j], coordInputDependency.getDependencyMap().get(inputSets[j]).get(i));
                                availableList.add(coordInputDependency.getDependencyMap().get(inputSets[j]).get(i).getInputDataInstance());
                                log.debug(MessageFormat.format("{0} is found.", coordInputInstance.getInputDataInstance()));
                                found = true;
                            }
                        } else {
                            coordInputDependency.addToAvailableDependencies(inputSets[j], coordInputDependency.getDependencyMap().get(inputSets[j]).get(i));
                            availableList.add(coordInputDependency.getDependencyMap().get(inputSets[j]).get(i).getInputDataInstance());
                            found = true;
                        }
                    }
                } else {
                    coordInputDependency.addToAvailableDependencies(firstInputSet, coordInputInstance);
                    availableList.add(coordInputInstance.getInputDataInstance());
                    found = true;
                }
            } else {
                availableList.add(coordInputInstance.getInputDataInstance());
                found = true;
            }
            if (min < 0 && !found) {
                // Stop looking for dependencies, if min is not specified.
                break;
            }
        }
    } catch (Exception e) {
        log.error(e);
        throw new RuntimeException(ErrorCode.E1028.format("Error executing combine function " + e.getMessage()));
    }
    boolean allFound = availableList.size() == coordInputDependency.getDependencyMap().get(inputSets[0]).size();
    CoordInputLogicEvaluatorResult retData = getEvalResult(allFound, min, wait, availableList);
    log.debug("Resolved status of Data set [{0}] with min [{1}] and wait [{2}]  =  [{3}]", Arrays.toString(inputSets), min, wait, retData.getStatus());
    return retData;
}

Example 2 with CoordInputInstance

use of org.apache.oozie.coord.input.dependency.CoordInputInstance in project oozie by apache.

the class CoordCommandUtils method materializeInputDataEvents.

public static void materializeInputDataEvents(List<Element> events, SyncCoordAction appInst, Configuration conf, CoordinatorActionBean actionBean, boolean isInputLogicSpecified) throws Exception {
    if (events == null) {
        return;
    }
    CoordInputDependency coordPullInputDependency = CoordInputDependencyFactory.createPullInputDependencies(isInputLogicSpecified);
    CoordInputDependency coordPushInputDependency = CoordInputDependencyFactory.createPushInputDependencies(isInputLogicSpecified);
    List<Pair<String, String>> unresolvedList = new ArrayList<Pair<String, String>>();
    URIHandlerService uriService = Services.get().get(URIHandlerService.class);
    for (Element event : events) {
        StringBuilder instances = new StringBuilder();
        ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf);
        // Handle list of instance tag
        resolveInstances(event, instances, appInst, conf, eval);
        // Handle start-instance and end-instance
        resolveInstanceRange(event, instances, appInst, conf, eval);
        // Separate out the unresolved instances
        String resolvedList = separateResolvedAndUnresolved(event, instances);
        String name = event.getAttribute("name").getValue();
        if (!resolvedList.isEmpty()) {
            Element uri = event.getChild("dataset", event.getNamespace()).getChild("uri-template", event.getNamespace());
            String uriTemplate = uri.getText();
            URI baseURI = uriService.getAuthorityWithScheme(uriTemplate);
            URIHandler handler = uriService.getURIHandler(baseURI);
            List<CoordInputInstance> inputInstanceList = new ArrayList<CoordInputInstance>();
            for (String inputInstance : resolvedList.split("#")) {
                inputInstanceList.add(new CoordInputInstance(inputInstance, false));
            }
            if (handler.getDependencyType(baseURI).equals(DependencyType.PULL)) {
                coordPullInputDependency.addInputInstanceList(name, inputInstanceList);
            } else {
                coordPushInputDependency.addInputInstanceList(name, inputInstanceList);
            }
        }
        String tmpUnresolved = event.getChildTextTrim(UNRESOLVED_INSTANCES_TAG, event.getNamespace());
        if (tmpUnresolved != null) {
            unresolvedList.add(new Pair<String, String>(name, tmpUnresolved));
        }
    }
    for (Pair<String, String> unresolvedDataset : unresolvedList) {
        coordPullInputDependency.addUnResolvedList(unresolvedDataset.getFirst(), unresolvedDataset.getSecond());
    }
    actionBean.setPullInputDependencies(coordPullInputDependency);
    actionBean.setPushInputDependencies(coordPushInputDependency);
    actionBean.setMissingDependencies(coordPullInputDependency.serialize());
    actionBean.setPushMissingDependencies(coordPushInputDependency.serialize());
}

Example 3 with CoordInputInstance

use of org.apache.oozie.coord.input.dependency.CoordInputInstance in project oozie by apache.

the class CoordInputLogicEvaluatorPhaseOne method input.

/**
 * Evaluate input function with min and wait
 *
 * @param coordInputDependency the dependency
 * @param dataSet the dataset
 * @param min the minimum number of available dataset
 * @param wait time to wait in minutes
 * @return the coord input logic evaluator result
 */
public CoordInputLogicEvaluatorResult input(AbstractCoordInputDependency coordInputDependency, String dataSet, int min, int wait) {
    List<String> availableList = new ArrayList<String>();
    if (coordInputDependency.getDependencyMap().get(dataSet) == null) {
        CoordInputLogicEvaluatorResult retData = new CoordInputLogicEvaluatorResult();
        if (((CoordPullInputDependency) coordAction.getPullInputDependencies()).getUnResolvedDependency(dataSet) != null) {
            log.debug("Data set [{0}] is unresolved set, will get resolved in phase two", dataSet);
            retData.setStatus(CoordInputLogicEvaluatorResult.STATUS.PHASE_TWO_EVALUATION);
        } else {
            return getResultFromPullPush(coordAction, dataSet, min);
        }
        return retData;
    }
    boolean allFound = true;
    try {
        Configuration actionConf = new XConfiguration(new StringReader(coordAction.getRunConf()));
        List<CoordInputInstance> firstInputSetList = coordInputDependency.getDependencyMap().get(dataSet);
        for (int i = 0; i < firstInputSetList.size(); i++) {
            CoordInputInstance coordInputInstance = firstInputSetList.get(i);
            if (!coordInputInstance.isAvailable()) {
                if (pathExists(coordInputInstance.getInputDataInstance(), actionConf)) {
                    availableList.add(coordInputInstance.getInputDataInstance());
                    coordInputDependency.addToAvailableDependencies(dataSet, coordInputInstance);
                } else {
                    log.debug("[{0} is not found ", coordInputInstance.getInputDataInstance());
                    allFound = false;
                    // Stop looking for dependencies, if min is not specified.
                    if (min < 0) {
                        break;
                    }
                }
            } else {
                availableList.add(coordInputInstance.getInputDataInstance());
            }
        }
    } catch (Exception e) {
        log.error(e);
        throw new RuntimeException(ErrorCode.E1028.format("Error executing input function " + e.getMessage()));
    }
    CoordInputLogicEvaluatorResult retData = getEvalResult(allFound, min, wait, availableList);
    log.debug("Resolved status of Data set [{0}] with min [{1}] and wait [{2}]  =  [{3}]", dataSet, min, wait, retData.getStatus());
    return retData;
}

Example 4 with CoordInputInstance

use of org.apache.oozie.coord.input.dependency.CoordInputInstance in project oozie by apache.

the class CoordInputLogicEvaluatorPhaseThree method combine.

public CoordInputLogicEvaluatorResult combine(AbstractCoordInputDependency coordInputDependency, String[] inputSets, int min, int wait) {
    List<String> availableList = new ArrayList<String>();
    if (coordInputDependency.getDependencyMap().get(inputSets[0]) == null) {
        return new CoordInputLogicEvaluatorResult(CoordInputLogicEvaluatorResult.STATUS.FALSE);
    }
    try {
        String firstInputSet = inputSets[0];
        List<CoordInputInstance> firstInputSetList = coordInputDependency.getDependencyMap().get(firstInputSet);
        for (int i = 0; i < firstInputSetList.size(); i++) {
            CoordInputInstance coordInputInstance = firstInputSetList.get(i);
            if (!coordInputInstance.isAvailable()) {
                for (int j = 1; j < inputSets.length; j++) {
                    if (coordInputDependency.getDependencyMap().get(inputSets[j]).get(i).isAvailable()) {
                        availableList.add(getPathWithoutDoneFlag(coordInputDependency.getDependencyMap().get(inputSets[j]).get(i).getInputDataInstance(), inputSets[j]));
                    }
                }
            } else {
                availableList.add(getPathWithoutDoneFlag(coordInputInstance.getInputDataInstance(), firstInputSet));
            }
        }
    } catch (Exception e) {
        log.error(e);
        throw new RuntimeException(ErrorCode.E1028.format("Error executing combine function " + e.getMessage()));
    }
    boolean allFound = availableList.size() == coordInputDependency.getDependencyMap().get(inputSets[0]).size();
    return getEvalResult(allFound, min, wait, availableList);
}