Examples with URIHandlerService org.apache.oozie.service.URIHandlerService used on opensource projects

Example 11 with URIHandlerService

use of org.apache.oozie.service.URIHandlerService in project oozie by apache.

the class CoordCommandUtils method pathExists.

public static boolean pathExists(String sPath, Configuration actionConf, String user) throws IOException, URISyntaxException, URIHandlerException {
    URI uri = new URI(sPath);
    URIHandlerService service = Services.get().get(URIHandlerService.class);
    URIHandler handler = service.getURIHandler(uri);
    return handler.exists(uri, actionConf, user);
}

Example 12 with URIHandlerService

use of org.apache.oozie.service.URIHandlerService in project oozie by apache.

the class CoordCommandUtils method materializeInputDataEvents.

public static void materializeInputDataEvents(List<Element> events, SyncCoordAction appInst, Configuration conf, CoordinatorActionBean actionBean, boolean isInputLogicSpecified) throws Exception {
    if (events == null) {
        return;
    }
    CoordInputDependency coordPullInputDependency = CoordInputDependencyFactory.createPullInputDependencies(isInputLogicSpecified);
    CoordInputDependency coordPushInputDependency = CoordInputDependencyFactory.createPushInputDependencies(isInputLogicSpecified);
    List<Pair<String, String>> unresolvedList = new ArrayList<Pair<String, String>>();
    URIHandlerService uriService = Services.get().get(URIHandlerService.class);
    for (Element event : events) {
        StringBuilder instances = new StringBuilder();
        ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf);
        // Handle list of instance tag
        resolveInstances(event, instances, appInst, conf, eval);
        // Handle start-instance and end-instance
        resolveInstanceRange(event, instances, appInst, conf, eval);
        // Separate out the unresolved instances
        String resolvedList = separateResolvedAndUnresolved(event, instances);
        String name = event.getAttribute("name").getValue();
        if (!resolvedList.isEmpty()) {
            Element uri = event.getChild("dataset", event.getNamespace()).getChild("uri-template", event.getNamespace());
            String uriTemplate = uri.getText();
            URI baseURI = uriService.getAuthorityWithScheme(uriTemplate);
            URIHandler handler = uriService.getURIHandler(baseURI);
            List<CoordInputInstance> inputInstanceList = new ArrayList<CoordInputInstance>();
            for (String inputInstance : resolvedList.split("#")) {
                inputInstanceList.add(new CoordInputInstance(inputInstance, false));
            }
            if (handler.getDependencyType(baseURI).equals(DependencyType.PULL)) {
                coordPullInputDependency.addInputInstanceList(name, inputInstanceList);
            } else {
                coordPushInputDependency.addInputInstanceList(name, inputInstanceList);
            }
        }
        String tmpUnresolved = event.getChildTextTrim(UNRESOLVED_INSTANCES_TAG, event.getNamespace());
        if (tmpUnresolved != null) {
            unresolvedList.add(new Pair<String, String>(name, tmpUnresolved));
        }
    }
    for (Pair<String, String> unresolvedDataset : unresolvedList) {
        coordPullInputDependency.addUnResolvedList(unresolvedDataset.getFirst(), unresolvedDataset.getSecond());
    }
    actionBean.setPullInputDependencies(coordPullInputDependency);
    actionBean.setPushInputDependencies(coordPushInputDependency);
    actionBean.setMissingDependencies(coordPullInputDependency.serialize());
    actionBean.setPushMissingDependencies(coordPushInputDependency.serialize());
}

Example 13 with URIHandlerService

use of org.apache.oozie.service.URIHandlerService in project oozie by apache.

the class CoordELFunctions method coord_futureRange_sync.

private static String coord_futureRange_sync(int startOffset, int endOffset, int instance) throws Exception {
    final XLog LOG = XLog.getLog(CoordELFunctions.class);
    final Thread currentThread = Thread.currentThread();
    ELEvaluator eval = ELEvaluator.getCurrent();
    String retVal = "";
    // in minutes
    int datasetFrequency = (int) getDSFrequency();
    TimeUnit dsTimeUnit = getDSTimeUnit();
    int[] instCount = new int[1];
    Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
    StringBuilder resolvedInstances = new StringBuilder();
    StringBuilder resolvedURIPaths = new StringBuilder();
    if (nominalInstanceCal != null) {
        Calendar initInstance = getInitialInstanceCal();
        nominalInstanceCal = (Calendar) initInstance.clone();
        nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
        SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
        if (ds == null) {
            throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
        }
        String uriTemplate = ds.getUriTemplate();
        Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
        if (conf == null) {
            throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
        }
        int available = 0, checkedInstance = 0;
        boolean resolved = false;
        String user = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
        String doneFlag = ds.getDoneFlag();
        URIHandlerService uriService = Services.get().get(URIHandlerService.class);
        URIHandler uriHandler = null;
        Context uriContext = null;
        try {
            while (instance >= checkedInstance && !currentThread.isInterrupted()) {
                ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
                String uriPath = uriEval.evaluate(uriTemplate, String.class);
                if (uriHandler == null) {
                    URI uri = new URI(uriPath);
                    uriHandler = uriService.getURIHandler(uri);
                    uriContext = uriHandler.getContext(uri, conf, user, true);
                }
                String uriWithDoneFlag = uriHandler.getURIWithDoneFlag(uriPath, doneFlag);
                if (uriHandler.exists(new URI(uriWithDoneFlag), uriContext)) {
                    if (available == endOffset) {
                        LOG.debug("Matched future(" + available + "): " + uriWithDoneFlag);
                        resolved = true;
                        resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal));
                        resolvedURIPaths.append(uriPath);
                        retVal = resolvedInstances.toString();
                        eval.setVariable(CoordELConstants.RESOLVED_PATH, resolvedURIPaths.toString());
                        break;
                    } else if (available >= startOffset) {
                        LOG.debug("Matched future(" + available + "): " + uriWithDoneFlag);
                        resolvedInstances.append(DateUtils.formatDateOozieTZ(nominalInstanceCal)).append(INSTANCE_SEPARATOR);
                        resolvedURIPaths.append(uriPath).append(INSTANCE_SEPARATOR);
                    }
                    available++;
                }
                // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency);
                nominalInstanceCal = (Calendar) initInstance.clone();
                instCount[0]++;
                nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
                checkedInstance++;
            // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
            }
            if (!StringUtils.isEmpty(resolvedURIPaths.toString()) && eval.getVariable(CoordELConstants.RESOLVED_PATH) == null) {
                eval.setVariable(CoordELConstants.RESOLVED_PATH, resolvedURIPaths.toString());
            }
        } finally {
            if (uriContext != null) {
                uriContext.destroy();
            }
        }
        if (!resolved) {
            // return unchanged future function with variable 'is_resolved'
            // to 'false'
            eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.FALSE);
            if (startOffset == endOffset) {
                retVal = "${coord:future(" + startOffset + ", " + instance + ")}";
            } else {
                retVal = "${coord:futureRange(" + startOffset + ", " + endOffset + ", " + instance + ")}";
            }
        } else {
            eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.TRUE);
        }
    } else {
        // No feasible nominal time
        eval.setVariable(CoordELConstants.IS_RESOLVED, Boolean.TRUE);
        retVal = "";
    }
    return retVal;
}

Example 14 with URIHandlerService

use of org.apache.oozie.service.URIHandlerService in project oozie by apache.

the class TestLauncherFSURIHandler method setUp.

@Override
protected void setUp() throws Exception {
    super.setUp();
    services = new Services();
    services.init();
    URIHandlerService uriService = services.get(URIHandlerService.class);
    uriHandlerFactory = new LauncherURIHandlerFactory(uriService.getLauncherConfig());
    conf = createJobConf();
}

Example 15 with URIHandlerService

use of org.apache.oozie.service.URIHandlerService in project oozie by apache.

the class TestURIHandlerService method testGetURIHandler.

@Test
public void testGetURIHandler() throws Exception {
    URIHandlerService uriService = services.get(URIHandlerService.class);
    URI uri = uriService.getAuthorityWithScheme("/tmp/file");
    URIHandler uriHandler = uriService.getURIHandler(uri);
    assertTrue(uriHandler instanceof FSURIHandler);
}