Examples with HadoopAccessorService org.apache.oozie.service.HadoopAccessorService used on opensource projects

Example 16 with HadoopAccessorService

use of org.apache.oozie.service.HadoopAccessorService in project oozie by apache.

the class BundleSubmitXCommand method readDefinition.

/**
 * Read bundle definition.
 *
 * @param appPath application path.
 * @return bundle definition.
 * @throws BundleJobException thrown if the definition could not be read.
 */
protected String readDefinition(String appPath) throws BundleJobException {
    String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
    // Configuration confHadoop = CoordUtils.getHadoopConf(conf);
    try {
        URI uri = new URI(appPath);
        LOG.debug("user =" + user);
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        Configuration fsConf = has.createConfiguration(uri.getAuthority());
        FileSystem fs = has.createFileSystem(user, uri, fsConf);
        Path appDefPath = null;
        // app path could be a directory
        Path path = new Path(uri.getPath());
        if (!fs.isFile(path)) {
            appDefPath = new Path(path, BUNDLE_XML_FILE);
        } else {
            appDefPath = path;
        }
        Reader reader = new InputStreamReader(fs.open(appDefPath));
        StringWriter writer = new StringWriter();
        IOUtils.copyCharStream(reader, writer);
        return writer.toString();
    } catch (IOException ex) {
        LOG.warn("IOException :" + XmlUtils.prettyPrint(conf), ex);
        throw new BundleJobException(ErrorCode.E1301, ex.getMessage(), ex);
    } catch (URISyntaxException ex) {
        LOG.warn("URISyException :" + ex.getMessage());
        throw new BundleJobException(ErrorCode.E1302, appPath, ex.getMessage(), ex);
    } catch (HadoopAccessorException ex) {
        throw new BundleJobException(ex);
    } catch (Exception ex) {
        LOG.warn("Exception :", ex);
        throw new BundleJobException(ErrorCode.E1301, ex.getMessage(), ex);
    }
}

Example 17 with HadoopAccessorService

use of org.apache.oozie.service.HadoopAccessorService in project oozie by apache.

the class CoordSubmitXCommand method mergeDefaultConfig.

/**
 * Merge default configuration with user-defined configuration.
 *
 * @throws CommandException thrown if failed to read or merge configurations
 */
protected void mergeDefaultConfig() throws CommandException {
    Path configDefault = null;
    try {
        String coordAppPathStr = conf.get(OozieClient.COORDINATOR_APP_PATH);
        Path coordAppPath = new Path(coordAppPathStr);
        String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        Configuration fsConf = has.createConfiguration(coordAppPath.toUri().getAuthority());
        FileSystem fs = has.createFileSystem(user, coordAppPath.toUri(), fsConf);
        // app path could be a directory
        if (!fs.isFile(coordAppPath)) {
            configDefault = new Path(coordAppPath, CONFIG_DEFAULT);
        } else {
            configDefault = new Path(coordAppPath.getParent(), CONFIG_DEFAULT);
        }
        if (fs.exists(configDefault)) {
            Configuration defaultConf = new XConfiguration(fs.open(configDefault));
            PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
            XConfiguration.injectDefaults(defaultConf, conf);
        } else {
            LOG.info("configDefault Doesn't exist " + configDefault);
        }
        PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
        // Resolving all variables in the job properties.
        // This ensures the Hadoop Configuration semantics is preserved.
        XConfiguration resolvedVarsConf = new XConfiguration();
        for (Map.Entry<String, String> entry : conf) {
            resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
        }
        conf = resolvedVarsConf;
    } catch (IOException e) {
        throw new CommandException(ErrorCode.E0702, e.getMessage() + " : Problem reading default config " + configDefault, e);
    } catch (HadoopAccessorException e) {
        throw new CommandException(e);
    }
    LOG.debug("Merged CONF :" + XmlUtils.prettyPrint(conf).toString());
}

Example 18 with HadoopAccessorService

use of org.apache.oozie.service.HadoopAccessorService in project oozie by apache.

the class WfEndXCommand method getAppFileSystem.

protected FileSystem getAppFileSystem(WorkflowJob workflow) throws HadoopAccessorException, IOException, URISyntaxException {
    URI uri = new URI(workflow.getAppPath());
    HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
    Configuration fsConf = has.createConfiguration(uri.getAuthority());
    return has.createFileSystem(workflow.getUser(), uri, fsConf);
}

Example 19 with HadoopAccessorService

use of org.apache.oozie.service.HadoopAccessorService in project oozie by apache.

the class JobUtils method normalizeAppPath.

/**
 * Normalize appPath in job conf with the provided user/group - If it's not jobs via proxy submission, after
 * normalization appPath always points to job's Xml definition file.
 * <p>
 *
 * @param user user
 * @param group group
 * @param conf job configuration.
 * @throws IOException thrown if normalization can not be done properly.
 */
public static void normalizeAppPath(String user, String group, Configuration conf) throws IOException {
    ParamChecker.notNull(user, "user");
    if (conf.get(XOozieClient.IS_PROXY_SUBMISSION) != null) {
        // do nothing for proxy submission job;
        return;
    }
    String wfPathStr = conf.get(OozieClient.APP_PATH);
    String coordPathStr = conf.get(OozieClient.COORDINATOR_APP_PATH);
    String bundlePathStr = conf.get(OozieClient.BUNDLE_APP_PATH);
    String appPathStr = wfPathStr != null ? wfPathStr : (coordPathStr != null ? coordPathStr : bundlePathStr);
    FileSystem fs = null;
    try {
        URI uri = new Path(appPathStr).toUri();
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        Configuration fsConf = has.createConfiguration(uri.getAuthority());
        fs = has.createFileSystem(user, uri, fsConf);
    } catch (HadoopAccessorException ex) {
        throw new IOException(ex.getMessage());
    }
    Path appPath = new Path(appPathStr);
    String normalizedAppPathStr = appPathStr;
    if (!fs.exists(appPath)) {
        throw new IOException("Error: " + appPathStr + " does not exist");
    }
    if (wfPathStr != null) {
        conf.set(OozieClient.APP_PATH, normalizedAppPathStr);
    } else if (coordPathStr != null) {
        conf.set(OozieClient.COORDINATOR_APP_PATH, normalizedAppPathStr);
    } else if (bundlePathStr != null) {
        conf.set(OozieClient.BUNDLE_APP_PATH, normalizedAppPathStr);
    }
}

Example 20 with HadoopAccessorService

use of org.apache.oozie.service.HadoopAccessorService in project oozie by apache.

the class XFsTestCase method setUp.

/**
 * Set up the testcase.
 *
 * @throws Exception thrown if the test case could no be set up.
 */
protected void setUp() throws Exception {
    super.setUp();
    Configuration conf = new XConfiguration();
    conf.setBoolean("oozie.service.HadoopAccessorService.kerberos.enabled", System.getProperty("oozie.test.hadoop.security", "simple").equals("kerberos"));
    conf.set("oozie.service.HadoopAccessorService.keytab.file", getKeytabFile());
    conf.set("oozie.service.HadoopAccessorService.kerberos.principal", getOoziePrincipal());
    conf.set("local.realm", getRealm());
    conf.set("oozie.service.HadoopAccessorService.hadoop.configurations", "*=hadoop-conf");
    conf.set("oozie.service.HadoopAccessorService.action.configurations", "*=action-conf");
    has = new HadoopAccessorService();
    has.init(conf);
    Configuration jobConf = has.createConfiguration(getNameNodeUri());
    XConfiguration.copy(conf, jobConf);
    fileSystem = has.createFileSystem(getTestUser(), new URI(getNameNodeUri()), jobConf);
    fsTestDir = initFileSystem(fileSystem);
    if (System.getProperty("oozie.test.hadoop.minicluster2", "false").equals("true")) {
        fileSystem2 = has.createFileSystem(getTestUser(), new URI(getNameNode2Uri()), jobConf);
        fsTestDir2 = initFileSystem(fileSystem2);
    }
}