Examples with HadoopAccessorService org.apache.oozie.service.HadoopAccessorService used on opensource projects

Example 1 with HadoopAccessorService

use of org.apache.oozie.service.HadoopAccessorService in project oozie by apache.

the class FsELFunctions method getFileSystem.

private static FileSystem getFileSystem(URI uri) throws HadoopAccessorException {
    WorkflowJob workflow = DagELFunctions.getWorkflow();
    String user = workflow.getUser();
    HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
    Configuration conf = has.createConfiguration(uri.getAuthority());
    return has.createFileSystem(user, uri, conf);
}

Example 2 with HadoopAccessorService

use of org.apache.oozie.service.HadoopAccessorService in project oozie by apache.

the class JavaActionExecutor method setupLauncherConf.

Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException {
    try {
        Namespace ns = actionXml.getNamespace();
        XConfiguration launcherConf = new XConfiguration();
        // Inject action defaults for launcher
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        XConfiguration actionDefaultConf = has.createActionDefaultConf(conf.get(HADOOP_YARN_RM), getType());
        new LauncherConfigurationInjector(actionDefaultConf).inject(launcherConf);
        // Inject <job-xml> and <configuration> for launcher
        try {
            parseJobXmlAndConfiguration(context, actionXml, appPath, launcherConf, true);
        } catch (HadoopAccessorException ex) {
            throw convertException(ex);
        } catch (URISyntaxException ex) {
            throw convertException(ex);
        }
        XConfiguration.copy(launcherConf, conf);
        // Inject config-class for launcher to use for action
        Element e = actionXml.getChild("config-class", ns);
        if (e != null) {
            conf.set(LauncherAMUtils.OOZIE_ACTION_CONFIG_CLASS, e.getTextTrim());
        }
        checkForDisallowedProps(launcherConf, "launcher configuration");
        return conf;
    } catch (IOException ex) {
        throw convertException(ex);
    }
}

Example 3 with HadoopAccessorService

use of org.apache.oozie.service.HadoopAccessorService in project oozie by apache.

the class JavaActionExecutor method parseJobXmlAndConfiguration.

public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf, boolean isLauncher) throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException {
    Namespace ns = element.getNamespace();
    @SuppressWarnings("unchecked") Iterator<Element> it = element.getChildren("job-xml", ns).iterator();
    HashMap<String, FileSystem> filesystemsMap = new HashMap<String, FileSystem>();
    HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
    while (it.hasNext()) {
        Element e = it.next();
        String jobXml = e.getTextTrim();
        Path pathSpecified = new Path(jobXml);
        Path path = pathSpecified.isAbsolute() ? pathSpecified : new Path(appPath, jobXml);
        FileSystem fs;
        if (filesystemsMap.containsKey(path.toUri().getAuthority())) {
            fs = filesystemsMap.get(path.toUri().getAuthority());
        } else {
            if (path.toUri().getAuthority() != null) {
                fs = has.createFileSystem(context.getWorkflow().getUser(), path.toUri(), has.createConfiguration(path.toUri().getAuthority()));
            } else {
                fs = context.getAppFileSystem();
            }
            filesystemsMap.put(path.toUri().getAuthority(), fs);
        }
        Configuration jobXmlConf = new XConfiguration(fs.open(path));
        try {
            String jobXmlConfString = XmlUtils.prettyPrint(jobXmlConf).toString();
            jobXmlConfString = XmlUtils.removeComments(jobXmlConfString);
            jobXmlConfString = context.getELEvaluator().evaluate(jobXmlConfString, String.class);
            jobXmlConf = new XConfiguration(new StringReader(jobXmlConfString));
        } catch (ELEvaluationException ex) {
            throw new ActionExecutorException(ActionExecutorException.ErrorType.TRANSIENT, "EL_EVAL_ERROR", ex.getMessage(), ex);
        } catch (Exception ex) {
            context.setErrorInfo("EL_ERROR", ex.getMessage());
        }
        checkForDisallowedProps(jobXmlConf, "job-xml");
        if (isLauncher) {
            new LauncherConfigurationInjector(jobXmlConf).inject(conf);
        } else {
            XConfiguration.copy(jobXmlConf, conf);
        }
    }
    Element e = element.getChild("configuration", ns);
    if (e != null) {
        String strConf = XmlUtils.prettyPrint(e).toString();
        XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
        checkForDisallowedProps(inlineConf, "inline configuration");
        if (isLauncher) {
            new LauncherConfigurationInjector(inlineConf).inject(conf);
        } else {
            XConfiguration.copy(inlineConf, conf);
        }
    }
}

Example 4 with HadoopAccessorService

use of org.apache.oozie.service.HadoopAccessorService in project oozie by apache.

the class JavaActionExecutor method createAppSubmissionContext.

private ApplicationSubmissionContext createAppSubmissionContext(final ApplicationId appId, final Configuration launcherJobConf, final Context actionContext, final Configuration actionConf, final String actionName, final Credentials credentials, final Element actionXml) throws IOException, HadoopAccessorException, URISyntaxException {
    ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
    setResources(launcherJobConf, appContext);
    setPriority(launcherJobConf, appContext);
    setQueue(launcherJobConf, appContext);
    appContext.setApplicationId(appId);
    setApplicationName(actionContext, actionName, appContext);
    appContext.setApplicationType("Oozie Launcher");
    setMaxAttempts(launcherJobConf, appContext);
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    YarnACLHandler yarnACL = new YarnACLHandler(launcherJobConf);
    yarnACL.setACLs(amContainer);
    final String user = actionContext.getWorkflow().getUser();
    // Set the resources to localize
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    ClientDistributedCacheManager.determineTimestampsAndCacheVisibilities(launcherJobConf);
    MRApps.setupDistributedCache(launcherJobConf, localResources);
    // Add the Launcher and Action configs as Resources
    HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
    launcherJobConf.set(LauncherAM.OOZIE_SUBMITTER_USER, user);
    LocalResource launcherJobConfLR = has.createLocalResourceForConfigurationFile(LauncherAM.LAUNCHER_JOB_CONF_XML, user, launcherJobConf, actionContext.getAppFileSystem().getUri(), actionContext.getActionDir());
    localResources.put(LauncherAM.LAUNCHER_JOB_CONF_XML, launcherJobConfLR);
    LocalResource actionConfLR = has.createLocalResourceForConfigurationFile(LauncherAM.ACTION_CONF_XML, user, actionConf, actionContext.getAppFileSystem().getUri(), actionContext.getActionDir());
    localResources.put(LauncherAM.ACTION_CONF_XML, actionConfLR);
    amContainer.setLocalResources(localResources);
    setEnvironmentVariables(launcherJobConf, amContainer);
    List<String> vargs = createCommand(launcherJobConf, actionContext);
    setJavaOpts(launcherJobConf, actionXml, vargs);
    vargs.add(LauncherAM.class.getCanonicalName());
    vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR + ApplicationConstants.STDOUT);
    vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + Path.SEPARATOR + ApplicationConstants.STDERR);
    StringBuilder mergedCommand = new StringBuilder();
    for (CharSequence str : vargs) {
        mergedCommand.append(str).append(" ");
    }
    List<String> vargsFinal = ImmutableList.of(mergedCommand.toString());
    LOG.debug("Command to launch container for ApplicationMaster is: {0}", mergedCommand);
    amContainer.setCommands(vargsFinal);
    appContext.setAMContainerSpec(amContainer);
    // Set tokens
    if (credentials != null) {
        DataOutputBuffer dob = new DataOutputBuffer();
        credentials.writeTokenStorageToStream(dob);
        amContainer.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
    }
    appContext.setCancelTokensWhenComplete(true);
    return appContext;
}

Example 5 with HadoopAccessorService

use of org.apache.oozie.service.HadoopAccessorService in project oozie by apache.

the class FsActionExecutor method getFileSystemFor.

/**
 * @param path
 * @param context
 * @param fsConf
 * @return FileSystem
 * @throws HadoopAccessorException
 */
private FileSystem getFileSystemFor(Path path, Context context, XConfiguration fsConf) throws HadoopAccessorException {
    String user = context.getWorkflow().getUser();
    HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
    Configuration conf = has.createConfiguration(path.toUri().getAuthority());
    XConfiguration.copy(context.getProtoActionConf(), conf);
    if (fsConf != null) {
        XConfiguration.copy(fsConf, conf);
    }
    return has.createFileSystem(user, path.toUri(), conf);
}