Examples with Namespace com.google.cloud.servicedirectory.v1.Namespace used on opensource projects

Example 56 with Namespace

use of com.google.cloud.servicedirectory.v1.Namespace in project oozie by apache.

the class JavaActionExecutor method parseJobXmlAndConfiguration.

public static void parseJobXmlAndConfiguration(Context context, Element element, Path appPath, Configuration conf, boolean isLauncher) throws IOException, ActionExecutorException, HadoopAccessorException, URISyntaxException {
    Namespace ns = element.getNamespace();
    @SuppressWarnings("unchecked") Iterator<Element> it = element.getChildren("job-xml", ns).iterator();
    HashMap<String, FileSystem> filesystemsMap = new HashMap<String, FileSystem>();
    HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
    while (it.hasNext()) {
        Element e = it.next();
        String jobXml = e.getTextTrim();
        Path pathSpecified = new Path(jobXml);
        Path path = pathSpecified.isAbsolute() ? pathSpecified : new Path(appPath, jobXml);
        FileSystem fs;
        if (filesystemsMap.containsKey(path.toUri().getAuthority())) {
            fs = filesystemsMap.get(path.toUri().getAuthority());
        } else {
            if (path.toUri().getAuthority() != null) {
                fs = has.createFileSystem(context.getWorkflow().getUser(), path.toUri(), has.createConfiguration(path.toUri().getAuthority()));
            } else {
                fs = context.getAppFileSystem();
            }
            filesystemsMap.put(path.toUri().getAuthority(), fs);
        }
        Configuration jobXmlConf = new XConfiguration(fs.open(path));
        try {
            String jobXmlConfString = XmlUtils.prettyPrint(jobXmlConf).toString();
            jobXmlConfString = XmlUtils.removeComments(jobXmlConfString);
            jobXmlConfString = context.getELEvaluator().evaluate(jobXmlConfString, String.class);
            jobXmlConf = new XConfiguration(new StringReader(jobXmlConfString));
        } catch (ELEvaluationException ex) {
            throw new ActionExecutorException(ActionExecutorException.ErrorType.TRANSIENT, "EL_EVAL_ERROR", ex.getMessage(), ex);
        } catch (Exception ex) {
            context.setErrorInfo("EL_ERROR", ex.getMessage());
        }
        checkForDisallowedProps(jobXmlConf, "job-xml");
        if (isLauncher) {
            new LauncherConfigurationInjector(jobXmlConf).inject(conf);
        } else {
            XConfiguration.copy(jobXmlConf, conf);
        }
    }
    Element e = element.getChild("configuration", ns);
    if (e != null) {
        String strConf = XmlUtils.prettyPrint(e).toString();
        XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
        checkForDisallowedProps(inlineConf, "inline configuration");
        if (isLauncher) {
            new LauncherConfigurationInjector(inlineConf).inject(conf);
        } else {
            XConfiguration.copy(inlineConf, conf);
        }
    }
}

Example 57 with Namespace

use of com.google.cloud.servicedirectory.v1.Namespace in project oozie by apache.

the class JavaActionExecutor method setupLauncherConf.

Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException {
    try {
        Namespace ns = actionXml.getNamespace();
        XConfiguration launcherConf = new XConfiguration();
        // Inject action defaults for launcher
        HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
        XConfiguration actionDefaultConf = has.createActionDefaultConf(conf.get(HADOOP_YARN_RM), getType());
        new LauncherConfigurationInjector(actionDefaultConf).inject(launcherConf);
        // Inject <job-xml> and <configuration> for launcher
        try {
            parseJobXmlAndConfiguration(context, actionXml, appPath, launcherConf, true);
        } catch (HadoopAccessorException ex) {
            throw convertException(ex);
        } catch (URISyntaxException ex) {
            throw convertException(ex);
        }
        XConfiguration.copy(launcherConf, conf);
        // Inject config-class for launcher to use for action
        Element e = actionXml.getChild("config-class", ns);
        if (e != null) {
            conf.set(LauncherAMUtils.OOZIE_ACTION_CONFIG_CLASS, e.getTextTrim());
        }
        checkForDisallowedProps(launcherConf, "launcher configuration");
        return conf;
    } catch (IOException ex) {
        throw convertException(ex);
    }
}

Example 58 with Namespace

use of com.google.cloud.servicedirectory.v1.Namespace in project oozie by apache.

the class JavaActionExecutor method setJavaMain.

private void setJavaMain(Configuration actionConf, Element actionXml) {
    Namespace ns = actionXml.getNamespace();
    Element e = actionXml.getChild("main-class", ns);
    if (e != null) {
        actionConf.set(JavaMain.JAVA_MAIN_CLASS, e.getTextTrim());
    }
}

Example 59 with Namespace

use of com.google.cloud.servicedirectory.v1.Namespace in project oozie by apache.

the class JavaActionExecutor method createBaseHadoopConf.

protected Configuration createBaseHadoopConf(Context context, Element actionXml, boolean loadResources) {
    Namespace ns = actionXml.getNamespace();
    String resourceManager;
    final Element resourceManagerTag = actionXml.getChild("resource-manager", ns);
    if (resourceManagerTag != null) {
        resourceManager = resourceManagerTag.getTextTrim();
    } else {
        resourceManager = actionXml.getChild("job-tracker", ns).getTextTrim();
    }
    String nameNode = actionXml.getChild("name-node", ns).getTextTrim();
    Configuration conf = null;
    if (loadResources) {
        conf = Services.get().get(HadoopAccessorService.class).createConfiguration(resourceManager);
    } else {
        conf = new Configuration(false);
    }
    conf.set(HADOOP_USER, context.getProtoActionConf().get(WorkflowAppService.HADOOP_USER));
    conf.set(HADOOP_YARN_RM, resourceManager);
    conf.set(HADOOP_NAME_NODE, nameNode);
    conf.set("mapreduce.fileoutputcommitter.marksuccessfuljobs", "true");
    // FIXME - think about this!
    Element e = actionXml.getChild("config-class", ns);
    if (e != null) {
        conf.set(LauncherAMUtils.OOZIE_ACTION_CONFIG_CLASS, e.getTextTrim());
    }
    return conf;
}

Example 60 with Namespace

use of com.google.cloud.servicedirectory.v1.Namespace in project oozie by apache.

the class MapReduceActionExecutor method setupActionConf.

@Override
@SuppressWarnings("unchecked")
Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath) throws ActionExecutorException {
    boolean regularMR = false;
    injectConfigClass(actionConf, actionXml);
    Namespace ns = actionXml.getNamespace();
    if (actionXml.getChild("streaming", ns) != null) {
        Element streamingXml = actionXml.getChild("streaming", ns);
        String mapper = streamingXml.getChildTextTrim("mapper", ns);
        String reducer = streamingXml.getChildTextTrim("reducer", ns);
        String recordReader = streamingXml.getChildTextTrim("record-reader", ns);
        List<Element> list = (List<Element>) streamingXml.getChildren("record-reader-mapping", ns);
        String[] recordReaderMapping = new String[list.size()];
        for (int i = 0; i < list.size(); i++) {
            recordReaderMapping[i] = list.get(i).getTextTrim();
        }
        list = (List<Element>) streamingXml.getChildren("env", ns);
        String[] env = new String[list.size()];
        for (int i = 0; i < list.size(); i++) {
            env[i] = list.get(i).getTextTrim();
        }
        setStreaming(actionConf, mapper, reducer, recordReader, recordReaderMapping, env);
    } else {
        if (actionXml.getChild("pipes", ns) != null) {
            Element pipesXml = actionXml.getChild("pipes", ns);
            String map = pipesXml.getChildTextTrim("map", ns);
            String reduce = pipesXml.getChildTextTrim("reduce", ns);
            String inputFormat = pipesXml.getChildTextTrim("inputformat", ns);
            String partitioner = pipesXml.getChildTextTrim("partitioner", ns);
            String writer = pipesXml.getChildTextTrim("writer", ns);
            String program = pipesXml.getChildTextTrim("program", ns);
            PipesMain.setPipes(actionConf, map, reduce, inputFormat, partitioner, writer, program, appPath);
        } else {
            regularMR = true;
        }
    }
    actionConf = super.setupActionConf(actionConf, context, actionXml, appPath);
    setJobName(actionConf, context);
    // For "regular" (not streaming or pipes) MR jobs
    if (regularMR) {
        // Resolve uber jar path (has to be done after super because oozie.mapreduce.uber.jar is under <configuration>)
        String uberJar = actionConf.get(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR);
        if (uberJar != null) {
            if (!ConfigurationService.getBoolean(OOZIE_MAPREDUCE_UBER_JAR_ENABLE)) {
                throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "MR003", "{0} property is not allowed.  Set {1} to true in oozie-site to enable.", MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR, OOZIE_MAPREDUCE_UBER_JAR_ENABLE);
            }
            String nameNode = actionXml.getChildTextTrim("name-node", ns);
            if (nameNode != null) {
                Path uberJarPath = new Path(uberJar);
                if (uberJarPath.toUri().getScheme() == null || uberJarPath.toUri().getAuthority() == null) {
                    if (uberJarPath.isAbsolute()) {
                        // absolute path without namenode --> prepend namenode
                        Path nameNodePath = new Path(nameNode);
                        String nameNodeSchemeAuthority = nameNodePath.toUri().getScheme() + "://" + nameNodePath.toUri().getAuthority();
                        actionConf.set(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR, new Path(nameNodeSchemeAuthority + uberJarPath).toString());
                    } else {
                        // relative path --> prepend app path
                        actionConf.set(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR, new Path(appPath, uberJarPath).toString());
                    }
                }
            }
        }
    } else {
        if (actionConf.get(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR) != null) {
            log.warn("The " + MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR + " property is only applicable for MapReduce (not" + "streaming nor pipes) workflows, ignoring");
            actionConf.set(MapReduceMain.OOZIE_MAPREDUCE_UBER_JAR, "");
        }
    }
    // child job cancel delegation token for mapred action
    actionConf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true);
    return actionConf;
}