Search in sources :

Example 26 with Namespace

use of org.osate.aadl2.Namespace in project oozie by apache.

the class JavaActionExecutor method createBaseHadoopConf.

protected Configuration createBaseHadoopConf(Context context, Element actionXml, boolean loadResources) {
    Namespace ns = actionXml.getNamespace();
    String resourceManager;
    final Element resourceManagerTag = actionXml.getChild("resource-manager", ns);
    if (resourceManagerTag != null) {
        resourceManager = resourceManagerTag.getTextTrim();
    } else {
        resourceManager = actionXml.getChild("job-tracker", ns).getTextTrim();
    }
    String nameNode = actionXml.getChild("name-node", ns).getTextTrim();
    Configuration conf = null;
    if (loadResources) {
        conf = Services.get().get(HadoopAccessorService.class).createConfiguration(resourceManager);
    } else {
        conf = new Configuration(false);
    }
    conf.set(HADOOP_USER, context.getProtoActionConf().get(WorkflowAppService.HADOOP_USER));
    conf.set(HADOOP_YARN_RM, resourceManager);
    conf.set(HADOOP_NAME_NODE, nameNode);
    conf.set("mapreduce.fileoutputcommitter.marksuccessfuljobs", "true");
    // FIXME - think about this!
    Element e = actionXml.getChild("config-class", ns);
    if (e != null) {
        conf.set(LauncherAMUtils.OOZIE_ACTION_CONFIG_CLASS, e.getTextTrim());
    }
    return conf;
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) XConfiguration(org.apache.oozie.util.XConfiguration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) Element(org.jdom2.Element) Namespace(org.jdom2.Namespace)

Example 27 with Namespace

use of org.osate.aadl2.Namespace in project oozie by apache.

the class JavaActionExecutor method setJavaMain.

private void setJavaMain(Configuration actionConf, Element actionXml) {
    Namespace ns = actionXml.getNamespace();
    Element e = actionXml.getChild("main-class", ns);
    if (e != null) {
        actionConf.set(JavaMain.JAVA_MAIN_CLASS, e.getTextTrim());
    }
}
Also used : Element(org.jdom2.Element) Namespace(org.jdom2.Namespace)

Example 28 with Namespace

use of org.osate.aadl2.Namespace in project oozie by apache.

the class JavaActionExecutor method getCaptureOutput.

protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
    Element eConf = XmlUtils.parseXml(action.getConf());
    Namespace ns = eConf.getNamespace();
    Element captureOutput = eConf.getChild("capture-output", ns);
    return captureOutput != null;
}
Also used : Element(org.jdom2.Element) Namespace(org.jdom2.Namespace)

Example 29 with Namespace

use of org.osate.aadl2.Namespace in project oozie by apache.

the class GitActionExecutor method setupActionConf.

@Override
Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath) throws ActionExecutorException {
    super.setupActionConf(actionConf, context, actionXml, appPath);
    Namespace ns = actionXml.getNamespace();
    ActionConfVerifier confChecker = new ActionConfVerifier(actionConf);
    confChecker.checkTrimAndSet(GitActionExecutor.NAME_NODE, actionXml.getChild("name-node", ns));
    confChecker.checkTrimAndSet(GitActionExecutor.DESTINATION_URI, actionXml.getChild("destination-uri", ns));
    confChecker.checkTrimAndSet(GitActionExecutor.GIT_URI, actionXml.getChild("git-uri", ns));
    confChecker.trimAndSet(KEY_PATH, actionXml.getChild("key-path", ns));
    String keyPath = actionConf.get(KEY_PATH);
    if (keyPath != null && keyPath.length() > 0) {
        try {
            confChecker.verifyKeyPermissions(context.getAppFileSystem(), new Path(keyPath));
        } catch (HadoopAccessorException | URISyntaxException | IOException e) {
            throw new ActionExecutorException(ActionExecutorException.ErrorType.ERROR, "GIT011", XLog.format("Not able to verify permissions on key file {0}", keyPath), e);
        }
    }
    confChecker.trimAndSet(GIT_BRANCH, actionXml.getChild("branch", ns));
    actionConf.set(ACTION_TYPE, getType());
    actionConf.set(ACTION_NAME, GIT_ACTION_TYPE);
    return actionConf;
}
Also used : Path(org.apache.hadoop.fs.Path) HadoopAccessorException(org.apache.oozie.service.HadoopAccessorException) ActionExecutorException(org.apache.oozie.action.ActionExecutorException) URISyntaxException(java.net.URISyntaxException) IOException(java.io.IOException) Namespace(org.jdom2.Namespace)

Example 30 with Namespace

use of org.osate.aadl2.Namespace in project oozie by apache.

the class SparkActionExecutor method setupActionConf.

@Override
Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath) throws ActionExecutorException {
    actionConf = super.setupActionConf(actionConf, context, actionXml, appPath);
    Namespace ns = actionXml.getNamespace();
    String master = actionXml.getChildTextTrim("master", ns);
    actionConf.set(SPARK_MASTER, master);
    String mode = actionXml.getChildTextTrim("mode", ns);
    if (mode != null) {
        actionConf.set(SPARK_MODE, mode);
    }
    String jobName = actionXml.getChildTextTrim("name", ns);
    actionConf.set(SPARK_JOB_NAME, jobName);
    String sparkClass = actionXml.getChildTextTrim("class", ns);
    if (sparkClass != null) {
        actionConf.set(SPARK_CLASS, sparkClass);
    }
    String jarLocation = actionXml.getChildTextTrim("jar", ns);
    actionConf.set(SPARK_JAR, jarLocation);
    if (master.startsWith("yarn")) {
        String resourceManager = actionConf.get(HADOOP_YARN_RM);
        Properties sparkConfig = Services.get().get(SparkConfigurationService.class).getSparkConfig(resourceManager);
        if (!sparkConfig.isEmpty()) {
            try (final StringWriter sw = new StringWriter()) {
                sparkConfig.store(sw, "Generated by Oozie server SparkActionExecutor");
                actionConf.set(SPARK_DEFAULT_OPTS, sw.toString());
            } catch (IOException e) {
                LOG.warn("Could not propagate Spark default configuration!", e);
            }
        }
    }
    String sparkOpts = actionXml.getChildTextTrim("spark-opts", ns);
    if (!Strings.isNullOrEmpty(sparkOpts)) {
        actionConf.set(SPARK_OPTS, sparkOpts.toString().trim());
    }
    // Setting if SparkMain should setup hadoop config *-site.xml
    boolean setupHadoopConf = actionConf.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, ConfigurationService.getBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR));
    actionConf.setBoolean(CONF_OOZIE_SPARK_SETUP_HADOOP_CONF_DIR, setupHadoopConf);
    return actionConf;
}
Also used : StringWriter(java.io.StringWriter) IOException(java.io.IOException) Properties(java.util.Properties) Namespace(org.jdom2.Namespace) SparkConfigurationService(org.apache.oozie.service.SparkConfigurationService)

Aggregations

Namespace (org.jdom2.Namespace)151 Element (org.jdom2.Element)123 IOException (java.io.IOException)29 ArrayList (java.util.ArrayList)23 HashMap (java.util.HashMap)21 List (java.util.List)21 Document (org.jdom2.Document)21 XConfiguration (org.apache.oozie.util.XConfiguration)19 Attribute (org.jdom2.Attribute)17 Configuration (org.apache.hadoop.conf.Configuration)15 StringReader (java.io.StringReader)14 ActionExecutorException (org.apache.oozie.action.ActionExecutorException)14 Namespace.getNamespace (org.jdom2.Namespace.getNamespace)14 MigrationReport (com.mulesoft.tools.migration.step.category.MigrationReport)13 Map (java.util.Map)12 Path (org.apache.hadoop.fs.Path)12 JDOMException (org.jdom2.JDOMException)12 SAXBuilder (org.jdom2.input.SAXBuilder)11 File (java.io.File)10 AbstractApplicationModelMigrationStep (com.mulesoft.tools.migration.step.AbstractApplicationModelMigrationStep)9