Examples with XConfiguration org.apache.oozie.util.XConfiguration used on opensource projects

Example 11 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestShareLibService method testAddShareLib_pig_withVersion.

@Test
public void testAddShareLib_pig_withVersion() throws Exception {
    setShipLauncherInOozieConfig();
    FileSystem fs = getFileSystem();
    Date time = new Date(System.currentTimeMillis());
    Path basePath = new Path(getOozieConfig().get(WorkflowAppService.SYSTEM_LIB_PATH));
    Path libpath = new Path(basePath, ShareLibService.SHARE_LIB_PREFIX + ShareLibService.dateFormat.format(time));
    fs.mkdirs(libpath);
    Path pigPath = new Path(libpath.toString() + Path.SEPARATOR + "pig");
    Path pigPath1 = new Path(libpath.toString() + Path.SEPARATOR + "pig_9");
    Path pigPath2 = new Path(libpath.toString() + Path.SEPARATOR + "pig_10");
    fs.mkdirs(pigPath);
    fs.mkdirs(pigPath1);
    fs.mkdirs(pigPath2);
    createFiles(libpath.toString() + Path.SEPARATOR + "pig_10" + Path.SEPARATOR + "pig-10.jar");
    services.init();
    String actionXml = "<pig>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<property><name>oozie.action.sharelib.for.pig</name><value>pig_10</value></property>" + "</pig>";
    Element eActionXml = XmlUtils.parseXml(actionXml);
    XConfiguration protoConf = new XConfiguration();
    protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
    WorkflowJobBean wfj = new WorkflowJobBean();
    protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
    wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
    wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());
    Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
    PigActionExecutor ae = new PigActionExecutor();
    Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
    jobConf.set("oozie.action.sharelib.for.pig", "pig_10");
    ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
    verifyFilesInDistributedCache(DistributedCache.getCacheFiles(jobConf), "MyPig.jar", "MyOozie.jar", "pig-10.jar");
}

Example 12 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestShareLibService method setupSharelibConf.

private void setupSharelibConf(final FileSystem fs, final String file, final String tag, Properties prop) throws IOException, ServiceException {
    Configuration conf = getOozieConfig();
    setShipLauncherInOozieConfig();
    setShareLibMappingFileInOozieConfig(fs, conf);
    XConfiguration hiveConf = new XConfiguration();
    hiveConf.set(tag + "-sharelib-test", "test");
    createDirs(getFileSystem(), new Path(SHARELIB_PATH));
    FSDataOutputStream out = getFileSystem().create(new Path(SHARELIB_PATH, file), true);
    PrintWriter bufOut = new PrintWriter(out);
    bufOut.write(hiveConf.toXmlString(false));
    bufOut.close();
    createTestShareLibMappingFile(getFileSystem(), prop);
    services.init();
}

Example 13 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestShareLibService method testConfFileAddedToActionConf.

@Test
public void testConfFileAddedToActionConf() throws Exception {
    try {
        XConfiguration protoConf = new XConfiguration();
        protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
        WorkflowJobBean wfj = new WorkflowJobBean();
        protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
        wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
        wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());
        Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
        // Test hive-site.xml in sharelib cache
        setupSharelibConf("hive-site.xml", "oozie.hive_conf");
        ShareLibService shareLibService = services.get(ShareLibService.class);
        assertEquals(shareLibService.getShareLibConfigMap().get("hive_conf").values().size(), 1);
        assertEquals(shareLibService.getShareLibConfigMap().get("hive_conf").keySet().toArray(new Path[] {})[0].getName(), "hive-site.xml");
        // Test hive-site.xml not in distributed cache
        setupSharelibConf("hive-site.xml", "oozie.hive_conf");
        String actionXml = "<hive>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<script>test</script>" + "</hive>";
        Element eActionXml = XmlUtils.parseXml(actionXml);
        HiveActionExecutor ae = new HiveActionExecutor();
        Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
        Configuration actionConf = ae.createBaseHadoopConf(context, eActionXml);
        jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
        URI[] cacheFiles = DistributedCache.getCacheFiles(actionConf);
        String cacheFilesStr = Arrays.toString(cacheFiles);
        assertFalse(cacheFilesStr.contains("hive-site.xml"));
        // Test hive-site.xml property in jobconf with linkname
        jobConf = ae.createBaseHadoopConf(context, eActionXml);
        Properties prop = new Properties();
        actionConf = ae.createBaseHadoopConf(context, eActionXml);
        prop.put("oozie.hive_conf", TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml#hive-site.xml");
        setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
        jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
        assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
        // Test hive-site.xml property in jobconf with linkname
        // and with hdfs path
        prop = new Properties();
        jobConf = ae.createBaseHadoopConf(context, eActionXml);
        actionConf = ae.createBaseHadoopConf(context, eActionXml);
        prop.put("oozie.hive_conf", "hdfs://" + TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml#hive-site.xml");
        setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
        jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
        assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
        cacheFiles = DistributedCache.getCacheFiles(actionConf);
        cacheFilesStr = Arrays.toString(cacheFiles);
        assertFalse(cacheFilesStr.contains("hive-site.xml"));
        // Test hive-site.xml property in jobconf with non hdfs path
        prop = new Properties();
        jobConf = ae.createBaseHadoopConf(context, eActionXml);
        actionConf = ae.createBaseHadoopConf(context, eActionXml);
        prop.put("oozie.hive_conf", TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml");
        setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
        jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
        assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
        cacheFiles = DistributedCache.getCacheFiles(actionConf);
        cacheFilesStr = Arrays.toString(cacheFiles);
        assertFalse(cacheFilesStr.contains("hive-site.xml"));
        // Test hive-site.xml property in jobconf with non hdfs path with
        // link name
        prop = new Properties();
        jobConf = ae.createBaseHadoopConf(context, eActionXml);
        actionConf = ae.createBaseHadoopConf(context, eActionXml);
        prop.put("oozie.hive_conf", TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml#hive-site.xml");
        setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
        jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
        assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
        cacheFiles = DistributedCache.getCacheFiles(actionConf);
        cacheFilesStr = Arrays.toString(cacheFiles);
        assertFalse(cacheFilesStr.contains("hive-site.xml"));
    } finally {
        getFileSystem().delete(new Path(SHARELIB_PATH), true);
    }
}

Example 14 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestAuthorizationService method _testAuthorizationService.

private void _testAuthorizationService(boolean useDefaultGroup) throws Exception {
    init(useDefaultGroup, true);
    Reader reader = IOUtils.getResourceAsReader("wf-ext-schema-valid.xml", -1);
    Writer writer = new FileWriter(new File(getTestCaseDir(), "workflow.xml"));
    IOUtils.copyCharStream(reader, writer);
    final DagEngine engine = new DagEngine(getTestUser());
    Configuration jobConf = new XConfiguration();
    jobConf.set(OozieClient.APP_PATH, getTestCaseFileUri("workflow.xml"));
    jobConf.set(OozieClient.USER_NAME, getTestUser());
    if (useDefaultGroup) {
        jobConf.set(OozieClient.GROUP_NAME, getTestGroup());
    } else {
        jobConf.set(OozieClient.GROUP_NAME, getTestGroup() + ",foogrp");
    }
    jobConf.set(OozieClient.LOG_TOKEN, "t");
    jobConf.set("external-status", "ok");
    jobConf.set("signal-value", "based_on_action_status");
    final String jobId = engine.submitJob(jobConf, true);
    HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
    URI uri = getFileSystem().getUri();
    Configuration fsConf = has.createConfiguration(uri.getAuthority());
    FileSystem fileSystem = has.createFileSystem(getTestUser(), uri, fsConf);
    Path path = new Path(fileSystem.getWorkingDirectory(), UUID.randomUUID().toString());
    Path fsTestDir = fileSystem.makeQualified(path);
    System.out.println(XLog.format("Setting FS testcase work dir[{0}]", fsTestDir));
    fileSystem.delete(fsTestDir, true);
    if (!fileSystem.mkdirs(path)) {
        throw new IOException(XLog.format("Could not create FS testcase dir [{0}]", fsTestDir));
    }
    String appPath = fsTestDir.toString() + "/app";
    Path jobXmlPath = new Path(appPath, "workflow.xml");
    fileSystem.create(jobXmlPath).close();
    fileSystem.setOwner(jobXmlPath, getTestUser(), getTestGroup());
    FsPermission permissions = new FsPermission(FsAction.READ_WRITE, FsAction.READ, FsAction.NONE);
    fileSystem.setPermission(jobXmlPath, permissions);
    AuthorizationService as = services.get(AuthorizationService.class);
    assertNotNull(as);
    as.authorizeForGroup(getTestUser(), getTestGroup());
    assertNotNull(as.getDefaultGroup(getTestUser()));
    as.authorizeForApp(getTestUser2(), getTestGroup(), appPath, jobConf);
    try {
        as.authorizeForApp(getTestUser3(), getTestGroup(), appPath, jobConf);
        fail();
    } catch (AuthorizationException ex) {
    }
    as.authorizeForJob(getTestUser(), jobId, false);
    as.authorizeForJob(getTestUser(), jobId, true);
    if (!useDefaultGroup) {
        as.authorizeForJob("foo", jobId, true);
    }
    try {
        as.authorizeForJob("bar", jobId, true);
        fail();
    } catch (AuthorizationException ex) {
    }
}

Example 15 with XConfiguration

use of org.apache.oozie.util.XConfiguration in project oozie by apache.

the class TestHadoopAccessorService method testActionConfigurations.

public void testActionConfigurations() throws Exception {
    Services services = Services.get();
    HadoopAccessorService has = services.get(HadoopAccessorService.class);
    assertNotNull(has);
    XConfiguration conf = has.createActionDefaultConf("jt", "action");
    assertNotNull(conf);
    // Check that the param only in default.xml is still present
    assertEquals("default.bar", conf.get("default.foo"));
    // And a property that is present in one of the conf files in default dir
    assertEquals("default.bus", conf.get("default.car"));
    // Check that a default param is overridden by one of the action config files
    assertEquals("action.bar", conf.get("action.foo"));
    // Check that params from <action-dir>/action/conf files is still present
    assertEquals("action.car", conf.get("action.boo"));
    assertEquals("action.carcar", conf.get("oozie.launcher.action.booboo"));
    /*
            Check precedence - Order of precedence - 0 is the lowest.   Parameters in files of
            lower precedence will be overridden by redefinitions in higher precedence files.

            0 - All files in defaultdir/*.xml (sorted by lexical name)
               Files with names lexically lower have lesser precedence than the following ones.
            1 - default.xml
            2 - All files in actiondir/*.xml (sort by lexical name)
               Files with names lexically lower have lesser precedence than the following ones
            3 - action.xml
         */
    assertEquals("100", conf.get("action.testprop"));
    assertEquals("1", conf.get("default.testprop"));
    assertEquals("org.apache.log4j.ConsoleAppender", conf.get("log4j.appender.oozie"));
    assertEquals("NONE, null", conf.get("log4j.logger.a"));
}