Examples with HiveActionExecutor org.apache.oozie.action.hadoop.HiveActionExecutor used on opensource projects

Example 1 with HiveActionExecutor

use of org.apache.oozie.action.hadoop.HiveActionExecutor in project oozie by apache.

the class TestShareLibService method testConfFileAddedToActionConf.

@Test
public void testConfFileAddedToActionConf() throws Exception {
    try {
        XConfiguration protoConf = new XConfiguration();
        protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
        WorkflowJobBean wfj = new WorkflowJobBean();
        protoConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
        wfj.setProtoActionConf(XmlUtils.prettyPrint(protoConf).toString());
        wfj.setConf(XmlUtils.prettyPrint(protoConf).toString());
        Context context = new TestJavaActionExecutor().new Context(wfj, new WorkflowActionBean());
        // Test hive-site.xml in sharelib cache
        setupSharelibConf("hive-site.xml", "oozie.hive_conf");
        ShareLibService shareLibService = services.get(ShareLibService.class);
        assertEquals(shareLibService.getShareLibConfigMap().get("hive_conf").values().size(), 1);
        assertEquals(shareLibService.getShareLibConfigMap().get("hive_conf").keySet().toArray(new Path[] {})[0].getName(), "hive-site.xml");
        // Test hive-site.xml not in distributed cache
        setupSharelibConf("hive-site.xml", "oozie.hive_conf");
        String actionXml = "<hive>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<script>test</script>" + "</hive>";
        Element eActionXml = XmlUtils.parseXml(actionXml);
        HiveActionExecutor ae = new HiveActionExecutor();
        Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
        Configuration actionConf = ae.createBaseHadoopConf(context, eActionXml);
        jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
        URI[] cacheFiles = DistributedCache.getCacheFiles(actionConf);
        String cacheFilesStr = Arrays.toString(cacheFiles);
        assertFalse(cacheFilesStr.contains("hive-site.xml"));
        // Test hive-site.xml property in jobconf with linkname
        jobConf = ae.createBaseHadoopConf(context, eActionXml);
        Properties prop = new Properties();
        actionConf = ae.createBaseHadoopConf(context, eActionXml);
        prop.put("oozie.hive_conf", TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml#hive-site.xml");
        setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
        jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
        assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
        // Test hive-site.xml property in jobconf with linkname
        // and with hdfs path
        prop = new Properties();
        jobConf = ae.createBaseHadoopConf(context, eActionXml);
        actionConf = ae.createBaseHadoopConf(context, eActionXml);
        prop.put("oozie.hive_conf", "hdfs://" + TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml#hive-site.xml");
        setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
        jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
        assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
        cacheFiles = DistributedCache.getCacheFiles(actionConf);
        cacheFilesStr = Arrays.toString(cacheFiles);
        assertFalse(cacheFilesStr.contains("hive-site.xml"));
        // Test hive-site.xml property in jobconf with non hdfs path
        prop = new Properties();
        jobConf = ae.createBaseHadoopConf(context, eActionXml);
        actionConf = ae.createBaseHadoopConf(context, eActionXml);
        prop.put("oozie.hive_conf", TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml");
        setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
        jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
        assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
        cacheFiles = DistributedCache.getCacheFiles(actionConf);
        cacheFilesStr = Arrays.toString(cacheFiles);
        assertFalse(cacheFilesStr.contains("hive-site.xml"));
        // Test hive-site.xml property in jobconf with non hdfs path with
        // link name
        prop = new Properties();
        jobConf = ae.createBaseHadoopConf(context, eActionXml);
        actionConf = ae.createBaseHadoopConf(context, eActionXml);
        prop.put("oozie.hive_conf", TEST_HDFS_HOME + SHARELIB_PATH + "hive-site.xml#hive-site.xml");
        setupSharelibConf("hive-site.xml", "oozie.hive_conf", prop);
        jobConf.set("oozie.action.sharelib.for.hive", "hive_conf");
        ae.setLibFilesArchives(context, eActionXml, new Path("hdfs://dummyAppPath"), jobConf);
        assertEquals(jobConf.get("oozie.hive_conf-sharelib-test"), "test");
        cacheFiles = DistributedCache.getCacheFiles(actionConf);
        cacheFilesStr = Arrays.toString(cacheFiles);
        assertFalse(cacheFilesStr.contains("hive-site.xml"));
    } finally {
        getFileSystem().delete(new Path(SHARELIB_PATH), true);
    }
}