Examples with WorkflowAppService org.apache.oozie.service.WorkflowAppService used on opensource projects

Example 6 with WorkflowAppService

use of org.apache.oozie.service.WorkflowAppService in project oozie by apache.

the class OozieSharelibCLI method run.

public synchronized int run(String[] args) throws Exception {
    if (used) {
        throw new IllegalStateException("CLI instance already used");
    }
    used = true;
    CLIParser parser = new CLIParser("oozie-setup.sh", HELP_INFO);
    String oozieHome = System.getProperty(OOZIE_HOME);
    parser.addCommand(HELP_CMD, "", "display usage for all commands or specified command", new Options(), false);
    parser.addCommand(CREATE_CMD, "", "create a new timestamped version of oozie sharelib", createUpgradeOptions(CREATE_CMD), false);
    parser.addCommand(UPGRADE_CMD, "", "[deprecated][use command \"create\" to create new version]   upgrade oozie sharelib \n", createUpgradeOptions(UPGRADE_CMD), false);
    try {
        final CLIParser.Command command = parser.parse(args);
        String sharelibAction = command.getName();
        if (sharelibAction.equals(HELP_CMD)) {
            parser.showHelp(command.getCommandLine());
            return 0;
        }
        if (!command.getCommandLine().hasOption(FS_OPT)) {
            throw new Exception("-fs option must be specified");
        }
        int threadPoolSize = Integer.valueOf(command.getCommandLine().getOptionValue(CONCURRENCY_OPT, "1"));
        File srcFile = null;
        // Check whether user provided locallib
        if (command.getCommandLine().hasOption(LIB_OPT)) {
            srcFile = new File(command.getCommandLine().getOptionValue(LIB_OPT));
        } else {
            // Since user did not provide locallib, find the default one under oozie home dir
            Collection<File> files = FileUtils.listFiles(new File(oozieHome), new WildcardFileFilter("oozie-sharelib*.tar.gz"), null);
            if (files.size() > 1) {
                throw new IOException("more than one sharelib tar found at " + oozieHome);
            }
            if (files.isEmpty()) {
                throw new IOException("default sharelib tar not found in oozie home dir: " + oozieHome);
            }
            srcFile = files.iterator().next();
        }
        File temp = File.createTempFile("oozie", ".dir");
        temp.delete();
        temp.mkdir();
        temp.deleteOnExit();
        // Check whether the lib is a tar file or folder
        if (!srcFile.isDirectory()) {
            FileUtil.unTar(srcFile, temp);
            srcFile = new File(temp.toString() + "/share/lib");
        } else {
            // Get the lib directory since it's a folder
            srcFile = new File(srcFile, "lib");
        }
        String hdfsUri = command.getCommandLine().getOptionValue(FS_OPT);
        Path srcPath = new Path(srcFile.toString());
        Services services = new Services();
        services.getConf().set(Services.CONF_SERVICE_CLASSES, "org.apache.oozie.service.LiteWorkflowAppService, org.apache.oozie.service.HadoopAccessorService");
        services.getConf().set(Services.CONF_SERVICE_EXT_CLASSES, "");
        services.init();
        WorkflowAppService lwas = services.get(WorkflowAppService.class);
        HadoopAccessorService has = services.get(HadoopAccessorService.class);
        Path dstPath = lwas.getSystemLibPath();
        URI uri = new Path(hdfsUri).toUri();
        Configuration fsConf = has.createConfiguration(uri.getAuthority());
        FileSystem fs = FileSystem.get(uri, fsConf);
        if (!fs.exists(dstPath)) {
            fs.mkdirs(dstPath);
        }
        ECPolicyDisabler.tryDisableECPolicyForPath(fs, dstPath);
        if (sharelibAction.equals(CREATE_CMD) || sharelibAction.equals(UPGRADE_CMD)) {
            dstPath = new Path(dstPath.toString() + Path.SEPARATOR + SHARE_LIB_PREFIX + getTimestampDirectory());
        }
        System.out.println("the destination path for sharelib is: " + dstPath);
        if (!srcFile.exists()) {
            throw new IOException(srcPath + " cannot be found");
        }
        if (threadPoolSize > 1) {
            concurrentCopyFromLocal(fs, threadPoolSize, srcFile, dstPath);
        } else {
            fs.copyFromLocalFile(false, srcPath, dstPath);
        }
        services.destroy();
        FileUtils.deleteDirectory(temp);
        return 0;
    } catch (ParseException ex) {
        System.err.println("Invalid sub-command: " + ex.getMessage());
        System.err.println();
        System.err.println(parser.shortHelp());
        return 1;
    } catch (Exception ex) {
        logError(ex.getMessage(), ex);
        return 1;
    }
}

Example 7 with WorkflowAppService

use of org.apache.oozie.service.WorkflowAppService in project oozie by apache.

the class SubmitHttpXCommand method execute.

/* (non-Javadoc)
     * @see org.apache.oozie.command.XCommand#execute()
     */
@Override
protected String execute() throws CommandException {
    InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    try {
        XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
        String wfXml = getWorkflowXml(conf);
        LOG.debug("workflow xml created on the server side is :\n");
        LOG.debug(wfXml);
        WorkflowApp app = wps.parseDef(wfXml, conf);
        XConfiguration protoActionConf = wps.createProtoActionConf(conf, false);
        WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
        PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
        // Resolving all variables in the job properties.
        // This ensures the Hadoop Configuration semantics is preserved.
        XConfiguration resolvedVarsConf = new XConfiguration();
        for (Map.Entry<String, String> entry : conf) {
            resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
        }
        conf = resolvedVarsConf;
        WorkflowInstance wfInstance;
        try {
            wfInstance = workflowLib.createInstance(app, conf);
        } catch (WorkflowException e) {
            throw new StoreException(e);
        }
        Configuration conf = wfInstance.getConf();
        WorkflowJobBean workflow = new WorkflowJobBean();
        workflow.setId(wfInstance.getId());
        workflow.setAppName(app.getName());
        workflow.setAppPath(conf.get(OozieClient.APP_PATH));
        workflow.setConf(XmlUtils.prettyPrint(conf).toString());
        workflow.setProtoActionConf(protoActionConf.toXmlString());
        workflow.setCreatedTime(new Date());
        workflow.setLastModifiedTime(new Date());
        workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
        workflow.setStatus(WorkflowJob.Status.PREP);
        workflow.setRun(0);
        workflow.setUser(conf.get(OozieClient.USER_NAME));
        workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
        workflow.setWorkflowInstance(wfInstance);
        workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
        LogUtils.setLogInfo(workflow);
        JPAService jpaService = Services.get().get(JPAService.class);
        if (jpaService != null) {
            jpaService.execute(new WorkflowJobInsertJPAExecutor(workflow));
        } else {
            LOG.error(ErrorCode.E0610);
            return null;
        }
        return workflow.getId();
    } catch (WorkflowException ex) {
        throw new CommandException(ex);
    } catch (Exception ex) {
        throw new CommandException(ErrorCode.E0803, ex.getMessage(), ex);
    }
}

Example 8 with WorkflowAppService

use of org.apache.oozie.service.WorkflowAppService in project oozie by apache.

the class TestJavaActionExecutor method testAddActionShareLib.

@Test
public void testAddActionShareLib() throws Exception {
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    Path systemLibPath = new Path(wps.getSystemLibPath(), ShareLibService.SHARE_LIB_PREFIX + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()).toString());
    Path javaShareLibPath = new Path(systemLibPath, "java");
    getFileSystem().mkdirs(javaShareLibPath);
    Path jar1Path = new Path(javaShareLibPath, "jar1.jar");
    getFileSystem().create(jar1Path).close();
    Path jar2Path = new Path(javaShareLibPath, "jar2.jar");
    getFileSystem().create(jar2Path).close();
    Path hcatShareLibPath = new Path(systemLibPath, "hcat");
    getFileSystem().mkdirs(hcatShareLibPath);
    Path jar3Path = new Path(hcatShareLibPath, "jar3.jar");
    getFileSystem().create(jar3Path).close();
    Path jar4Path = new Path(hcatShareLibPath, "jar4.jar");
    getFileSystem().create(jar4Path).close();
    Path otherShareLibPath = new Path(systemLibPath, "other");
    getFileSystem().mkdirs(otherShareLibPath);
    Path jar5Path = new Path(otherShareLibPath, "jar5.jar");
    getFileSystem().create(jar5Path).close();
    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>MAIN-CLASS</main-class>" + "</java>";
    Element eActionXml = XmlUtils.parseXml(actionXml);
    Context context = createContext(actionXml, null);
    Services.get().setService(ShareLibService.class);
    // Test oozie server action sharelib setting
    WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
    XConfiguration wfConf = new XConfiguration();
    wfConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
    wfConf.set(OozieClient.APP_PATH, new Path(getAppPath(), "workflow.xml").toString());
    wfConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
    workflow.setConf(XmlUtils.prettyPrint(wfConf).toString());
    ConfigurationService.set("oozie.action.sharelib.for.java", "java,hcat");
    JavaActionExecutor ae = new JavaActionExecutor();
    Configuration jobConf = ae.createBaseHadoopConf(context, eActionXml);
    ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
    try {
        ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
        fail();
    } catch (ActionExecutorException aee) {
        assertEquals("EJ001", aee.getErrorCode());
        assertEquals("Could not locate Oozie sharelib", aee.getMessage());
    }
    Path launcherPath = new Path(systemLibPath, "oozie");
    getFileSystem().mkdirs(launcherPath);
    Path jar6Path = new Path(launcherPath, "jar6.jar");
    getFileSystem().create(jar6Path).close();
    Services.get().get(ShareLibService.class).updateShareLib();
    ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
    URI[] cacheFiles = DistributedCache.getCacheFiles(jobConf);
    String cacheFilesStr = Arrays.toString(cacheFiles);
    assertTrue(cacheFilesStr.contains(jar1Path.toString()));
    assertTrue(cacheFilesStr.contains(jar2Path.toString()));
    assertTrue(cacheFilesStr.contains(jar3Path.toString()));
    assertTrue(cacheFilesStr.contains(jar4Path.toString()));
    assertFalse(cacheFilesStr.contains(jar5Path.toString()));
    assertTrue(cacheFilesStr.contains(jar6Path.toString()));
    // Test per workflow action sharelib setting
    workflow = (WorkflowJobBean) context.getWorkflow();
    wfConf = new XConfiguration();
    wfConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
    wfConf.set(OozieClient.APP_PATH, new Path(getAppPath(), "workflow.xml").toString());
    wfConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
    wfConf.set("oozie.action.sharelib.for.java", "other,hcat");
    workflow.setConf(XmlUtils.prettyPrint(wfConf).toString());
    ConfigurationService.set("oozie.action.sharelib.for.java", "java");
    ae = new JavaActionExecutor();
    jobConf = ae.createBaseHadoopConf(context, eActionXml);
    ae.setupLauncherConf(jobConf, eActionXml, getAppPath(), context);
    ae.setLibFilesArchives(context, eActionXml, getAppPath(), jobConf);
    cacheFiles = DistributedCache.getCacheFiles(jobConf);
    cacheFilesStr = Arrays.toString(cacheFiles);
    // The oozie server setting should have been overridden by workflow setting
    assertFalse(cacheFilesStr.contains(jar1Path.toString()));
    assertFalse(cacheFilesStr.contains(jar2Path.toString()));
    assertTrue(cacheFilesStr.contains(jar3Path.toString()));
    assertTrue(cacheFilesStr.contains(jar4Path.toString()));
    assertTrue(cacheFilesStr.contains(jar5Path.toString()));
    assertTrue(cacheFilesStr.contains(jar6Path.toString()));
}

Example 9 with WorkflowAppService

use of org.apache.oozie.service.WorkflowAppService in project oozie by apache.

the class TestJavaActionExecutor method testAddShareLibSchemeAndAuthority.

public void testAddShareLibSchemeAndAuthority() throws Exception {
    JavaActionExecutor ae = new JavaActionExecutor() {

        @Override
        public String getDefaultShareLibName(Element actionXml) {
            return "java-action-executor";
        }
    };
    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNode2Uri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "</java>";
    Element eActionXml = XmlUtils.parseXml(actionXml);
    Context context = createContext(actionXml, null);
    // Set sharelib to a relative path (i.e. no scheme nor authority)
    Services.get().destroy();
    setSystemProperty(WorkflowAppService.SYSTEM_LIB_PATH, "/user/" + getTestUser() + "/share/");
    new Services().init();
    // Create the dir
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    Path systemLibPath = new Path(wps.getSystemLibPath(), ShareLibService.SHARE_LIB_PREFIX + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()).toString());
    Path javaShareLibPath = new Path(systemLibPath, "java-action-executor");
    getFileSystem().mkdirs(javaShareLibPath);
    Services.get().setService(ShareLibService.class);
    Configuration conf = ae.createBaseHadoopConf(context, eActionXml);
    // Despite systemLibPath is not fully qualified and the action refers to the
    // second namenode the next line won't throw exception because default fs is used
    ae.addShareLib(conf, new String[] { "java-action-executor" });
    // Set sharelib to a full path (i.e. include scheme and authority)
    Services.get().destroy();
    setSystemProperty(WorkflowAppService.SYSTEM_LIB_PATH, getNameNodeUri() + "/user/" + getTestUser() + "/share/");
    new Services().init();
    Services.get().setService(ShareLibService.class);
    conf = ae.createBaseHadoopConf(context, eActionXml);
    // The next line should not throw an Exception because it will get the scheme and authority from the sharelib path
    ae.addShareLib(conf, new String[] { "java-action-executor" });
}

Example 10 with WorkflowAppService

use of org.apache.oozie.service.WorkflowAppService in project oozie by apache.

the class TestJavaActionExecutor method testActionShareLibWithNonDefaultNamenode.

public void testActionShareLibWithNonDefaultNamenode() throws Exception {
    WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
    Path systemLibPath = new Path(wps.getSystemLibPath(), ShareLibService.SHARE_LIB_PREFIX + new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()).toString());
    File jarFile = IOUtils.createJar(new File(getTestCaseDir()), "sourcejar.jar", LauncherMainTester.class);
    InputStream is = new FileInputStream(jarFile);
    Path javaShareLibPath = new Path(systemLibPath, "java");
    getFileSystem().mkdirs(javaShareLibPath);
    Path jar1Path = new Path(javaShareLibPath, "jar1.jar");
    OutputStream os1 = getFileSystem().create(jar1Path);
    IOUtils.copyStream(is, os1);
    Path jar2Path = new Path(javaShareLibPath, "jar2.jar");
    OutputStream os2 = getFileSystem().create(jar2Path);
    // is not resetable
    is = new FileInputStream(jarFile);
    IOUtils.copyStream(is, os2);
    Path launcherPath = new Path(systemLibPath, "oozie");
    getFileSystem().mkdirs(launcherPath);
    Path jar3Path = new Path(launcherPath, "jar3.jar");
    OutputStream os3 = getFileSystem().create(jar3Path);
    is = new FileInputStream(jarFile);
    IOUtils.copyStream(is, os3);
    String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNode2Uri() + "</name-node>" + "<job-xml>job.xml</job-xml>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "</java>";
    XConfiguration jConf = new XConfiguration();
    jConf.set("p", "v");
    OutputStream os = getFileSystem().create(new Path(getAppPath(), "job.xml"));
    jConf.writeXml(os);
    os.close();
    Context context = createContext(actionXml, null);
    Services.get().setService(ShareLibService.class);
    // Test oozie server action sharelib setting
    WorkflowJobBean workflow = (WorkflowJobBean) context.getWorkflow();
    XConfiguration wfConf = new XConfiguration();
    wfConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
    wfConf.set(OozieClient.APP_PATH, new Path(getAppPath(), "workflow.xml").toString());
    wfConf.setBoolean(OozieClient.USE_SYSTEM_LIBPATH, true);
    workflow.setConf(XmlUtils.prettyPrint(wfConf).toString());
    ConfigurationService.set("oozie.action.sharelib.for.java", "java");
    final String runningJob = submitAction(context);
    waitUntilYarnAppDoneAndAssertSuccess(runningJob);
}