Search in sources :

Example 6 with JobProcess

use of org.codelibs.fess.util.JobProcess in project fess by codelibs.

the class CrawlJob method executeCrawler.

protected void executeCrawler() {
    final List<String> cmdList = new ArrayList<>();
    final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
    final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    cmdList.add(fessConfig.getJavaCommandPath());
    // -cp
    cmdList.add("-cp");
    final StringBuilder buf = new StringBuilder(100);
    ResourceUtil.getOverrideConfPath().ifPresent(p -> {
        buf.append(p);
        buf.append(cpSeparator);
    });
    final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
    if (StringUtil.isNotBlank(confPath)) {
        buf.append(confPath);
        buf.append(cpSeparator);
    }
    // WEB-INF/env/crawler/resources
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("env");
    buf.append(File.separator);
    buf.append(getExecuteType());
    buf.append(File.separator);
    buf.append("resources");
    buf.append(cpSeparator);
    // WEB-INF/classes
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("classes");
    // target/classes
    final String userDir = System.getProperty("user.dir");
    final File targetDir = new File(userDir, "target");
    final File targetClassesDir = new File(targetDir, "classes");
    if (targetClassesDir.isDirectory()) {
        buf.append(cpSeparator);
        buf.append(targetClassesDir.getAbsolutePath());
    }
    // WEB-INF/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib" + File.separator);
    // WEB-INF/env/crawler/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
    // WEB-INF/plugin
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin" + File.separator);
    final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
    if (targetLibDir.isDirectory()) {
        appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
    }
    cmdList.add(buf.toString());
    if (useLocalFesen) {
        final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
        if (StringUtil.isNotBlank(httpAddress)) {
            cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
        }
    }
    final String systemLastaEnv = System.getProperty("lasta.env");
    if (StringUtil.isNotBlank(systemLastaEnv)) {
        if ("web".equals(systemLastaEnv)) {
            cmdList.add("-Dlasta.env=" + getExecuteType());
        } else {
            cmdList.add("-Dlasta.env=" + systemLastaEnv);
        }
    } else if (StringUtil.isNotBlank(lastaEnv)) {
        cmdList.add("-Dlasta.env=" + lastaEnv);
    } else {
        cmdList.add("-Dlasta.env=" + getExecuteType());
    }
    addFessConfigProperties(cmdList);
    addFessSystemProperties(cmdList);
    addSystemProperty(cmdList, Constants.FESS_CONF_PATH, null, null);
    cmdList.add("-Dfess." + getExecuteType() + ".process=true");
    cmdList.add("-Dfess.log.path=" + (logFilePath != null ? logFilePath : systemHelper.getLogFilePath()));
    addSystemProperty(cmdList, "fess.log.name", "fess-" + getExecuteType(), "-" + getExecuteType());
    if (logLevel == null) {
        addSystemProperty(cmdList, "fess.log.level", null, null);
    } else {
        cmdList.add("-Dfess.log.level=" + logLevel);
        if ("debug".equalsIgnoreCase(logLevel)) {
            cmdList.add("-Dorg.apache.tika.service.error.warn=true");
        }
    }
    stream(fessConfig.getJvmCrawlerOptionsAsArray()).of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
    File ownTmpDir = null;
    final String tmpDir = System.getProperty("java.io.tmpdir");
    if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
        ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
        if (ownTmpDir.mkdirs()) {
            cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
            cmdList.add("-Dpdfbox.fontcache=" + ownTmpDir.getAbsolutePath());
        } else {
            ownTmpDir = null;
        }
    }
    cmdList.add(ComponentUtil.getThumbnailManager().getThumbnailPathOption());
    if (!jvmOptions.isEmpty()) {
        jvmOptions.stream().filter(StringUtil::isNotBlank).forEach(cmdList::add);
    }
    cmdList.add(Crawler.class.getCanonicalName());
    cmdList.add("--sessionId");
    cmdList.add(sessionId);
    cmdList.add("--name");
    cmdList.add(namespace);
    if (webConfigIds != null && webConfigIds.length > 0) {
        cmdList.add("-w");
        cmdList.add(StringUtils.join(webConfigIds, ','));
    }
    if (fileConfigIds != null && fileConfigIds.length > 0) {
        cmdList.add("-f");
        cmdList.add(StringUtils.join(fileConfigIds, ','));
    }
    if (dataConfigIds != null && dataConfigIds.length > 0) {
        cmdList.add("-d");
        cmdList.add(StringUtils.join(dataConfigIds, ','));
    }
    if (documentExpires >= -1) {
        cmdList.add("-e");
        cmdList.add(Integer.toString(documentExpires));
    }
    final File propFile = ComponentUtil.getSystemHelper().createTempFile(getExecuteType() + "_", ".properties");
    try {
        cmdList.add("-p");
        cmdList.add(propFile.getAbsolutePath());
        createSystemProperties(cmdList, propFile);
        final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
        if (logger.isInfoEnabled()) {
            logger.info("Crawler: \nDirectory={}\nOptions={}", baseDir, cmdList);
        }
        final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
            pb.directory(baseDir);
            pb.redirectErrorStream(true);
        });
        final InputStreamThread it = jobProcess.getInputStreamThread();
        it.start();
        final Process currentProcess = jobProcess.getProcess();
        currentProcess.waitFor();
        it.join(5000);
        final int exitValue = currentProcess.exitValue();
        if (logger.isInfoEnabled()) {
            logger.info("Crawler: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
        }
        if (exitValue != 0) {
            final StringBuilder out = new StringBuilder();
            if (processTimeout) {
                out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
            }
            out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
            throw new JobProcessingException(out.toString());
        }
    } catch (final JobProcessingException e) {
        throw e;
    } catch (final Exception e) {
        throw new JobProcessingException("Crawler Process terminated.", e);
    } finally {
        try {
            processHelper.destroyProcess(sessionId);
        } finally {
            if (propFile != null && !propFile.delete()) {
                logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
            }
            deleteTempDir(ownTmpDir);
        }
    }
}
Also used : Constants(org.codelibs.fess.Constants) Date(java.util.Date) InputStreamThread(org.codelibs.fess.util.InputStreamThread) SimpleDateFormat(java.text.SimpleDateFormat) StringUtils(org.apache.commons.lang3.StringUtils) ArrayList(java.util.ArrayList) ResourceUtil(org.codelibs.fess.util.ResourceUtil) FessConfig(org.codelibs.fess.mylasta.direction.FessConfig) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ScheduledJobBhv(org.codelibs.fess.es.config.exbhv.ScheduledJobBhv) JobProcessingException(org.codelibs.fess.exception.JobProcessingException) JobProcess(org.codelibs.fess.util.JobProcess) StreamUtil.stream(org.codelibs.core.stream.StreamUtil.stream) ProcessHelper(org.codelibs.fess.helper.ProcessHelper) SystemUtils(org.apache.commons.lang3.SystemUtils) StringUtil(org.codelibs.core.lang.StringUtil) File(java.io.File) List(java.util.List) Logger(org.apache.logging.log4j.Logger) ComponentUtil(org.codelibs.fess.util.ComponentUtil) SystemHelper(org.codelibs.fess.helper.SystemHelper) Crawler(org.codelibs.fess.exec.Crawler) ServletContext(javax.servlet.ServletContext) TimeoutTask(org.codelibs.core.timer.TimeoutTask) LogManager(org.apache.logging.log4j.LogManager) ArrayList(java.util.ArrayList) JobProcess(org.codelibs.fess.util.JobProcess) Crawler(org.codelibs.fess.exec.Crawler) ProcessHelper(org.codelibs.fess.helper.ProcessHelper) FessConfig(org.codelibs.fess.mylasta.direction.FessConfig) JobProcess(org.codelibs.fess.util.JobProcess) JobProcessingException(org.codelibs.fess.exception.JobProcessingException) JobProcessingException(org.codelibs.fess.exception.JobProcessingException) InputStreamThread(org.codelibs.fess.util.InputStreamThread) SystemHelper(org.codelibs.fess.helper.SystemHelper) ServletContext(javax.servlet.ServletContext) StringUtil(org.codelibs.core.lang.StringUtil) File(java.io.File)

Example 7 with JobProcess

use of org.codelibs.fess.util.JobProcess in project fess by codelibs.

the class GenerateThumbnailJob method executeThumbnailGenerator.

protected void executeThumbnailGenerator() {
    final List<String> cmdList = new ArrayList<>();
    final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
    final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
    final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    cmdList.add(fessConfig.getJavaCommandPath());
    // -cp
    cmdList.add("-cp");
    final StringBuilder buf = new StringBuilder(100);
    ResourceUtil.getOverrideConfPath().ifPresent(p -> {
        buf.append(p);
        buf.append(cpSeparator);
    });
    final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
    if (StringUtil.isNotBlank(confPath)) {
        buf.append(confPath);
        buf.append(cpSeparator);
    }
    // WEB-INF/env/thumbnail/resources
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("env");
    buf.append(File.separator);
    buf.append(getExecuteType());
    buf.append(File.separator);
    buf.append("resources");
    buf.append(cpSeparator);
    // WEB-INF/classes
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("classes");
    // target/classes
    final String userDir = System.getProperty("user.dir");
    final File targetDir = new File(userDir, "target");
    final File targetClassesDir = new File(targetDir, "classes");
    if (targetClassesDir.isDirectory()) {
        buf.append(cpSeparator);
        buf.append(targetClassesDir.getAbsolutePath());
    }
    // WEB-INF/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib" + File.separator);
    // WEB-INF/env/thumbnail/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
    // WEB-INF/plugin
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin" + File.separator);
    final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
    if (targetLibDir.isDirectory()) {
        appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
    }
    cmdList.add(buf.toString());
    if (useLocalFesen) {
        final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
        if (StringUtil.isNotBlank(httpAddress)) {
            cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
        }
    }
    final String systemLastaEnv = System.getProperty("lasta.env");
    if (StringUtil.isNotBlank(systemLastaEnv)) {
        if ("web".equals(systemLastaEnv)) {
            cmdList.add("-Dlasta.env=" + getExecuteType());
        } else {
            cmdList.add("-Dlasta.env=" + systemLastaEnv);
        }
    } else if (StringUtil.isNotBlank(lastaEnv)) {
        cmdList.add("-Dlasta.env=" + lastaEnv);
    } else {
        cmdList.add("-Dlasta.env=" + getExecuteType());
    }
    addFessConfigProperties(cmdList);
    addFessSystemProperties(cmdList);
    addSystemProperty(cmdList, Constants.FESS_CONF_PATH, null, null);
    cmdList.add("-Dfess." + getExecuteType() + ".process=true");
    if (logFilePath == null) {
        final String value = System.getProperty("fess.log.path");
        logFilePath = value != null ? value : new File(targetDir, "logs").getAbsolutePath();
    }
    cmdList.add("-Dfess.log.path=" + logFilePath);
    addSystemProperty(cmdList, Constants.FESS_VAR_PATH, null, null);
    addSystemProperty(cmdList, Constants.FESS_THUMBNAIL_PATH, null, null);
    addSystemProperty(cmdList, "fess.log.name", "fess-" + getExecuteType(), "-" + getExecuteType());
    if (logLevel != null) {
        cmdList.add("-Dfess.log.level=" + logLevel);
    }
    stream(fessConfig.getJvmThumbnailOptionsAsArray()).of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
    File ownTmpDir = null;
    final String tmpDir = System.getProperty("java.io.tmpdir");
    if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
        ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
        if (ownTmpDir.mkdirs()) {
            cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
        } else {
            ownTmpDir = null;
        }
    }
    if (!jvmOptions.isEmpty()) {
        jvmOptions.stream().filter(StringUtil::isNotBlank).forEach(cmdList::add);
    }
    cmdList.add(ThumbnailGenerator.class.getCanonicalName());
    cmdList.add("--sessionId");
    cmdList.add(sessionId);
    cmdList.add("--numOfThreads");
    cmdList.add(Integer.toString(numOfThreads));
    if (cleanup) {
        cmdList.add("--cleanup");
    }
    final File propFile = ComponentUtil.getSystemHelper().createTempFile(getExecuteType() + "_", ".properties");
    try {
        cmdList.add("-p");
        cmdList.add(propFile.getAbsolutePath());
        createSystemProperties(cmdList, propFile);
        final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
        if (logger.isInfoEnabled()) {
            logger.info("ThumbnailGenerator: \nDirectory={}\nOptions={}", baseDir, cmdList);
        }
        final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
            pb.directory(baseDir);
            pb.redirectErrorStream(true);
        });
        final InputStreamThread it = jobProcess.getInputStreamThread();
        it.start();
        final Process currentProcess = jobProcess.getProcess();
        currentProcess.waitFor();
        it.join(5000);
        final int exitValue = currentProcess.exitValue();
        if (logger.isInfoEnabled()) {
            logger.info("ThumbnailGenerator: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
        }
        if (exitValue != 0) {
            final StringBuilder out = new StringBuilder();
            if (processTimeout) {
                out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
            }
            out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
            throw new JobProcessingException(out.toString());
        }
        ComponentUtil.getPopularWordHelper().clearCache();
    } catch (final JobProcessingException e) {
        throw e;
    } catch (final Exception e) {
        throw new JobProcessingException("ThumbnailGenerator Process terminated.", e);
    } finally {
        try {
            processHelper.destroyProcess(sessionId);
        } finally {
            if (propFile != null && !propFile.delete()) {
                logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
            }
            deleteTempDir(ownTmpDir);
        }
    }
}
Also used : Constants(org.codelibs.fess.Constants) StreamUtil.stream(org.codelibs.core.stream.StreamUtil.stream) ProcessHelper(org.codelibs.fess.helper.ProcessHelper) ThumbnailGenerator(org.codelibs.fess.exec.ThumbnailGenerator) SystemUtils(org.apache.commons.lang3.SystemUtils) InputStreamThread(org.codelibs.fess.util.InputStreamThread) StringUtil(org.codelibs.core.lang.StringUtil) File(java.io.File) ArrayList(java.util.ArrayList) ResourceUtil(org.codelibs.fess.util.ResourceUtil) List(java.util.List) FessConfig(org.codelibs.fess.mylasta.direction.FessConfig) Logger(org.apache.logging.log4j.Logger) ComponentUtil(org.codelibs.fess.util.ComponentUtil) ServletContext(javax.servlet.ServletContext) RandomStringUtils(org.apache.commons.lang3.RandomStringUtils) JobProcessingException(org.codelibs.fess.exception.JobProcessingException) TimeoutTask(org.codelibs.core.timer.TimeoutTask) LogManager(org.apache.logging.log4j.LogManager) JobProcess(org.codelibs.fess.util.JobProcess) ArrayList(java.util.ArrayList) JobProcess(org.codelibs.fess.util.JobProcess) ProcessHelper(org.codelibs.fess.helper.ProcessHelper) FessConfig(org.codelibs.fess.mylasta.direction.FessConfig) JobProcess(org.codelibs.fess.util.JobProcess) JobProcessingException(org.codelibs.fess.exception.JobProcessingException) JobProcessingException(org.codelibs.fess.exception.JobProcessingException) InputStreamThread(org.codelibs.fess.util.InputStreamThread) ThumbnailGenerator(org.codelibs.fess.exec.ThumbnailGenerator) ServletContext(javax.servlet.ServletContext) StringUtil(org.codelibs.core.lang.StringUtil) File(java.io.File)

Example 8 with JobProcess

use of org.codelibs.fess.util.JobProcess in project fess by codelibs.

the class PythonJob method executePython.

protected void executePython() {
    final List<String> cmdList = new ArrayList<>();
    final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
    final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    if (StringUtil.isBlank(filename)) {
        throw new JobProcessingException("Python script is not specified.");
    }
    cmdList.add(fessConfig.getPythonCommandPath());
    cmdList.add(getPyFilePath());
    cmdList.addAll(argList);
    try {
        final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
        if (logger.isInfoEnabled()) {
            logger.info("Python: \nDirectory={}\nOptions={}", baseDir, cmdList);
        }
        final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
            pb.directory(baseDir);
            pb.redirectErrorStream(true);
        });
        final InputStreamThread it = jobProcess.getInputStreamThread();
        it.start();
        final Process currentProcess = jobProcess.getProcess();
        currentProcess.waitFor();
        it.join(5000);
        final int exitValue = currentProcess.exitValue();
        if (logger.isInfoEnabled()) {
            logger.info("Python: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
        }
        if (exitValue != 0) {
            final StringBuilder out = new StringBuilder();
            if (processTimeout) {
                out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
            }
            out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
            throw new JobProcessingException(out.toString());
        }
        ComponentUtil.getPopularWordHelper().clearCache();
    } catch (final JobProcessingException e) {
        throw e;
    } catch (final Exception e) {
        throw new JobProcessingException("Python Process terminated.", e);
    } finally {
        processHelper.destroyProcess(sessionId);
    }
}
Also used : ArrayList(java.util.ArrayList) JobProcess(org.codelibs.fess.util.JobProcess) ProcessHelper(org.codelibs.fess.helper.ProcessHelper) FessConfig(org.codelibs.fess.mylasta.direction.FessConfig) JobProcessingException(org.codelibs.fess.exception.JobProcessingException) JobProcess(org.codelibs.fess.util.JobProcess) JobProcessingException(org.codelibs.fess.exception.JobProcessingException) InputStreamThread(org.codelibs.fess.util.InputStreamThread) ServletContext(javax.servlet.ServletContext) File(java.io.File)

Aggregations

JobProcess (org.codelibs.fess.util.JobProcess)8 JobProcessingException (org.codelibs.fess.exception.JobProcessingException)7 InputStreamThread (org.codelibs.fess.util.InputStreamThread)5 File (java.io.File)4 IOException (java.io.IOException)4 ArrayList (java.util.ArrayList)4 ServletContext (javax.servlet.ServletContext)4 ProcessHelper (org.codelibs.fess.helper.ProcessHelper)4 FessConfig (org.codelibs.fess.mylasta.direction.FessConfig)4 List (java.util.List)3 SystemUtils (org.apache.commons.lang3.SystemUtils)3 LogManager (org.apache.logging.log4j.LogManager)3 Logger (org.apache.logging.log4j.Logger)3 StringUtil (org.codelibs.core.lang.StringUtil)3 StreamUtil.stream (org.codelibs.core.stream.StreamUtil.stream)3 TimeoutTask (org.codelibs.core.timer.TimeoutTask)3 Constants (org.codelibs.fess.Constants)3 ComponentUtil (org.codelibs.fess.util.ComponentUtil)3 ResourceUtil (org.codelibs.fess.util.ResourceUtil)3 RandomStringUtils (org.apache.commons.lang3.RandomStringUtils)2