Search in sources :

Example 1 with InputStreamThread

use of org.codelibs.fess.util.InputStreamThread in project fess by codelibs.

the class ProcessHelper method destroyProcess.

protected int destroyProcess(final String sessionId, final JobProcess jobProcess) {
    if (jobProcess != null) {
        final InputStreamThread ist = jobProcess.getInputStreamThread();
        try {
            ist.interrupt();
        } catch (final Exception e) {
            logger.warn("Could not interrupt a thread of an input stream.", e);
        }
        final CountDownLatch latch = new CountDownLatch(3);
        final Process process = jobProcess.getProcess();
        new Thread(() -> {
            try {
                IOUtils.closeQuietly(process.getInputStream());
            } catch (final Exception e) {
                logger.warn("Could not close a process input stream.", e);
            } finally {
                latch.countDown();
            }
        }, "ProcessCloser-input-" + sessionId).start();
        new Thread(() -> {
            try {
                IOUtils.closeQuietly(process.getErrorStream());
            } catch (final Exception e) {
                logger.warn("Could not close a process error stream.", e);
            } finally {
                latch.countDown();
            }
        }, "ProcessCloser-error-" + sessionId).start();
        new Thread(() -> {
            try {
                IOUtils.closeQuietly(process.getOutputStream());
            } catch (final Exception e) {
                logger.warn("Could not close a process output stream.", e);
            } finally {
                latch.countDown();
            }
        }, "ProcessCloser-output-" + sessionId).start();
        try {
            latch.await(10, TimeUnit.SECONDS);
        } catch (final InterruptedException e) {
            logger.warn("Interrupted to wait a process.", e);
        }
        try {
            process.destroyForcibly().waitFor(processDestroyTimeout, TimeUnit.SECONDS);
            return process.exitValue();
        } catch (final Exception e) {
            logger.error("Could not destroy a process correctly.", e);
        }
    }
    return -1;
}
Also used : InputStreamThread(org.codelibs.fess.util.InputStreamThread) JobProcess(org.codelibs.fess.util.JobProcess) CountDownLatch(java.util.concurrent.CountDownLatch) FessSystemException(org.codelibs.fess.exception.FessSystemException) IOException(java.io.IOException) InputStreamThread(org.codelibs.fess.util.InputStreamThread)

Example 2 with InputStreamThread

use of org.codelibs.fess.util.InputStreamThread in project fess by codelibs.

the class SuggestJob method executeSuggestCreator.

protected void executeSuggestCreator() {
    final List<String> cmdList = new ArrayList<>();
    final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
    final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
    final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    cmdList.add(fessConfig.getJavaCommandPath());
    // -cp
    cmdList.add("-cp");
    final StringBuilder buf = new StringBuilder(100);
    final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
    if (StringUtil.isNotBlank(confPath)) {
        buf.append(confPath);
        buf.append(cpSeparator);
    }
    // WEB-INF/suggest/resources
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("suggest");
    buf.append(File.separator);
    buf.append("resources");
    buf.append(cpSeparator);
    // WEB-INF/classes
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("classes");
    // target/classes
    final String userDir = System.getProperty("user.dir");
    final File targetDir = new File(userDir, "target");
    final File targetClassesDir = new File(targetDir, "classes");
    if (targetClassesDir.isDirectory()) {
        buf.append(cpSeparator);
        buf.append(targetClassesDir.getAbsolutePath());
    }
    // WEB-INF/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF/lib" + File.separator);
    // WEB-INF/crawler/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/suggest/lib")), "WEB-INF/suggest" + File.separator + "lib" + File.separator);
    final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
    if (targetLibDir.isDirectory()) {
        appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
    }
    cmdList.add(buf.toString());
    if (useLocaleElasticsearch) {
        final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
        if (StringUtil.isNotBlank(transportAddresses)) {
            cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
        }
        final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
        if (StringUtil.isNotBlank(clusterName)) {
            cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
        }
    }
    final String lastaEnv = System.getProperty("lasta.env");
    if (StringUtil.isNotBlank(lastaEnv)) {
        if (lastaEnv.equals("web")) {
            cmdList.add("-Dlasta.env=suggest");
        } else {
            cmdList.add("-Dlasta.env=" + lastaEnv);
        }
    }
    cmdList.add("-Dfess.suggest.process=true");
    if (logFilePath == null) {
        final String value = System.getProperty("fess.log.path");
        logFilePath = value != null ? value : new File(targetDir, "logs").getAbsolutePath();
    }
    cmdList.add("-Dfess.log.path=" + logFilePath);
    addSystemProperty(cmdList, "fess.log.name", "fess-suggest", "-suggest");
    if (logLevel == null) {
        addSystemProperty(cmdList, "fess.log.level", null, null);
    } else {
        cmdList.add("-Dfess.log.level=" + logLevel);
    }
    stream(fessConfig.getJvmSuggestOptionsAsArray()).of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
    File ownTmpDir = null;
    final String tmpDir = System.getProperty("java.io.tmpdir");
    if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
        ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
        if (ownTmpDir.mkdirs()) {
            cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
        } else {
            ownTmpDir = null;
        }
    }
    cmdList.add(SuggestCreator.class.getCanonicalName());
    cmdList.add("--sessionId");
    cmdList.add(sessionId);
    File propFile = null;
    try {
        cmdList.add("-p");
        propFile = File.createTempFile("crawler_", ".properties");
        cmdList.add(propFile.getAbsolutePath());
        try (FileOutputStream out = new FileOutputStream(propFile)) {
            final Properties prop = new Properties();
            prop.putAll(ComponentUtil.getSystemProperties());
            prop.store(out, cmdList.toString());
        }
        final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
        if (logger.isInfoEnabled()) {
            logger.info("SuggestCreator: \nDirectory=" + baseDir + "\nOptions=" + cmdList);
        }
        final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
            pb.directory(baseDir);
            pb.redirectErrorStream(true);
        });
        final InputStreamThread it = jobProcess.getInputStreamThread();
        it.start();
        final Process currentProcess = jobProcess.getProcess();
        currentProcess.waitFor();
        it.join(5000);
        final int exitValue = currentProcess.exitValue();
        if (logger.isInfoEnabled()) {
            logger.info("SuggestCreator: Exit Code=" + exitValue + " - SuggestCreator Process Output:\n" + it.getOutput());
        }
        if (exitValue != 0) {
            throw new FessSystemException("Exit Code: " + exitValue + "\nOutput:\n" + it.getOutput());
        }
        ComponentUtil.getPopularWordHelper().clearCache();
    } catch (final FessSystemException e) {
        throw e;
    } catch (final InterruptedException e) {
        logger.warn("SuggestCreator Process interrupted.");
    } catch (final Exception e) {
        throw new FessSystemException("SuggestCreator Process terminated.", e);
    } finally {
        try {
            processHelper.destroyProcess(sessionId);
        } finally {
            if (propFile != null && !propFile.delete()) {
                logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
            }
            deleteTempDir(ownTmpDir);
        }
    }
}
Also used : FilenameFilter(java.io.FilenameFilter) Constants(org.codelibs.fess.Constants) StreamUtil.stream(org.codelibs.core.stream.StreamUtil.stream) Properties(java.util.Properties) ProcessHelper(org.codelibs.fess.helper.ProcessHelper) Logger(org.slf4j.Logger) FessSystemException(org.codelibs.fess.exception.FessSystemException) Date(java.util.Date) SystemUtils(org.apache.commons.lang3.SystemUtils) InputStreamThread(org.codelibs.fess.util.InputStreamThread) StringUtil(org.codelibs.core.lang.StringUtil) LoggerFactory(org.slf4j.LoggerFactory) FileOutputStream(java.io.FileOutputStream) SimpleDateFormat(java.text.SimpleDateFormat) SuggestCreator(org.codelibs.fess.exec.SuggestCreator) FileUtils(org.apache.commons.io.FileUtils) File(java.io.File) ArrayList(java.util.ArrayList) List(java.util.List) FessConfig(org.codelibs.fess.mylasta.direction.FessConfig) ComponentUtil(org.codelibs.fess.util.ComponentUtil) ServletContext(javax.servlet.ServletContext) JobProcess(org.codelibs.fess.util.JobProcess) SuggestCreator(org.codelibs.fess.exec.SuggestCreator) ArrayList(java.util.ArrayList) JobProcess(org.codelibs.fess.util.JobProcess) Properties(java.util.Properties) ProcessHelper(org.codelibs.fess.helper.ProcessHelper) FessConfig(org.codelibs.fess.mylasta.direction.FessConfig) JobProcess(org.codelibs.fess.util.JobProcess) FessSystemException(org.codelibs.fess.exception.FessSystemException) FessSystemException(org.codelibs.fess.exception.FessSystemException) InputStreamThread(org.codelibs.fess.util.InputStreamThread) FileOutputStream(java.io.FileOutputStream) ServletContext(javax.servlet.ServletContext) StringUtil(org.codelibs.core.lang.StringUtil) File(java.io.File)

Example 3 with InputStreamThread

use of org.codelibs.fess.util.InputStreamThread in project fess by codelibs.

the class CrawlJob method executeCrawler.

protected void executeCrawler() {
    final List<String> cmdList = new ArrayList<>();
    final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
    final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    cmdList.add(fessConfig.getJavaCommandPath());
    // -cp
    cmdList.add("-cp");
    final StringBuilder buf = new StringBuilder(100);
    final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
    if (StringUtil.isNotBlank(confPath)) {
        buf.append(confPath);
        buf.append(cpSeparator);
    }
    // WEB-INF/crawler/resources
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("crawler");
    buf.append(File.separator);
    buf.append("resources");
    buf.append(cpSeparator);
    // WEB-INF/classes
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("classes");
    // target/classes
    final String userDir = System.getProperty("user.dir");
    final File targetDir = new File(userDir, "target");
    final File targetClassesDir = new File(targetDir, "classes");
    if (targetClassesDir.isDirectory()) {
        buf.append(cpSeparator);
        buf.append(targetClassesDir.getAbsolutePath());
    }
    // WEB-INF/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF/lib" + File.separator);
    // WEB-INF/crawler/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/crawler/lib")), "WEB-INF/crawler" + File.separator + "lib" + File.separator);
    final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
    if (targetLibDir.isDirectory()) {
        appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
    }
    cmdList.add(buf.toString());
    if (useLocalElasticsearch) {
        final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
        if (StringUtil.isNotBlank(transportAddresses)) {
            cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
        }
        final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
        if (StringUtil.isNotBlank(clusterName)) {
            cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
        }
    }
    final String systemLastaEnv = System.getProperty("lasta.env");
    if (StringUtil.isNotBlank(systemLastaEnv)) {
        if (systemLastaEnv.equals("web")) {
            cmdList.add("-Dlasta.env=crawler");
        } else {
            cmdList.add("-Dlasta.env=" + systemLastaEnv);
        }
    } else if (StringUtil.isNotBlank(lastaEnv)) {
        cmdList.add("-Dlasta.env=" + lastaEnv);
    }
    cmdList.add("-Dfess.crawler.process=true");
    cmdList.add("-Dfess.log.path=" + (logFilePath != null ? logFilePath : systemHelper.getLogFilePath()));
    addSystemProperty(cmdList, "fess.log.name", "fess-crawler", "-crawler");
    if (logLevel == null) {
        addSystemProperty(cmdList, "fess.log.level", null, null);
    } else {
        cmdList.add("-Dfess.log.level=" + logLevel);
    }
    stream(fessConfig.getJvmCrawlerOptionsAsArray()).of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
    File ownTmpDir = null;
    final String tmpDir = System.getProperty("java.io.tmpdir");
    if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
        ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
        if (ownTmpDir.mkdirs()) {
            cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
            cmdList.add("-Dpdfbox.fontcache=" + ownTmpDir.getAbsolutePath());
        } else {
            ownTmpDir = null;
        }
    }
    cmdList.add(ComponentUtil.getThumbnailManager().getThumbnailPathOption());
    if (StringUtil.isNotBlank(jvmOptions)) {
        split(jvmOptions, " ").of(stream -> stream.filter(StringUtil::isNotBlank).forEach(s -> cmdList.add(s)));
    }
    cmdList.add(Crawler.class.getCanonicalName());
    cmdList.add("--sessionId");
    cmdList.add(sessionId);
    cmdList.add("--name");
    cmdList.add(namespace);
    if (webConfigIds != null && webConfigIds.length > 0) {
        cmdList.add("-w");
        cmdList.add(StringUtils.join(webConfigIds, ','));
    }
    if (fileConfigIds != null && fileConfigIds.length > 0) {
        cmdList.add("-f");
        cmdList.add(StringUtils.join(fileConfigIds, ','));
    }
    if (dataConfigIds != null && dataConfigIds.length > 0) {
        cmdList.add("-d");
        cmdList.add(StringUtils.join(dataConfigIds, ','));
    }
    if (documentExpires >= -1) {
        cmdList.add("-e");
        cmdList.add(Integer.toString(documentExpires));
    }
    File propFile = null;
    try {
        cmdList.add("-p");
        propFile = File.createTempFile("crawler_", ".properties");
        cmdList.add(propFile.getAbsolutePath());
        try (FileOutputStream out = new FileOutputStream(propFile)) {
            final Properties prop = new Properties();
            prop.putAll(ComponentUtil.getSystemProperties());
            prop.store(out, cmdList.toString());
        }
        final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
        if (logger.isInfoEnabled()) {
            logger.info("Crawler: \nDirectory=" + baseDir + "\nOptions=" + cmdList);
        }
        final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
            pb.directory(baseDir);
            pb.redirectErrorStream(true);
        });
        final InputStreamThread it = jobProcess.getInputStreamThread();
        it.start();
        final Process currentProcess = jobProcess.getProcess();
        currentProcess.waitFor();
        it.join(5000);
        final int exitValue = currentProcess.exitValue();
        if (logger.isInfoEnabled()) {
            logger.info("Crawler: Exit Code=" + exitValue + " - Crawler Process Output:\n" + it.getOutput());
        }
        if (exitValue != 0) {
            throw new FessSystemException("Exit Code: " + exitValue + "\nOutput:\n" + it.getOutput());
        }
    } catch (final FessSystemException e) {
        throw e;
    } catch (final InterruptedException e) {
        logger.warn("Crawler Process interrupted.");
    } catch (final Exception e) {
        throw new FessSystemException("Crawler Process terminated.", e);
    } finally {
        try {
            processHelper.destroyProcess(sessionId);
        } finally {
            if (propFile != null && !propFile.delete()) {
                logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
            }
            deleteTempDir(ownTmpDir);
        }
    }
}
Also used : FilenameFilter(java.io.FilenameFilter) Constants(org.codelibs.fess.Constants) FessSystemException(org.codelibs.fess.exception.FessSystemException) Date(java.util.Date) InputStreamThread(org.codelibs.fess.util.InputStreamThread) LoggerFactory(org.slf4j.LoggerFactory) SimpleDateFormat(java.text.SimpleDateFormat) StringUtils(org.apache.commons.lang3.StringUtils) ArrayList(java.util.ArrayList) FessConfig(org.codelibs.fess.mylasta.direction.FessConfig) StreamUtil.split(org.codelibs.core.stream.StreamUtil.split) JobProcess(org.codelibs.fess.util.JobProcess) StreamUtil.stream(org.codelibs.core.stream.StreamUtil.stream) Properties(java.util.Properties) ProcessHelper(org.codelibs.fess.helper.ProcessHelper) Logger(org.slf4j.Logger) SystemUtils(org.apache.commons.lang3.SystemUtils) StringUtil(org.codelibs.core.lang.StringUtil) FileOutputStream(java.io.FileOutputStream) FileUtils(org.apache.commons.io.FileUtils) File(java.io.File) List(java.util.List) ComponentUtil(org.codelibs.fess.util.ComponentUtil) SystemHelper(org.codelibs.fess.helper.SystemHelper) Crawler(org.codelibs.fess.exec.Crawler) ServletContext(javax.servlet.ServletContext) ArrayList(java.util.ArrayList) JobProcess(org.codelibs.fess.util.JobProcess) Crawler(org.codelibs.fess.exec.Crawler) Properties(java.util.Properties) ProcessHelper(org.codelibs.fess.helper.ProcessHelper) FessConfig(org.codelibs.fess.mylasta.direction.FessConfig) JobProcess(org.codelibs.fess.util.JobProcess) FessSystemException(org.codelibs.fess.exception.FessSystemException) FessSystemException(org.codelibs.fess.exception.FessSystemException) InputStreamThread(org.codelibs.fess.util.InputStreamThread) SystemHelper(org.codelibs.fess.helper.SystemHelper) FileOutputStream(java.io.FileOutputStream) ServletContext(javax.servlet.ServletContext) StringUtil(org.codelibs.core.lang.StringUtil) File(java.io.File)

Aggregations

FessSystemException (org.codelibs.fess.exception.FessSystemException)3 InputStreamThread (org.codelibs.fess.util.InputStreamThread)3 JobProcess (org.codelibs.fess.util.JobProcess)3 File (java.io.File)2 FileOutputStream (java.io.FileOutputStream)2 FilenameFilter (java.io.FilenameFilter)2 SimpleDateFormat (java.text.SimpleDateFormat)2 ArrayList (java.util.ArrayList)2 Date (java.util.Date)2 List (java.util.List)2 Properties (java.util.Properties)2 ServletContext (javax.servlet.ServletContext)2 FileUtils (org.apache.commons.io.FileUtils)2 SystemUtils (org.apache.commons.lang3.SystemUtils)2 StringUtil (org.codelibs.core.lang.StringUtil)2 StreamUtil.stream (org.codelibs.core.stream.StreamUtil.stream)2 Constants (org.codelibs.fess.Constants)2 ProcessHelper (org.codelibs.fess.helper.ProcessHelper)2 FessConfig (org.codelibs.fess.mylasta.direction.FessConfig)2 ComponentUtil (org.codelibs.fess.util.ComponentUtil)2