use of org.codelibs.fess.helper.ProcessHelper in project fess by codelibs.
the class SuggestJob method executeSuggestCreator.
protected void executeSuggestCreator() {
final List<String> cmdList = new ArrayList<>();
final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
cmdList.add(fessConfig.getJavaCommandPath());
// -cp
cmdList.add("-cp");
final StringBuilder buf = new StringBuilder(100);
ResourceUtil.getOverrideConfPath().ifPresent(p -> {
buf.append(p);
buf.append(cpSeparator);
});
final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
if (StringUtil.isNotBlank(confPath)) {
buf.append(confPath);
buf.append(cpSeparator);
}
// WEB-INF/env/suggest/resources
buf.append("WEB-INF");
buf.append(File.separator);
buf.append("env");
buf.append(File.separator);
buf.append(getExecuteType());
buf.append(File.separator);
buf.append("resources");
buf.append(cpSeparator);
// WEB-INF/classes
buf.append("WEB-INF");
buf.append(File.separator);
buf.append("classes");
// target/classes
final String userDir = System.getProperty("user.dir");
final File targetDir = new File(userDir, "target");
final File targetClassesDir = new File(targetDir, "classes");
if (targetClassesDir.isDirectory()) {
buf.append(cpSeparator);
buf.append(targetClassesDir.getAbsolutePath());
}
// WEB-INF/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib" + File.separator);
// WEB-INF/env/suggest/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
// WEB-INF/plugin
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin" + File.separator);
final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
if (targetLibDir.isDirectory()) {
appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
}
cmdList.add(buf.toString());
if (useLocalFesen) {
final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isNotBlank(httpAddress)) {
cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
}
}
final String systemLastaEnv = System.getProperty("lasta.env");
if (StringUtil.isNotBlank(systemLastaEnv)) {
if ("web".equals(systemLastaEnv)) {
cmdList.add("-Dlasta.env=" + getExecuteType());
} else {
cmdList.add("-Dlasta.env=" + systemLastaEnv);
}
} else if (StringUtil.isNotBlank(lastaEnv)) {
cmdList.add("-Dlasta.env=" + lastaEnv);
} else {
cmdList.add("-Dlasta.env=" + getExecuteType());
}
addFessConfigProperties(cmdList);
addFessSystemProperties(cmdList);
addSystemProperty(cmdList, Constants.FESS_CONF_PATH, null, null);
cmdList.add("-Dfess." + getExecuteType() + ".process=true");
if (logFilePath == null) {
final String value = System.getProperty("fess.log.path");
logFilePath = value != null ? value : new File(targetDir, "logs").getAbsolutePath();
}
cmdList.add("-Dfess.log.path=" + logFilePath);
addSystemProperty(cmdList, "fess.log.name", "fess-" + getExecuteType(), "-" + getExecuteType());
if (logLevel == null) {
addSystemProperty(cmdList, "fess.log.level", null, null);
} else {
cmdList.add("-Dfess.log.level=" + logLevel);
}
stream(fessConfig.getJvmSuggestOptionsAsArray()).of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
File ownTmpDir = null;
final String tmpDir = System.getProperty("java.io.tmpdir");
if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
if (ownTmpDir.mkdirs()) {
cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
} else {
ownTmpDir = null;
}
}
if (!jvmOptions.isEmpty()) {
jvmOptions.stream().filter(StringUtil::isNotBlank).forEach(cmdList::add);
}
cmdList.add(SuggestCreator.class.getCanonicalName());
cmdList.add("--sessionId");
cmdList.add(sessionId);
final File propFile = ComponentUtil.getSystemHelper().createTempFile(getExecuteType() + "_", ".properties");
try {
cmdList.add("-p");
cmdList.add(propFile.getAbsolutePath());
createSystemProperties(cmdList, propFile);
final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
if (logger.isInfoEnabled()) {
logger.info("SuggestCreator: \nDirectory={}\nOptions={}", baseDir, cmdList);
}
final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
pb.directory(baseDir);
pb.redirectErrorStream(true);
});
final InputStreamThread it = jobProcess.getInputStreamThread();
it.start();
final Process currentProcess = jobProcess.getProcess();
currentProcess.waitFor();
it.join(5000);
final int exitValue = currentProcess.exitValue();
if (logger.isInfoEnabled()) {
logger.info("SuggestCreator: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
}
if (exitValue != 0) {
final StringBuilder out = new StringBuilder();
if (processTimeout) {
out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
}
out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
throw new JobProcessingException(out.toString());
}
ComponentUtil.getPopularWordHelper().clearCache();
} catch (final JobProcessingException e) {
throw e;
} catch (final Exception e) {
throw new JobProcessingException("SuggestCreator Process terminated.", e);
} finally {
try {
processHelper.destroyProcess(sessionId);
} finally {
if (propFile != null && !propFile.delete()) {
logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
}
deleteTempDir(ownTmpDir);
}
}
}
use of org.codelibs.fess.helper.ProcessHelper in project fess by codelibs.
the class CrawlJob method executeCrawler.
protected void executeCrawler() {
final List<String> cmdList = new ArrayList<>();
final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
cmdList.add(fessConfig.getJavaCommandPath());
// -cp
cmdList.add("-cp");
final StringBuilder buf = new StringBuilder(100);
ResourceUtil.getOverrideConfPath().ifPresent(p -> {
buf.append(p);
buf.append(cpSeparator);
});
final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
if (StringUtil.isNotBlank(confPath)) {
buf.append(confPath);
buf.append(cpSeparator);
}
// WEB-INF/env/crawler/resources
buf.append("WEB-INF");
buf.append(File.separator);
buf.append("env");
buf.append(File.separator);
buf.append(getExecuteType());
buf.append(File.separator);
buf.append("resources");
buf.append(cpSeparator);
// WEB-INF/classes
buf.append("WEB-INF");
buf.append(File.separator);
buf.append("classes");
// target/classes
final String userDir = System.getProperty("user.dir");
final File targetDir = new File(userDir, "target");
final File targetClassesDir = new File(targetDir, "classes");
if (targetClassesDir.isDirectory()) {
buf.append(cpSeparator);
buf.append(targetClassesDir.getAbsolutePath());
}
// WEB-INF/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib" + File.separator);
// WEB-INF/env/crawler/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
// WEB-INF/plugin
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin" + File.separator);
final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
if (targetLibDir.isDirectory()) {
appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
}
cmdList.add(buf.toString());
if (useLocalFesen) {
final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isNotBlank(httpAddress)) {
cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
}
}
final String systemLastaEnv = System.getProperty("lasta.env");
if (StringUtil.isNotBlank(systemLastaEnv)) {
if ("web".equals(systemLastaEnv)) {
cmdList.add("-Dlasta.env=" + getExecuteType());
} else {
cmdList.add("-Dlasta.env=" + systemLastaEnv);
}
} else if (StringUtil.isNotBlank(lastaEnv)) {
cmdList.add("-Dlasta.env=" + lastaEnv);
} else {
cmdList.add("-Dlasta.env=" + getExecuteType());
}
addFessConfigProperties(cmdList);
addFessSystemProperties(cmdList);
addSystemProperty(cmdList, Constants.FESS_CONF_PATH, null, null);
cmdList.add("-Dfess." + getExecuteType() + ".process=true");
cmdList.add("-Dfess.log.path=" + (logFilePath != null ? logFilePath : systemHelper.getLogFilePath()));
addSystemProperty(cmdList, "fess.log.name", "fess-" + getExecuteType(), "-" + getExecuteType());
if (logLevel == null) {
addSystemProperty(cmdList, "fess.log.level", null, null);
} else {
cmdList.add("-Dfess.log.level=" + logLevel);
if ("debug".equalsIgnoreCase(logLevel)) {
cmdList.add("-Dorg.apache.tika.service.error.warn=true");
}
}
stream(fessConfig.getJvmCrawlerOptionsAsArray()).of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
File ownTmpDir = null;
final String tmpDir = System.getProperty("java.io.tmpdir");
if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
if (ownTmpDir.mkdirs()) {
cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
cmdList.add("-Dpdfbox.fontcache=" + ownTmpDir.getAbsolutePath());
} else {
ownTmpDir = null;
}
}
cmdList.add(ComponentUtil.getThumbnailManager().getThumbnailPathOption());
if (!jvmOptions.isEmpty()) {
jvmOptions.stream().filter(StringUtil::isNotBlank).forEach(cmdList::add);
}
cmdList.add(Crawler.class.getCanonicalName());
cmdList.add("--sessionId");
cmdList.add(sessionId);
cmdList.add("--name");
cmdList.add(namespace);
if (webConfigIds != null && webConfigIds.length > 0) {
cmdList.add("-w");
cmdList.add(StringUtils.join(webConfigIds, ','));
}
if (fileConfigIds != null && fileConfigIds.length > 0) {
cmdList.add("-f");
cmdList.add(StringUtils.join(fileConfigIds, ','));
}
if (dataConfigIds != null && dataConfigIds.length > 0) {
cmdList.add("-d");
cmdList.add(StringUtils.join(dataConfigIds, ','));
}
if (documentExpires >= -1) {
cmdList.add("-e");
cmdList.add(Integer.toString(documentExpires));
}
final File propFile = ComponentUtil.getSystemHelper().createTempFile(getExecuteType() + "_", ".properties");
try {
cmdList.add("-p");
cmdList.add(propFile.getAbsolutePath());
createSystemProperties(cmdList, propFile);
final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
if (logger.isInfoEnabled()) {
logger.info("Crawler: \nDirectory={}\nOptions={}", baseDir, cmdList);
}
final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
pb.directory(baseDir);
pb.redirectErrorStream(true);
});
final InputStreamThread it = jobProcess.getInputStreamThread();
it.start();
final Process currentProcess = jobProcess.getProcess();
currentProcess.waitFor();
it.join(5000);
final int exitValue = currentProcess.exitValue();
if (logger.isInfoEnabled()) {
logger.info("Crawler: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
}
if (exitValue != 0) {
final StringBuilder out = new StringBuilder();
if (processTimeout) {
out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
}
out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
throw new JobProcessingException(out.toString());
}
} catch (final JobProcessingException e) {
throw e;
} catch (final Exception e) {
throw new JobProcessingException("Crawler Process terminated.", e);
} finally {
try {
processHelper.destroyProcess(sessionId);
} finally {
if (propFile != null && !propFile.delete()) {
logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
}
deleteTempDir(ownTmpDir);
}
}
}
use of org.codelibs.fess.helper.ProcessHelper in project fess by codelibs.
the class GenerateThumbnailJob method executeThumbnailGenerator.
protected void executeThumbnailGenerator() {
final List<String> cmdList = new ArrayList<>();
final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
cmdList.add(fessConfig.getJavaCommandPath());
// -cp
cmdList.add("-cp");
final StringBuilder buf = new StringBuilder(100);
ResourceUtil.getOverrideConfPath().ifPresent(p -> {
buf.append(p);
buf.append(cpSeparator);
});
final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
if (StringUtil.isNotBlank(confPath)) {
buf.append(confPath);
buf.append(cpSeparator);
}
// WEB-INF/env/thumbnail/resources
buf.append("WEB-INF");
buf.append(File.separator);
buf.append("env");
buf.append(File.separator);
buf.append(getExecuteType());
buf.append(File.separator);
buf.append("resources");
buf.append(cpSeparator);
// WEB-INF/classes
buf.append("WEB-INF");
buf.append(File.separator);
buf.append("classes");
// target/classes
final String userDir = System.getProperty("user.dir");
final File targetDir = new File(userDir, "target");
final File targetClassesDir = new File(targetDir, "classes");
if (targetClassesDir.isDirectory()) {
buf.append(cpSeparator);
buf.append(targetClassesDir.getAbsolutePath());
}
// WEB-INF/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib" + File.separator);
// WEB-INF/env/thumbnail/lib
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
// WEB-INF/plugin
appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin" + File.separator);
final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
if (targetLibDir.isDirectory()) {
appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
}
cmdList.add(buf.toString());
if (useLocalFesen) {
final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
if (StringUtil.isNotBlank(httpAddress)) {
cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
}
}
final String systemLastaEnv = System.getProperty("lasta.env");
if (StringUtil.isNotBlank(systemLastaEnv)) {
if ("web".equals(systemLastaEnv)) {
cmdList.add("-Dlasta.env=" + getExecuteType());
} else {
cmdList.add("-Dlasta.env=" + systemLastaEnv);
}
} else if (StringUtil.isNotBlank(lastaEnv)) {
cmdList.add("-Dlasta.env=" + lastaEnv);
} else {
cmdList.add("-Dlasta.env=" + getExecuteType());
}
addFessConfigProperties(cmdList);
addFessSystemProperties(cmdList);
addSystemProperty(cmdList, Constants.FESS_CONF_PATH, null, null);
cmdList.add("-Dfess." + getExecuteType() + ".process=true");
if (logFilePath == null) {
final String value = System.getProperty("fess.log.path");
logFilePath = value != null ? value : new File(targetDir, "logs").getAbsolutePath();
}
cmdList.add("-Dfess.log.path=" + logFilePath);
addSystemProperty(cmdList, Constants.FESS_VAR_PATH, null, null);
addSystemProperty(cmdList, Constants.FESS_THUMBNAIL_PATH, null, null);
addSystemProperty(cmdList, "fess.log.name", "fess-" + getExecuteType(), "-" + getExecuteType());
if (logLevel != null) {
cmdList.add("-Dfess.log.level=" + logLevel);
}
stream(fessConfig.getJvmThumbnailOptionsAsArray()).of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
File ownTmpDir = null;
final String tmpDir = System.getProperty("java.io.tmpdir");
if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
if (ownTmpDir.mkdirs()) {
cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
} else {
ownTmpDir = null;
}
}
if (!jvmOptions.isEmpty()) {
jvmOptions.stream().filter(StringUtil::isNotBlank).forEach(cmdList::add);
}
cmdList.add(ThumbnailGenerator.class.getCanonicalName());
cmdList.add("--sessionId");
cmdList.add(sessionId);
cmdList.add("--numOfThreads");
cmdList.add(Integer.toString(numOfThreads));
if (cleanup) {
cmdList.add("--cleanup");
}
final File propFile = ComponentUtil.getSystemHelper().createTempFile(getExecuteType() + "_", ".properties");
try {
cmdList.add("-p");
cmdList.add(propFile.getAbsolutePath());
createSystemProperties(cmdList, propFile);
final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
if (logger.isInfoEnabled()) {
logger.info("ThumbnailGenerator: \nDirectory={}\nOptions={}", baseDir, cmdList);
}
final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
pb.directory(baseDir);
pb.redirectErrorStream(true);
});
final InputStreamThread it = jobProcess.getInputStreamThread();
it.start();
final Process currentProcess = jobProcess.getProcess();
currentProcess.waitFor();
it.join(5000);
final int exitValue = currentProcess.exitValue();
if (logger.isInfoEnabled()) {
logger.info("ThumbnailGenerator: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
}
if (exitValue != 0) {
final StringBuilder out = new StringBuilder();
if (processTimeout) {
out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
}
out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
throw new JobProcessingException(out.toString());
}
ComponentUtil.getPopularWordHelper().clearCache();
} catch (final JobProcessingException e) {
throw e;
} catch (final Exception e) {
throw new JobProcessingException("ThumbnailGenerator Process terminated.", e);
} finally {
try {
processHelper.destroyProcess(sessionId);
} finally {
if (propFile != null && !propFile.delete()) {
logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
}
deleteTempDir(ownTmpDir);
}
}
}
use of org.codelibs.fess.helper.ProcessHelper in project fess by codelibs.
the class PythonJob method executePython.
protected void executePython() {
final List<String> cmdList = new ArrayList<>();
final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
final FessConfig fessConfig = ComponentUtil.getFessConfig();
if (StringUtil.isBlank(filename)) {
throw new JobProcessingException("Python script is not specified.");
}
cmdList.add(fessConfig.getPythonCommandPath());
cmdList.add(getPyFilePath());
cmdList.addAll(argList);
try {
final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
if (logger.isInfoEnabled()) {
logger.info("Python: \nDirectory={}\nOptions={}", baseDir, cmdList);
}
final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
pb.directory(baseDir);
pb.redirectErrorStream(true);
});
final InputStreamThread it = jobProcess.getInputStreamThread();
it.start();
final Process currentProcess = jobProcess.getProcess();
currentProcess.waitFor();
it.join(5000);
final int exitValue = currentProcess.exitValue();
if (logger.isInfoEnabled()) {
logger.info("Python: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
}
if (exitValue != 0) {
final StringBuilder out = new StringBuilder();
if (processTimeout) {
out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
}
out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
throw new JobProcessingException(out.toString());
}
ComponentUtil.getPopularWordHelper().clearCache();
} catch (final JobProcessingException e) {
throw e;
} catch (final Exception e) {
throw new JobProcessingException("Python Process terminated.", e);
} finally {
processHelper.destroyProcess(sessionId);
}
}
Aggregations