Examples with JobProcessingException org.codelibs.fess.exception.JobProcessingException used on opensource projects

Example 6 with JobProcessingException

use of org.codelibs.fess.exception.JobProcessingException in project fess by codelibs.

the class AbstractDataStoreTest method setUp.

@Override
public void setUp() throws Exception {
    super.setUp();
    dataStore = new AbstractDataStore() {

        @Override
        protected String getName() {
            return "Test";
        }

        @Override
        protected void storeData(DataConfig dataConfig, IndexUpdateCallback callback, Map<String, String> paramMap, Map<String, String> scriptMap, Map<String, Object> defaultDataMap) {
        // TODO nothing
        }
    };
    ScriptEngineFactory scriptEngineFactory = new ScriptEngineFactory();
    ComponentUtil.register(scriptEngineFactory, "scriptEngineFactory");
    new AbstractScriptEngine() {

        @Override
        public Object evaluate(String template, Map<String, Object> paramMap) {
            final Map<String, Object> bindingMap = new HashMap<>(paramMap);
            bindingMap.put("container", SingletonLaContainerFactory.getContainer());
            final GroovyShell groovyShell = new GroovyShell(new Binding(bindingMap));
            try {
                return groovyShell.evaluate(template);
            } catch (final JobProcessingException e) {
                throw e;
            } catch (final Exception e) {
                return null;
            } finally {
                final GroovyClassLoader loader = groovyShell.getClassLoader();
                loader.clearCache();
            }
        }

        @Override
        protected String getName() {
            return Constants.DEFAULT_SCRIPT;
        }
    }.register();
}

Example 7 with JobProcessingException

use of org.codelibs.fess.exception.JobProcessingException in project fess by codelibs.

the class CrawlJob method executeCrawler.

protected void executeCrawler() {
    final List<String> cmdList = new ArrayList<>();
    final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
    final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    cmdList.add(fessConfig.getJavaCommandPath());
    // -cp
    cmdList.add("-cp");
    final StringBuilder buf = new StringBuilder(100);
    ResourceUtil.getOverrideConfPath().ifPresent(p -> {
        buf.append(p);
        buf.append(cpSeparator);
    });
    final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
    if (StringUtil.isNotBlank(confPath)) {
        buf.append(confPath);
        buf.append(cpSeparator);
    }
    // WEB-INF/env/crawler/resources
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("env");
    buf.append(File.separator);
    buf.append(getExecuteType());
    buf.append(File.separator);
    buf.append("resources");
    buf.append(cpSeparator);
    // WEB-INF/classes
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("classes");
    // target/classes
    final String userDir = System.getProperty("user.dir");
    final File targetDir = new File(userDir, "target");
    final File targetClassesDir = new File(targetDir, "classes");
    if (targetClassesDir.isDirectory()) {
        buf.append(cpSeparator);
        buf.append(targetClassesDir.getAbsolutePath());
    }
    // WEB-INF/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib" + File.separator);
    // WEB-INF/env/crawler/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
    // WEB-INF/plugin
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin" + File.separator);
    final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
    if (targetLibDir.isDirectory()) {
        appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
    }
    cmdList.add(buf.toString());
    if (useLocalFesen) {
        final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
        if (StringUtil.isNotBlank(httpAddress)) {
            cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
        }
    }
    final String systemLastaEnv = System.getProperty("lasta.env");
    if (StringUtil.isNotBlank(systemLastaEnv)) {
        if ("web".equals(systemLastaEnv)) {
            cmdList.add("-Dlasta.env=" + getExecuteType());
        } else {
            cmdList.add("-Dlasta.env=" + systemLastaEnv);
        }
    } else if (StringUtil.isNotBlank(lastaEnv)) {
        cmdList.add("-Dlasta.env=" + lastaEnv);
    } else {
        cmdList.add("-Dlasta.env=" + getExecuteType());
    }
    addFessConfigProperties(cmdList);
    addFessSystemProperties(cmdList);
    addSystemProperty(cmdList, Constants.FESS_CONF_PATH, null, null);
    cmdList.add("-Dfess." + getExecuteType() + ".process=true");
    cmdList.add("-Dfess.log.path=" + (logFilePath != null ? logFilePath : systemHelper.getLogFilePath()));
    addSystemProperty(cmdList, "fess.log.name", "fess-" + getExecuteType(), "-" + getExecuteType());
    if (logLevel == null) {
        addSystemProperty(cmdList, "fess.log.level", null, null);
    } else {
        cmdList.add("-Dfess.log.level=" + logLevel);
        if ("debug".equalsIgnoreCase(logLevel)) {
            cmdList.add("-Dorg.apache.tika.service.error.warn=true");
        }
    }
    stream(fessConfig.getJvmCrawlerOptionsAsArray()).of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
    File ownTmpDir = null;
    final String tmpDir = System.getProperty("java.io.tmpdir");
    if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
        ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
        if (ownTmpDir.mkdirs()) {
            cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
            cmdList.add("-Dpdfbox.fontcache=" + ownTmpDir.getAbsolutePath());
        } else {
            ownTmpDir = null;
        }
    }
    cmdList.add(ComponentUtil.getThumbnailManager().getThumbnailPathOption());
    if (!jvmOptions.isEmpty()) {
        jvmOptions.stream().filter(StringUtil::isNotBlank).forEach(cmdList::add);
    }
    cmdList.add(Crawler.class.getCanonicalName());
    cmdList.add("--sessionId");
    cmdList.add(sessionId);
    cmdList.add("--name");
    cmdList.add(namespace);
    if (webConfigIds != null && webConfigIds.length > 0) {
        cmdList.add("-w");
        cmdList.add(StringUtils.join(webConfigIds, ','));
    }
    if (fileConfigIds != null && fileConfigIds.length > 0) {
        cmdList.add("-f");
        cmdList.add(StringUtils.join(fileConfigIds, ','));
    }
    if (dataConfigIds != null && dataConfigIds.length > 0) {
        cmdList.add("-d");
        cmdList.add(StringUtils.join(dataConfigIds, ','));
    }
    if (documentExpires >= -1) {
        cmdList.add("-e");
        cmdList.add(Integer.toString(documentExpires));
    }
    final File propFile = ComponentUtil.getSystemHelper().createTempFile(getExecuteType() + "_", ".properties");
    try {
        cmdList.add("-p");
        cmdList.add(propFile.getAbsolutePath());
        createSystemProperties(cmdList, propFile);
        final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
        if (logger.isInfoEnabled()) {
            logger.info("Crawler: \nDirectory={}\nOptions={}", baseDir, cmdList);
        }
        final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
            pb.directory(baseDir);
            pb.redirectErrorStream(true);
        });
        final InputStreamThread it = jobProcess.getInputStreamThread();
        it.start();
        final Process currentProcess = jobProcess.getProcess();
        currentProcess.waitFor();
        it.join(5000);
        final int exitValue = currentProcess.exitValue();
        if (logger.isInfoEnabled()) {
            logger.info("Crawler: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
        }
        if (exitValue != 0) {
            final StringBuilder out = new StringBuilder();
            if (processTimeout) {
                out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
            }
            out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
            throw new JobProcessingException(out.toString());
        }
    } catch (final JobProcessingException e) {
        throw e;
    } catch (final Exception e) {
        throw new JobProcessingException("Crawler Process terminated.", e);
    } finally {
        try {
            processHelper.destroyProcess(sessionId);
        } finally {
            if (propFile != null && !propFile.delete()) {
                logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
            }
            deleteTempDir(ownTmpDir);
        }
    }
}

Example 8 with JobProcessingException

use of org.codelibs.fess.exception.JobProcessingException in project fess by codelibs.

the class CrawlJob method execute.

@Override
public String execute() {
    // check # of crawler processes
    final int maxCrawlerProcesses = ComponentUtil.getFessConfig().getJobMaxCrawlerProcessesAsInteger();
    if (maxCrawlerProcesses > 0) {
        final int runningJobCount = getRunningJobCount();
        if (runningJobCount > maxCrawlerProcesses) {
            throw new JobProcessingException(runningJobCount + " crawler processes are running. Max processes are " + maxCrawlerProcesses + ".");
        }
    }
    final StringBuilder resultBuf = new StringBuilder(100);
    final boolean runAll = webConfigIds == null && fileConfigIds == null && dataConfigIds == null;
    if (sessionId == null) {
        // create session id
        final SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMddHHmmss");
        sessionId = sdf.format(new Date());
    }
    resultBuf.append("Session Id: ").append(sessionId).append("\n");
    resultBuf.append("Web  Config Id:");
    if (webConfigIds == null) {
        if (runAll) {
            resultBuf.append(" ALL\n");
        } else {
            resultBuf.append(" NONE\n");
        }
    } else {
        for (final String id : webConfigIds) {
            resultBuf.append(' ').append(id);
        }
        resultBuf.append('\n');
    }
    resultBuf.append("File Config Id:");
    if (fileConfigIds == null) {
        if (runAll) {
            resultBuf.append(" ALL\n");
        } else {
            resultBuf.append(" NONE\n");
        }
    } else {
        for (final String id : fileConfigIds) {
            resultBuf.append(' ').append(id);
        }
        resultBuf.append('\n');
    }
    resultBuf.append("Data Config Id:");
    if (dataConfigIds == null) {
        if (runAll) {
            resultBuf.append(" ALL\n");
        } else {
            resultBuf.append(" NONE\n");
        }
    } else {
        for (final String id : dataConfigIds) {
            resultBuf.append(' ').append(id);
        }
        resultBuf.append('\n');
    }
    if (jobExecutor != null) {
        jobExecutor.addShutdownListener(() -> ComponentUtil.getProcessHelper().destroyProcess(sessionId));
    }
    final TimeoutTask timeoutTask = createTimeoutTask();
    try {
        executeCrawler();
        ComponentUtil.getKeyMatchHelper().update();
    } catch (final JobProcessingException e) {
        throw e;
    } catch (final Exception e) {
        throw new JobProcessingException("Failed to execute a crawl job.", e);
    } finally {
        if (timeoutTask != null && !timeoutTask.isCanceled()) {
            timeoutTask.cancel();
        }
    }
    return resultBuf.toString();
}

Example 9 with JobProcessingException

use of org.codelibs.fess.exception.JobProcessingException in project fess by codelibs.

the class GenerateThumbnailJob method executeThumbnailGenerator.

protected void executeThumbnailGenerator() {
    final List<String> cmdList = new ArrayList<>();
    final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
    final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
    final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    cmdList.add(fessConfig.getJavaCommandPath());
    // -cp
    cmdList.add("-cp");
    final StringBuilder buf = new StringBuilder(100);
    ResourceUtil.getOverrideConfPath().ifPresent(p -> {
        buf.append(p);
        buf.append(cpSeparator);
    });
    final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
    if (StringUtil.isNotBlank(confPath)) {
        buf.append(confPath);
        buf.append(cpSeparator);
    }
    // WEB-INF/env/thumbnail/resources
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("env");
    buf.append(File.separator);
    buf.append(getExecuteType());
    buf.append(File.separator);
    buf.append("resources");
    buf.append(cpSeparator);
    // WEB-INF/classes
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("classes");
    // target/classes
    final String userDir = System.getProperty("user.dir");
    final File targetDir = new File(userDir, "target");
    final File targetClassesDir = new File(targetDir, "classes");
    if (targetClassesDir.isDirectory()) {
        buf.append(cpSeparator);
        buf.append(targetClassesDir.getAbsolutePath());
    }
    // WEB-INF/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib" + File.separator);
    // WEB-INF/env/thumbnail/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
    // WEB-INF/plugin
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin" + File.separator);
    final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
    if (targetLibDir.isDirectory()) {
        appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
    }
    cmdList.add(buf.toString());
    if (useLocalFesen) {
        final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
        if (StringUtil.isNotBlank(httpAddress)) {
            cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
        }
    }
    final String systemLastaEnv = System.getProperty("lasta.env");
    if (StringUtil.isNotBlank(systemLastaEnv)) {
        if ("web".equals(systemLastaEnv)) {
            cmdList.add("-Dlasta.env=" + getExecuteType());
        } else {
            cmdList.add("-Dlasta.env=" + systemLastaEnv);
        }
    } else if (StringUtil.isNotBlank(lastaEnv)) {
        cmdList.add("-Dlasta.env=" + lastaEnv);
    } else {
        cmdList.add("-Dlasta.env=" + getExecuteType());
    }
    addFessConfigProperties(cmdList);
    addFessSystemProperties(cmdList);
    addSystemProperty(cmdList, Constants.FESS_CONF_PATH, null, null);
    cmdList.add("-Dfess." + getExecuteType() + ".process=true");
    if (logFilePath == null) {
        final String value = System.getProperty("fess.log.path");
        logFilePath = value != null ? value : new File(targetDir, "logs").getAbsolutePath();
    }
    cmdList.add("-Dfess.log.path=" + logFilePath);
    addSystemProperty(cmdList, Constants.FESS_VAR_PATH, null, null);
    addSystemProperty(cmdList, Constants.FESS_THUMBNAIL_PATH, null, null);
    addSystemProperty(cmdList, "fess.log.name", "fess-" + getExecuteType(), "-" + getExecuteType());
    if (logLevel != null) {
        cmdList.add("-Dfess.log.level=" + logLevel);
    }
    stream(fessConfig.getJvmThumbnailOptionsAsArray()).of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
    File ownTmpDir = null;
    final String tmpDir = System.getProperty("java.io.tmpdir");
    if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
        ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
        if (ownTmpDir.mkdirs()) {
            cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
        } else {
            ownTmpDir = null;
        }
    }
    if (!jvmOptions.isEmpty()) {
        jvmOptions.stream().filter(StringUtil::isNotBlank).forEach(cmdList::add);
    }
    cmdList.add(ThumbnailGenerator.class.getCanonicalName());
    cmdList.add("--sessionId");
    cmdList.add(sessionId);
    cmdList.add("--numOfThreads");
    cmdList.add(Integer.toString(numOfThreads));
    if (cleanup) {
        cmdList.add("--cleanup");
    }
    final File propFile = ComponentUtil.getSystemHelper().createTempFile(getExecuteType() + "_", ".properties");
    try {
        cmdList.add("-p");
        cmdList.add(propFile.getAbsolutePath());
        createSystemProperties(cmdList, propFile);
        final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
        if (logger.isInfoEnabled()) {
            logger.info("ThumbnailGenerator: \nDirectory={}\nOptions={}", baseDir, cmdList);
        }
        final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
            pb.directory(baseDir);
            pb.redirectErrorStream(true);
        });
        final InputStreamThread it = jobProcess.getInputStreamThread();
        it.start();
        final Process currentProcess = jobProcess.getProcess();
        currentProcess.waitFor();
        it.join(5000);
        final int exitValue = currentProcess.exitValue();
        if (logger.isInfoEnabled()) {
            logger.info("ThumbnailGenerator: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
        }
        if (exitValue != 0) {
            final StringBuilder out = new StringBuilder();
            if (processTimeout) {
                out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
            }
            out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
            throw new JobProcessingException(out.toString());
        }
        ComponentUtil.getPopularWordHelper().clearCache();
    } catch (final JobProcessingException e) {
        throw e;
    } catch (final Exception e) {
        throw new JobProcessingException("ThumbnailGenerator Process terminated.", e);
    } finally {
        try {
            processHelper.destroyProcess(sessionId);
        } finally {
            if (propFile != null && !propFile.delete()) {
                logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
            }
            deleteTempDir(ownTmpDir);
        }
    }
}

Example 10 with JobProcessingException

use of org.codelibs.fess.exception.JobProcessingException in project fess by codelibs.

the class PythonJob method executePython.

protected void executePython() {
    final List<String> cmdList = new ArrayList<>();
    final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
    final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    if (StringUtil.isBlank(filename)) {
        throw new JobProcessingException("Python script is not specified.");
    }
    cmdList.add(fessConfig.getPythonCommandPath());
    cmdList.add(getPyFilePath());
    cmdList.addAll(argList);
    try {
        final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
        if (logger.isInfoEnabled()) {
            logger.info("Python: \nDirectory={}\nOptions={}", baseDir, cmdList);
        }
        final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
            pb.directory(baseDir);
            pb.redirectErrorStream(true);
        });
        final InputStreamThread it = jobProcess.getInputStreamThread();
        it.start();
        final Process currentProcess = jobProcess.getProcess();
        currentProcess.waitFor();
        it.join(5000);
        final int exitValue = currentProcess.exitValue();
        if (logger.isInfoEnabled()) {
            logger.info("Python: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
        }
        if (exitValue != 0) {
            final StringBuilder out = new StringBuilder();
            if (processTimeout) {
                out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
            }
            out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
            throw new JobProcessingException(out.toString());
        }
        ComponentUtil.getPopularWordHelper().clearCache();
    } catch (final JobProcessingException e) {
        throw e;
    } catch (final Exception e) {
        throw new JobProcessingException("Python Process terminated.", e);
    } finally {
        processHelper.destroyProcess(sessionId);
    }
}