Examples with SystemHelper org.codelibs.fess.helper.SystemHelper used on opensource projects

Example 31 with SystemHelper

use of org.codelibs.fess.helper.SystemHelper in project fess by codelibs.

the class IndexUpdateCallbackImpl method store.

/* (non-Javadoc)
     * @see org.codelibs.fess.ds.callback.IndexUpdateCallback#store(java.util.Map)
     */
@Override
public void store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    systemHelper.calibrateCpuLoad();
    final long startTime = System.currentTimeMillis();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
    if (logger.isDebugEnabled()) {
        logger.debug("Adding {}", dataMap);
    }
    // required check
    final Object urlObj = dataMap.get(fessConfig.getIndexFieldUrl());
    if (urlObj == null) {
        throw new DataStoreException("url is null. dataMap=" + dataMap);
    }
    final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
    final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
    dataMap.put(fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
    final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();
    if (fessConfig.getIndexerClickCountEnabledAsBoolean()) {
        addClickCountField(dataMap, url, fessConfig.getIndexFieldClickCount());
    }
    if (fessConfig.getIndexerFavoriteCountEnabledAsBoolean()) {
        addFavoriteCountField(dataMap, url, fessConfig.getIndexFieldFavoriteCount());
    }
    final Set<String> matchedLabelSet = ComponentUtil.getLabelTypeHelper().getMatchedLabelValueSet(url);
    if (!matchedLabelSet.isEmpty()) {
        final Set<String> newLabelSet = new HashSet<>();
        final String[] oldLabels = DocumentUtil.getValue(dataMap, fessConfig.getIndexFieldLabel(), String[].class);
        StreamUtil.stream(oldLabels).of(stream -> stream.forEach(newLabelSet::add));
        matchedLabelSet.stream().forEach(newLabelSet::add);
        dataMap.put(fessConfig.getIndexFieldLabel(), newLabelSet.toArray(new String[newLabelSet.size()]));
    }
    if (!dataMap.containsKey(fessConfig.getIndexFieldDocId())) {
        dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
    }
    ComponentUtil.getLanguageHelper().updateDocument(dataMap);
    synchronized (docList) {
        docList.add(ingest(paramMap, dataMap));
        final long contentSize = indexingHelper.calculateDocumentSize(dataMap);
        docList.addContentSize(contentSize);
        final long processingTime = System.currentTimeMillis() - startTime;
        docList.addProcessingTime(processingTime);
        if (logger.isDebugEnabled()) {
            logger.debug("Added the document({}, {}ms). The number of a document cache is {}.", MemoryUtil.byteCountToDisplaySize(contentSize), processingTime, docList.size());
        }
        if (docList.getContentSize() >= maxDocumentRequestSize || docList.size() >= maxDocumentCacheSize) {
            indexingHelper.sendDocuments(searchEngineClient, docList);
        }
        executeTime += processingTime;
    }
    documentSize.getAndIncrement();
    if (logger.isDebugEnabled()) {
        logger.debug("The number of an added document is {}.", documentSize.get());
    }
}

Example 32 with SystemHelper

use of org.codelibs.fess.helper.SystemHelper in project fess by codelibs.

the class WebConfig method initDocUrlPattern.

protected synchronized void initDocUrlPattern() {
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    if (includedDocUrlPatterns == null) {
        if (StringUtil.isNotBlank(getIncludedDocUrls())) {
            final List<Pattern> urlPatterList = new ArrayList<>();
            final String[] urls = getIncludedDocUrls().split("[\r\n]");
            for (final String u : urls) {
                final String v = systemHelper.normalizeConfigPath(u);
                if (StringUtil.isNotBlank(v)) {
                    urlPatterList.add(Pattern.compile(v));
                }
            }
            includedDocUrlPatterns = urlPatterList.toArray(new Pattern[urlPatterList.size()]);
        } else {
            includedDocUrlPatterns = new Pattern[0];
        }
    }
    if (excludedDocUrlPatterns == null) {
        if (StringUtil.isNotBlank(getExcludedDocUrls())) {
            final List<Pattern> urlPatterList = new ArrayList<>();
            final String[] urls = getExcludedDocUrls().split("[\r\n]");
            for (final String u : urls) {
                final String v = systemHelper.normalizeConfigPath(u);
                if (StringUtil.isNotBlank(v)) {
                    urlPatterList.add(Pattern.compile(v));
                }
            }
            excludedDocUrlPatterns = urlPatterList.toArray(new Pattern[urlPatterList.size()]);
        } else if (includedDocUrlPatterns.length > 0) {
            excludedDocUrlPatterns = new Pattern[] { Pattern.compile(".*") };
        } else {
            excludedDocUrlPatterns = new Pattern[0];
        }
    }
}

Example 33 with SystemHelper

use of org.codelibs.fess.helper.SystemHelper in project fess by codelibs.

the class CrawlJob method executeCrawler.

protected void executeCrawler() {
    final List<String> cmdList = new ArrayList<>();
    final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
    final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    cmdList.add(fessConfig.getJavaCommandPath());
    // -cp
    cmdList.add("-cp");
    final StringBuilder buf = new StringBuilder(100);
    ResourceUtil.getOverrideConfPath().ifPresent(p -> {
        buf.append(p);
        buf.append(cpSeparator);
    });
    final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
    if (StringUtil.isNotBlank(confPath)) {
        buf.append(confPath);
        buf.append(cpSeparator);
    }
    // WEB-INF/env/crawler/resources
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("env");
    buf.append(File.separator);
    buf.append(getExecuteType());
    buf.append(File.separator);
    buf.append("resources");
    buf.append(cpSeparator);
    // WEB-INF/classes
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("classes");
    // target/classes
    final String userDir = System.getProperty("user.dir");
    final File targetDir = new File(userDir, "target");
    final File targetClassesDir = new File(targetDir, "classes");
    if (targetClassesDir.isDirectory()) {
        buf.append(cpSeparator);
        buf.append(targetClassesDir.getAbsolutePath());
    }
    // WEB-INF/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF" + File.separator + "lib" + File.separator);
    // WEB-INF/env/crawler/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/env/" + getExecuteType() + "/lib")), "WEB-INF" + File.separator + "env" + File.separator + getExecuteType() + File.separator + "lib" + File.separator);
    // WEB-INF/plugin
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/plugin")), "WEB-INF" + File.separator + "plugin" + File.separator);
    final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
    if (targetLibDir.isDirectory()) {
        appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
    }
    cmdList.add(buf.toString());
    if (useLocalFesen) {
        final String httpAddress = System.getProperty(Constants.FESS_ES_HTTP_ADDRESS);
        if (StringUtil.isNotBlank(httpAddress)) {
            cmdList.add("-D" + Constants.FESS_ES_HTTP_ADDRESS + "=" + httpAddress);
        }
    }
    final String systemLastaEnv = System.getProperty("lasta.env");
    if (StringUtil.isNotBlank(systemLastaEnv)) {
        if ("web".equals(systemLastaEnv)) {
            cmdList.add("-Dlasta.env=" + getExecuteType());
        } else {
            cmdList.add("-Dlasta.env=" + systemLastaEnv);
        }
    } else if (StringUtil.isNotBlank(lastaEnv)) {
        cmdList.add("-Dlasta.env=" + lastaEnv);
    } else {
        cmdList.add("-Dlasta.env=" + getExecuteType());
    }
    addFessConfigProperties(cmdList);
    addFessSystemProperties(cmdList);
    addSystemProperty(cmdList, Constants.FESS_CONF_PATH, null, null);
    cmdList.add("-Dfess." + getExecuteType() + ".process=true");
    cmdList.add("-Dfess.log.path=" + (logFilePath != null ? logFilePath : systemHelper.getLogFilePath()));
    addSystemProperty(cmdList, "fess.log.name", "fess-" + getExecuteType(), "-" + getExecuteType());
    if (logLevel == null) {
        addSystemProperty(cmdList, "fess.log.level", null, null);
    } else {
        cmdList.add("-Dfess.log.level=" + logLevel);
        if ("debug".equalsIgnoreCase(logLevel)) {
            cmdList.add("-Dorg.apache.tika.service.error.warn=true");
        }
    }
    stream(fessConfig.getJvmCrawlerOptionsAsArray()).of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
    File ownTmpDir = null;
    final String tmpDir = System.getProperty("java.io.tmpdir");
    if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
        ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
        if (ownTmpDir.mkdirs()) {
            cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
            cmdList.add("-Dpdfbox.fontcache=" + ownTmpDir.getAbsolutePath());
        } else {
            ownTmpDir = null;
        }
    }
    cmdList.add(ComponentUtil.getThumbnailManager().getThumbnailPathOption());
    if (!jvmOptions.isEmpty()) {
        jvmOptions.stream().filter(StringUtil::isNotBlank).forEach(cmdList::add);
    }
    cmdList.add(Crawler.class.getCanonicalName());
    cmdList.add("--sessionId");
    cmdList.add(sessionId);
    cmdList.add("--name");
    cmdList.add(namespace);
    if (webConfigIds != null && webConfigIds.length > 0) {
        cmdList.add("-w");
        cmdList.add(StringUtils.join(webConfigIds, ','));
    }
    if (fileConfigIds != null && fileConfigIds.length > 0) {
        cmdList.add("-f");
        cmdList.add(StringUtils.join(fileConfigIds, ','));
    }
    if (dataConfigIds != null && dataConfigIds.length > 0) {
        cmdList.add("-d");
        cmdList.add(StringUtils.join(dataConfigIds, ','));
    }
    if (documentExpires >= -1) {
        cmdList.add("-e");
        cmdList.add(Integer.toString(documentExpires));
    }
    final File propFile = ComponentUtil.getSystemHelper().createTempFile(getExecuteType() + "_", ".properties");
    try {
        cmdList.add("-p");
        cmdList.add(propFile.getAbsolutePath());
        createSystemProperties(cmdList, propFile);
        final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
        if (logger.isInfoEnabled()) {
            logger.info("Crawler: \nDirectory={}\nOptions={}", baseDir, cmdList);
        }
        final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
            pb.directory(baseDir);
            pb.redirectErrorStream(true);
        });
        final InputStreamThread it = jobProcess.getInputStreamThread();
        it.start();
        final Process currentProcess = jobProcess.getProcess();
        currentProcess.waitFor();
        it.join(5000);
        final int exitValue = currentProcess.exitValue();
        if (logger.isInfoEnabled()) {
            logger.info("Crawler: Exit Code={} - Process Output:\n{}", exitValue, it.getOutput());
        }
        if (exitValue != 0) {
            final StringBuilder out = new StringBuilder();
            if (processTimeout) {
                out.append("Process is terminated due to ").append(timeout).append(" second exceeded.\n");
            }
            out.append("Exit Code: ").append(exitValue).append("\nOutput:\n").append(it.getOutput());
            throw new JobProcessingException(out.toString());
        }
    } catch (final JobProcessingException e) {
        throw e;
    } catch (final Exception e) {
        throw new JobProcessingException("Crawler Process terminated.", e);
    } finally {
        try {
            processHelper.destroyProcess(sessionId);
        } finally {
            if (propFile != null && !propFile.delete()) {
                logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
            }
            deleteTempDir(ownTmpDir);
        }
    }
}

Example 34 with SystemHelper

use of org.codelibs.fess.helper.SystemHelper in project fess by codelibs.

the class PingSearchEngineJob method execute.

public String execute() {
    final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    final StringBuilder resultBuf = new StringBuilder();
    final PingResponse ping = searchEngineClient.ping();
    final int status = ping.getStatus();
    if (systemHelper.isChangedClusterState(status)) {
        if (fessConfig.hasNotification()) {
            final String toStrs = fessConfig.getNotificationTo();
            final String[] toAddresses;
            if (StringUtil.isNotBlank(toStrs)) {
                toAddresses = toStrs.split(",");
            } else {
                toAddresses = StringUtil.EMPTY_STRINGS;
            }
            final Postbox postbox = ComponentUtil.getComponent(Postbox.class);
            try {
                final NotificationHelper notificationHelper = ComponentUtil.getNotificationHelper();
                SMailCallbackContext.setPreparedMessageHookOnThread(notificationHelper::send);
                EsStatusPostcard.droppedInto(postbox, postcard -> {
                    postcard.setFrom(fessConfig.getMailFromAddress(), fessConfig.getMailFromName());
                    postcard.addReplyTo(fessConfig.getMailReturnPath());
                    if (toAddresses.length > 0) {
                        stream(toAddresses).of(stream -> stream.map(String::trim).forEach(address -> {
                            postcard.addTo(address);
                        }));
                    } else {
                        postcard.addTo(fessConfig.getMailFromAddress());
                        postcard.dryrun();
                    }
                    postcard.setHostname(systemHelper.getHostname());
                    postcard.setClustername(ping.getClusterName());
                    postcard.setClusterstatus(ping.getClusterStatus());
                });
            } catch (final Exception e) {
                logger.warn("Failed to send a test mail.", e);
            } finally {
                SMailCallbackContext.clearPreparedMessageHookOnThread();
            }
        }
        resultBuf.append("Status of ").append(ping.getClusterName()).append(" is changed to ").append(ping.getClusterStatus()).append('.');
    } else if (status == 0) {
        resultBuf.append(ping.getClusterName()).append(" is alive.");
    } else {
        resultBuf.append(ping.getClusterName()).append(" is not available.");
    }
    return resultBuf.toString();
}

Example 35 with SystemHelper

use of org.codelibs.fess.helper.SystemHelper in project fess by codelibs.

the class ScriptExecutorJob method process.

protected void process(final LaJobRuntime runtime) {
    if (!runtime.getParameterMap().containsKey(Constants.SCHEDULED_JOB)) {
        logger.warn("{} is empty.", Constants.SCHEDULED_JOB);
        return;
    }
    runtime.stopIfNeeds();
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    final JobManager jobManager = ComponentUtil.getJobManager();
    final ScheduledJob scheduledJob = (ScheduledJob) runtime.getParameterMap().get(Constants.SCHEDULED_JOB);
    final String id = scheduledJob.getId();
    final String target = scheduledJob.getTarget();
    if (!ComponentUtil.getFessConfig().isSchedulerTarget(target)) {
        logger.info("Ignore Job {}:{} because of not target: {}", scheduledJob.getName(), id, scheduledJob.getTarget());
        return;
    }
    final JobHelper jobHelper = ComponentUtil.getJobHelper();
    if (!jobHelper.isAvailable(id)) {
        logger.info("Job {} is unavailable. Unregistering this job.", id);
        jobHelper.unregister(scheduledJob);
        return;
    }
    final JobLog jobLog = new JobLog(scheduledJob);
    final String scriptType = scheduledJob.getScriptType();
    final String script = scheduledJob.getScriptData();
    final JobExecutor jobExecutor = ComponentUtil.getJobExecutor(scriptType);
    if (jobExecutor == null) {
        throw new ScheduledJobException("No jobExecutor: " + scriptType);
    }
    if (!jobManager.findJobByUniqueOf(LaJobUnique.of(id)).isPresent()) {
        if (logger.isDebugEnabled()) {
            logger.debug("Job {} is running.", id);
        }
        return;
    }
    TimeoutTask task = null;
    try {
        if (scheduledJob.isLoggingEnabled()) {
            jobHelper.store(jobLog);
            task = jobHelper.startMonitorTask(jobLog);
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Starting Job {}. scriptType: {}, script: {}", id, scriptType, script);
        } else if (scheduledJob.isLoggingEnabled() && logger.isInfoEnabled()) {
            logger.info("Starting Job {}.", id);
        }
        final Object ret = jobExecutor.execute(Constants.DEFAULT_SCRIPT, script);
        if (ret == null) {
            if (scheduledJob.isLoggingEnabled() && logger.isInfoEnabled()) {
                logger.info("Finished Job {}.", id);
            }
        } else {
            if (scheduledJob.isLoggingEnabled() && logger.isInfoEnabled()) {
                logger.info("Finished Job {}. The return value is:\n{}", id, ret);
            }
            jobLog.setScriptResult(ret.toString());
        }
        jobLog.setJobStatus(Constants.OK);
    } catch (final Throwable t) {
        logger.warn("Failed to execute {}: {}", id, script, t);
        jobLog.setJobStatus(Constants.FAIL);
        jobLog.setScriptResult(systemHelper.abbreviateLongText(t.getLocalizedMessage()));
    } finally {
        if (task != null) {
            try {
                task.stop();
            } catch (final Exception e) {
                logger.warn("Failed to stop {}", jobLog, e);
            }
        }
        jobLog.setEndTime(ComponentUtil.getSystemHelper().getCurrentTimeAsLong());
        if (logger.isDebugEnabled()) {
            logger.debug("jobLog: {}", jobLog);
        }
        if (scheduledJob.isLoggingEnabled()) {
            jobHelper.store(jobLog);
        }
    }
}