Examples with FessConfig org.codelibs.fess.mylasta.direction.FessConfig used on opensource projects

Example 26 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class CrawlingInfoHelper method getSessionIdList.

public List<Map<String, String>> getSessionIdList(final FessEsClient fessEsClient) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    return fessEsClient.search(fessConfig.getIndexDocumentSearchIndex(), fessConfig.getIndexDocumentType(), queryRequestBuilder -> {
        queryRequestBuilder.setQuery(QueryBuilders.matchAllQuery());
        final TermsAggregationBuilder termsBuilder = AggregationBuilders.terms(fessConfig.getIndexFieldSegment()).field(fessConfig.getIndexFieldSegment()).size(maxSessionIdsInList).order(Order.term(false));
        queryRequestBuilder.addAggregation(termsBuilder);
        queryRequestBuilder.setPreference(Constants.SEARCH_PREFERENCE_PRIMARY);
        return true;
    }, (queryRequestBuilder, execTime, searchResponse) -> {
        final List<Map<String, String>> sessionIdList = new ArrayList<>();
        searchResponse.ifPresent(response -> {
            final Terms terms = response.getAggregations().get(fessConfig.getIndexFieldSegment());
            for (final Bucket bucket : terms.getBuckets()) {
                final Map<String, String> map = new HashMap<>(2);
                map.put(fessConfig.getIndexFieldSegment(), bucket.getKey().toString());
                map.put(FACET_COUNT_KEY, Long.toString(bucket.getDocCount()));
                sessionIdList.add(map);
            }
        });
        return sessionIdList;
    });
}

Example 27 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class IndexingHelper method getChildDocumentList.

public List<Map<String, Object>> getChildDocumentList(final FessEsClient fessEsClient, final String id, final String[] fields) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final QueryBuilder queryBuilder = QueryBuilders.termQuery(fessConfig.getIndexFieldParentId(), id);
    return getDocumentListByQuery(fessEsClient, queryBuilder, fields);
}

Example 28 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class IndexingHelper method getDocumentListByQuery.

protected List<Map<String, Object>> getDocumentListByQuery(final FessEsClient fessEsClient, final QueryBuilder queryBuilder, final String[] fields) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final SearchResponse countResponse = fessEsClient.prepareSearch(fessConfig.getIndexDocumentUpdateIndex()).setTypes(fessConfig.getIndexDocumentType()).setQuery(queryBuilder).setSize(0).execute().actionGet(fessConfig.getIndexSearchTimeout());
    final long numFound = countResponse.getHits().getTotalHits();
    return fessEsClient.getDocumentList(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), requestBuilder -> {
        requestBuilder.setQuery(queryBuilder).setSize((int) numFound);
        if (fields != null) {
            requestBuilder.setFetchSource(fields, null);
        }
        return true;
    });
}

Example 29 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class IndexingHelper method sendDocuments.

public void sendDocuments(final FessEsClient fessEsClient, final DocList docList) {
    if (docList.isEmpty()) {
        return;
    }
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    if (fessConfig.isResultCollapsed()) {
        docList.forEach(doc -> {
            doc.put("content_minhash", doc.get(fessConfig.getIndexFieldContent()));
        });
    }
    final long execTime = System.currentTimeMillis();
    if (logger.isDebugEnabled()) {
        logger.debug("Sending " + docList.size() + " documents to a server.");
    }
    try {
        if (fessConfig.isThumbnailCrawlerEnabled()) {
            final ThumbnailManager thumbnailManager = ComponentUtil.getThumbnailManager();
            docList.stream().forEach(doc -> thumbnailManager.offer(doc));
        }
        synchronized (fessEsClient) {
            deleteOldDocuments(fessEsClient, docList);
            fessEsClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), fessConfig.getIndexDocumentType(), docList);
        }
        if (logger.isInfoEnabled()) {
            if (docList.getContentSize() > 0) {
                logger.info("Sent " + docList.size() + " docs (Doc:{process " + docList.getProcessingTime() + "ms, send " + (System.currentTimeMillis() - execTime) + "ms, size " + MemoryUtil.byteCountToDisplaySize(docList.getContentSize()) + "}, " + MemoryUtil.getMemoryUsageLog() + ")");
            } else {
                logger.info("Sent " + docList.size() + " docs (Doc:{send " + (System.currentTimeMillis() - execTime) + "ms}, " + MemoryUtil.getMemoryUsageLog() + ")");
            }
        }
    } finally {
        docList.clear();
    }
}

Example 30 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class SuggestJob method executeSuggestCreator.

protected void executeSuggestCreator() {
    final List<String> cmdList = new ArrayList<>();
    final String cpSeparator = SystemUtils.IS_OS_WINDOWS ? ";" : ":";
    final ServletContext servletContext = ComponentUtil.getComponent(ServletContext.class);
    final ProcessHelper processHelper = ComponentUtil.getProcessHelper();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    cmdList.add(fessConfig.getJavaCommandPath());
    // -cp
    cmdList.add("-cp");
    final StringBuilder buf = new StringBuilder(100);
    final String confPath = System.getProperty(Constants.FESS_CONF_PATH);
    if (StringUtil.isNotBlank(confPath)) {
        buf.append(confPath);
        buf.append(cpSeparator);
    }
    // WEB-INF/suggest/resources
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("suggest");
    buf.append(File.separator);
    buf.append("resources");
    buf.append(cpSeparator);
    // WEB-INF/classes
    buf.append("WEB-INF");
    buf.append(File.separator);
    buf.append("classes");
    // target/classes
    final String userDir = System.getProperty("user.dir");
    final File targetDir = new File(userDir, "target");
    final File targetClassesDir = new File(targetDir, "classes");
    if (targetClassesDir.isDirectory()) {
        buf.append(cpSeparator);
        buf.append(targetClassesDir.getAbsolutePath());
    }
    // WEB-INF/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/lib")), "WEB-INF/lib" + File.separator);
    // WEB-INF/crawler/lib
    appendJarFile(cpSeparator, buf, new File(servletContext.getRealPath("/WEB-INF/suggest/lib")), "WEB-INF/suggest" + File.separator + "lib" + File.separator);
    final File targetLibDir = new File(targetDir, "fess" + File.separator + "WEB-INF" + File.separator + "lib");
    if (targetLibDir.isDirectory()) {
        appendJarFile(cpSeparator, buf, targetLibDir, targetLibDir.getAbsolutePath() + File.separator);
    }
    cmdList.add(buf.toString());
    if (useLocaleElasticsearch) {
        final String transportAddresses = System.getProperty(Constants.FESS_ES_TRANSPORT_ADDRESSES);
        if (StringUtil.isNotBlank(transportAddresses)) {
            cmdList.add("-D" + Constants.FESS_ES_TRANSPORT_ADDRESSES + "=" + transportAddresses);
        }
        final String clusterName = System.getProperty(Constants.FESS_ES_CLUSTER_NAME);
        if (StringUtil.isNotBlank(clusterName)) {
            cmdList.add("-D" + Constants.FESS_ES_CLUSTER_NAME + "=" + clusterName);
        }
    }
    final String lastaEnv = System.getProperty("lasta.env");
    if (StringUtil.isNotBlank(lastaEnv)) {
        if (lastaEnv.equals("web")) {
            cmdList.add("-Dlasta.env=suggest");
        } else {
            cmdList.add("-Dlasta.env=" + lastaEnv);
        }
    }
    cmdList.add("-Dfess.suggest.process=true");
    if (logFilePath == null) {
        final String value = System.getProperty("fess.log.path");
        logFilePath = value != null ? value : new File(targetDir, "logs").getAbsolutePath();
    }
    cmdList.add("-Dfess.log.path=" + logFilePath);
    addSystemProperty(cmdList, "fess.log.name", "fess-suggest", "-suggest");
    if (logLevel == null) {
        addSystemProperty(cmdList, "fess.log.level", null, null);
    } else {
        cmdList.add("-Dfess.log.level=" + logLevel);
    }
    stream(fessConfig.getJvmSuggestOptionsAsArray()).of(stream -> stream.filter(StringUtil::isNotBlank).forEach(value -> cmdList.add(value)));
    File ownTmpDir = null;
    final String tmpDir = System.getProperty("java.io.tmpdir");
    if (fessConfig.isUseOwnTmpDir() && StringUtil.isNotBlank(tmpDir)) {
        ownTmpDir = new File(tmpDir, "fessTmpDir_" + sessionId);
        if (ownTmpDir.mkdirs()) {
            cmdList.add("-Djava.io.tmpdir=" + ownTmpDir.getAbsolutePath());
        } else {
            ownTmpDir = null;
        }
    }
    cmdList.add(SuggestCreator.class.getCanonicalName());
    cmdList.add("--sessionId");
    cmdList.add(sessionId);
    File propFile = null;
    try {
        cmdList.add("-p");
        propFile = File.createTempFile("crawler_", ".properties");
        cmdList.add(propFile.getAbsolutePath());
        try (FileOutputStream out = new FileOutputStream(propFile)) {
            final Properties prop = new Properties();
            prop.putAll(ComponentUtil.getSystemProperties());
            prop.store(out, cmdList.toString());
        }
        final File baseDir = new File(servletContext.getRealPath("/WEB-INF")).getParentFile();
        if (logger.isInfoEnabled()) {
            logger.info("SuggestCreator: \nDirectory=" + baseDir + "\nOptions=" + cmdList);
        }
        final JobProcess jobProcess = processHelper.startProcess(sessionId, cmdList, pb -> {
            pb.directory(baseDir);
            pb.redirectErrorStream(true);
        });
        final InputStreamThread it = jobProcess.getInputStreamThread();
        it.start();
        final Process currentProcess = jobProcess.getProcess();
        currentProcess.waitFor();
        it.join(5000);
        final int exitValue = currentProcess.exitValue();
        if (logger.isInfoEnabled()) {
            logger.info("SuggestCreator: Exit Code=" + exitValue + " - SuggestCreator Process Output:\n" + it.getOutput());
        }
        if (exitValue != 0) {
            throw new FessSystemException("Exit Code: " + exitValue + "\nOutput:\n" + it.getOutput());
        }
        ComponentUtil.getPopularWordHelper().clearCache();
    } catch (final FessSystemException e) {
        throw e;
    } catch (final InterruptedException e) {
        logger.warn("SuggestCreator Process interrupted.");
    } catch (final Exception e) {
        throw new FessSystemException("SuggestCreator Process terminated.", e);
    } finally {
        try {
            processHelper.destroyProcess(sessionId);
        } finally {
            if (propFile != null && !propFile.delete()) {
                logger.warn("Failed to delete {}.", propFile.getAbsolutePath());
            }
            deleteTempDir(ownTmpDir);
        }
    }
}