Examples with IndexingHelper org.codelibs.fess.helper.IndexingHelper used on opensource projects

Example 11 with IndexingHelper

use of org.codelibs.fess.helper.IndexingHelper in project fess by codelibs.

the class IndexUpdateCallbackImpl method store.

/* (non-Javadoc)
     * @see org.codelibs.fess.ds.callback.IndexUpdateCallback#store(java.util.Map)
     */
@Override
public void store(final Map<String, String> paramMap, final Map<String, Object> dataMap) {
    final SystemHelper systemHelper = ComponentUtil.getSystemHelper();
    systemHelper.calibrateCpuLoad();
    final long startTime = System.currentTimeMillis();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final SearchEngineClient searchEngineClient = ComponentUtil.getSearchEngineClient();
    if (logger.isDebugEnabled()) {
        logger.debug("Adding {}", dataMap);
    }
    // required check
    final Object urlObj = dataMap.get(fessConfig.getIndexFieldUrl());
    if (urlObj == null) {
        throw new DataStoreException("url is null. dataMap=" + dataMap);
    }
    final IndexingHelper indexingHelper = ComponentUtil.getIndexingHelper();
    final CrawlingInfoHelper crawlingInfoHelper = ComponentUtil.getCrawlingInfoHelper();
    dataMap.put(fessConfig.getIndexFieldId(), crawlingInfoHelper.generateId(dataMap));
    final String url = dataMap.get(fessConfig.getIndexFieldUrl()).toString();
    if (fessConfig.getIndexerClickCountEnabledAsBoolean()) {
        addClickCountField(dataMap, url, fessConfig.getIndexFieldClickCount());
    }
    if (fessConfig.getIndexerFavoriteCountEnabledAsBoolean()) {
        addFavoriteCountField(dataMap, url, fessConfig.getIndexFieldFavoriteCount());
    }
    final Set<String> matchedLabelSet = ComponentUtil.getLabelTypeHelper().getMatchedLabelValueSet(url);
    if (!matchedLabelSet.isEmpty()) {
        final Set<String> newLabelSet = new HashSet<>();
        final String[] oldLabels = DocumentUtil.getValue(dataMap, fessConfig.getIndexFieldLabel(), String[].class);
        StreamUtil.stream(oldLabels).of(stream -> stream.forEach(newLabelSet::add));
        matchedLabelSet.stream().forEach(newLabelSet::add);
        dataMap.put(fessConfig.getIndexFieldLabel(), newLabelSet.toArray(new String[newLabelSet.size()]));
    }
    if (!dataMap.containsKey(fessConfig.getIndexFieldDocId())) {
        dataMap.put(fessConfig.getIndexFieldDocId(), systemHelper.generateDocId(dataMap));
    }
    ComponentUtil.getLanguageHelper().updateDocument(dataMap);
    synchronized (docList) {
        docList.add(ingest(paramMap, dataMap));
        final long contentSize = indexingHelper.calculateDocumentSize(dataMap);
        docList.addContentSize(contentSize);
        final long processingTime = System.currentTimeMillis() - startTime;
        docList.addProcessingTime(processingTime);
        if (logger.isDebugEnabled()) {
            logger.debug("Added the document({}, {}ms). The number of a document cache is {}.", MemoryUtil.byteCountToDisplaySize(contentSize), processingTime, docList.size());
        }
        if (docList.getContentSize() >= maxDocumentRequestSize || docList.size() >= maxDocumentCacheSize) {
            indexingHelper.sendDocuments(searchEngineClient, docList);
        }
        executeTime += processingTime;
    }
    documentSize.getAndIncrement();
    if (logger.isDebugEnabled()) {
        logger.debug("The number of an added document is {}.", documentSize.get());
    }
}