Examples with EsDataService org.codelibs.fess.crawler.service.impl.EsDataService used on opensource projects

Example 1 with EsDataService

use of org.codelibs.fess.crawler.service.impl.EsDataService in project fess-crawler by codelibs.

the class EsAccessResult method getAccessResultData.

@Override
public AccessResultData<String> getAccessResultData() {
    if (!initializedData) {
        final EsDataService dataService = SingletonLaContainer.getComponent(EsDataService.class);
        final EsAccessResult accessResult = dataService.getAccessResult(getSessionId(), getUrl());
        if (accessResult != null && accessResult.accessResultData != null) {
            setAccessResultData(accessResult.accessResultData);
        } else {
            setAccessResultData(null);
        }
    }
    return accessResultData;
}

Example 2 with EsDataService

use of org.codelibs.fess.crawler.service.impl.EsDataService in project fess by codelibs.

the class IndexUpdater method getAccessResultList.

private List<EsAccessResult> getAccessResultList(final Consumer<SearchRequestBuilder> cb, final long cleanupTime) {
    if (logger.isDebugEnabled()) {
        logger.debug("Getting documents in IndexUpdater queue.");
    }
    final long execTime = System.currentTimeMillis();
    final List<EsAccessResult> arList = ((EsDataService) dataService).getAccessResultList(cb);
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    if (!arList.isEmpty()) {
        final long commitMarginTime = fessConfig.getIndexerWebfsCommitMarginTimeAsInteger().longValue();
        for (final AccessResult<?> ar : arList.toArray(new AccessResult[arList.size()])) {
            if (ar.getCreateTime().longValue() > execTime - commitMarginTime) {
                arList.remove(ar);
            }
        }
    }
    final long totalHits = ((EsResultList<EsAccessResult>) arList).getTotalHits();
    if (logger.isInfoEnabled()) {
        final StringBuilder buf = new StringBuilder(100);
        buf.append("Processing ");
        if (totalHits > 0) {
            buf.append(arList.size()).append('/').append(totalHits).append(" docs (Doc:{access ");
        } else {
            buf.append("no docs in indexing queue (Doc:{access ");
        }
        buf.append(System.currentTimeMillis() - execTime).append("ms");
        if (cleanupTime >= 0) {
            buf.append(", cleanup ").append(cleanupTime).append("ms");
        }
        buf.append("}, ");
        buf.append(MemoryUtil.getMemoryUsageLog());
        buf.append(')');
        logger.info(buf.toString());
    }
    final long unprocessedDocumentSize = fessConfig.getIndexerUnprocessedDocumentSizeAsInteger().longValue();
    final IntervalControlHelper intervalControlHelper = ComponentUtil.getIntervalControlHelper();
    if (totalHits > unprocessedDocumentSize && intervalControlHelper.isCrawlerRunning()) {
        if (logger.isInfoEnabled()) {
            logger.info("Stopped all crawler threads. You have {} (>{}) unprocessed docs.", totalHits, unprocessedDocumentSize);
        }
        intervalControlHelper.setCrawlerRunning(false);
    }
    return arList;
}

Example 3 with EsDataService

use of org.codelibs.fess.crawler.service.impl.EsDataService in project fess by codelibs.

the class WebFsIndexHelper method deleteCrawlData.

protected void deleteCrawlData(final String sid) {
    final EsUrlFilterService urlFilterService = ComponentUtil.getComponent(EsUrlFilterService.class);
    final EsUrlQueueService urlQueueService = ComponentUtil.getComponent(EsUrlQueueService.class);
    final EsDataService dataService = ComponentUtil.getComponent(EsDataService.class);
    try {
        // clear url filter
        urlFilterService.delete(sid);
    } catch (final Exception e) {
        logger.warn("Failed to delete UrlFilter for {}", sid, e);
    }
    try {
        // clear queue
        urlQueueService.clearCache();
        urlQueueService.delete(sid);
    } catch (final Exception e) {
        logger.warn("Failed to delete UrlQueue for {}", sid, e);
    }
    try {
        // clear
        dataService.delete(sid);
    } catch (final Exception e) {
        logger.warn("Failed to delete AccessResult for {}", sid, e);
    }
}