Examples with EsAccessException org.codelibs.fess.crawler.exception.EsAccessException used on opensource projects

Example 1 with EsAccessException

use of org.codelibs.fess.crawler.exception.EsAccessException in project fess-crawler by codelibs.

the class AbstractCrawlerService method doInsertAll.

protected <T> BulkResponse doInsertAll(final List<T> list, final OpType opType) {
    try {
        return getClient().get(c -> {
            final BulkRequestBuilder bulkRequest = c.prepareBulk();
            for (final T target : list) {
                final String id = getId(getSessionId(target), getUrl(target));
                final XContentBuilder source = getXContentBuilder(target);
                bulkRequest.add(c.prepareIndex(index, type, id).setSource(source).setOpType(opType));
                setId(target, id);
            }
            return bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE).execute();
        });
    } catch (final Exception e) {
        throw new EsAccessException("Failed to insert " + list, e);
    }
}

Example 2 with EsAccessException

use of org.codelibs.fess.crawler.exception.EsAccessException in project fess-crawler by codelibs.

the class AbstractCrawlerService method insert.

protected IndexResponse insert(final Object target, final OpType opType) {
    final String id = getId(getSessionId(target), getUrl(target));
    final XContentBuilder source = getXContentBuilder(target);
    try {
        final IndexResponse response = getClient().get(c -> c.prepareIndex(index, type, id).setSource(source).setOpType(opType).setRefreshPolicy(RefreshPolicy.IMMEDIATE).execute());
        setId(target, id);
        return response;
    } catch (final Exception e) {
        throw new EsAccessException("Failed to insert " + id, e);
    }
}

Example 3 with EsAccessException

use of org.codelibs.fess.crawler.exception.EsAccessException in project fess-crawler by codelibs.

the class EsUrlQueueService method updateSessionId.

@Override
public void updateSessionId(final String oldSessionId, final String newSessionId) {
    SearchResponse response = null;
    while (true) {
        if (response == null) {
            response = getClient().get(c -> c.prepareSearch(index).setTypes(type).setScroll(new TimeValue(scrollTimeout)).setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.termQuery(SESSION_ID, oldSessionId))).setSize(scrollSize).execute());
        } else {
            final String scrollId = response.getScrollId();
            response = getClient().get(c -> c.prepareSearchScroll(scrollId).setScroll(new TimeValue(scrollTimeout)).execute());
        }
        final SearchHits searchHits = response.getHits();
        if (searchHits.getHits().length == 0) {
            break;
        }
        final BulkResponse bulkResponse = getClient().get(c -> {
            final BulkRequestBuilder builder = c.prepareBulk();
            for (final SearchHit searchHit : searchHits) {
                final UpdateRequestBuilder updateRequest = c.prepareUpdate(index, type, searchHit.getId()).setDoc(SESSION_ID, newSessionId);
                builder.add(updateRequest);
            }
            return builder.execute();
        });
        if (bulkResponse.hasFailures()) {
            throw new EsAccessException(bulkResponse.buildFailureMessage());
        }
    }
}

Example 4 with EsAccessException

use of org.codelibs.fess.crawler.exception.EsAccessException in project fess-crawler by codelibs.

the class EsUrlQueueService method poll.

@Override
public EsUrlQueue poll(final String sessionId) {
    final QueueHolder queueHolder = getQueueHolder(sessionId);
    final Queue<EsUrlQueue> waitingQueue = queueHolder.waitingQueue;
    final Queue<EsUrlQueue> crawlingQueue = queueHolder.crawlingQueue;
    EsUrlQueue urlQueue = waitingQueue.poll();
    if (urlQueue != null) {
        if (crawlingQueue.size() > maxCrawlingQueueSize) {
            crawlingQueue.poll();
        }
        crawlingQueue.add(urlQueue);
        return urlQueue;
    }
    synchronized (queueHolder) {
        urlQueue = waitingQueue.poll();
        if (urlQueue == null) {
            final List<EsUrlQueue> urlQueueList = getList(EsUrlQueue.class, sessionId, null, 0, pollingFetchSize, SortBuilders.fieldSort(CREATE_TIME).order(SortOrder.ASC));
            if (urlQueueList.isEmpty()) {
                return null;
            }
            if (logger.isDebugEnabled()) {
                logger.debug("Queued URL: {}", urlQueueList);
            }
            waitingQueue.addAll(urlQueueList);
            if (!urlQueueList.isEmpty()) {
                try {
                    // delete from es
                    final BulkResponse response = getClient().get(c -> {
                        final BulkRequestBuilder bulkBuilder = c.prepareBulk();
                        for (final EsUrlQueue uq : urlQueueList) {
                            bulkBuilder.add(c.prepareDelete(index, type, uq.getId()));
                        }
                        return bulkBuilder.setRefreshPolicy(RefreshPolicy.IMMEDIATE).execute();
                    });
                    if (response.hasFailures()) {
                        logger.warn(response.buildFailureMessage());
                    }
                } catch (final Exception e) {
                    throw new EsAccessException("Failed to delete " + urlQueueList, e);
                }
            }
            urlQueue = waitingQueue.poll();
            if (urlQueue == null) {
                return null;
            }
        }
    }
    if (crawlingQueue.size() > maxCrawlingQueueSize) {
        crawlingQueue.poll();
    }
    crawlingQueue.add(urlQueue);
    return urlQueue;
}

Example 5 with EsAccessException

use of org.codelibs.fess.crawler.exception.EsAccessException in project fess-crawler by codelibs.

the class AbstractCrawlerService method delete.

public void delete(final Consumer<SearchRequestBuilder> callback) {
    SearchResponse response = null;
    while (true) {
        if (response == null) {
            response = getClient().get(c -> {
                final SearchRequestBuilder builder = c.prepareSearch(index).setTypes(type).setScroll(new TimeValue(scrollTimeout)).setSize(scrollSize);
                callback.accept(builder);
                return builder.execute();
            });
        } else {
            final String scrollId = response.getScrollId();
            response = getClient().get(c -> c.prepareSearchScroll(scrollId).setScroll(new TimeValue(scrollTimeout)).execute());
        }
        final SearchHits searchHits = response.getHits();
        if (searchHits.getHits().length == 0) {
            break;
        }
        final BulkResponse bulkResponse = getClient().get(c -> {
            final BulkRequestBuilder bulkBuilder = c.prepareBulk();
            for (final SearchHit searchHit : searchHits) {
                bulkBuilder.add(c.prepareDelete(index, type, searchHit.getId()));
            }
            return bulkBuilder.execute();
        });
        if (bulkResponse.hasFailures()) {
            throw new EsAccessException(bulkResponse.buildFailureMessage());
        }
    }
    refresh();
}