Examples with SearchRequestBuilder org.opensearch.action.search.SearchRequestBuilder used on opensource projects

Example 6 with SearchRequestBuilder

use of org.opensearch.action.search.SearchRequestBuilder in project fess by codelibs.

the class EsAbstractBehavior method delegateQueryDelete.

@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForDelete).setSize(sizeForDelete);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        esCb.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    int count = 0;
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                bulkRequest.add(client.prepareDelete().setIndex(asEsIndex()).setId(hit.getId()));
            }
            count += hits.length;
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
            if (bulkResponse.hasFailures()) {
                throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
            }
            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
        }
    } finally {
        deleteScrollContext(scrollId);
    }
    return count;
}

Example 7 with SearchRequestBuilder

use of org.opensearch.action.search.SearchRequestBuilder in project fess by codelibs.

the class IndexUpdater method run.

@Override
public void run() {
    if (dataService == null) {
        throw new FessSystemException("DataService is null.");
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Starting indexUpdater.");
    }
    executeTime = 0;
    documentSize = 0;
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final long updateInterval = fessConfig.getIndexerWebfsUpdateIntervalAsInteger().longValue();
    final int maxEmptyListCount = fessConfig.getIndexerWebfsMaxEmptyListCountAsInteger();
    final IntervalControlHelper intervalControlHelper = ComponentUtil.getIntervalControlHelper();
    try {
        final Consumer<SearchRequestBuilder> cb = builder -> {
            final QueryBuilder queryBuilder = QueryBuilders.boolQuery().filter(QueryBuilders.termsQuery(EsAccessResult.SESSION_ID, sessionIdList)).filter(QueryBuilders.termQuery(EsAccessResult.STATUS, org.codelibs.fess.crawler.Constants.OK_STATUS));
            builder.setQuery(queryBuilder);
            builder.setFrom(0);
            final int maxDocumentCacheSize = fessConfig.getIndexerWebfsMaxDocumentCacheSizeAsInteger();
            builder.setSize(maxDocumentCacheSize <= 0 ? 1 : maxDocumentCacheSize);
            builder.addSort(EsAccessResult.CREATE_TIME, SortOrder.ASC);
        };
        final DocList docList = new DocList();
        final List<EsAccessResult> accessResultList = new ArrayList<>();
        long updateTime = System.currentTimeMillis();
        int errorCount = 0;
        int emptyListCount = 0;
        long cleanupTime = -1;
        while (!finishCrawling || !accessResultList.isEmpty()) {
            try {
                final int sessionIdListSize = finishedSessionIdList.size();
                intervalControlHelper.setCrawlerRunning(true);
                updateTime = System.currentTimeMillis() - updateTime;
                final long interval = updateInterval - updateTime;
                if (interval > 0) {
                    // sleep
                    // 10 sec (default)
                    ThreadUtil.sleep(interval);
                }
                systemHelper.calibrateCpuLoad();
                docList.clear();
                accessResultList.clear();
                intervalControlHelper.delayByRules();
                if (logger.isDebugEnabled()) {
                    logger.debug("Processing documents in IndexUpdater queue.");
                }
                updateTime = System.currentTimeMillis();
                List<EsAccessResult> arList = getAccessResultList(cb, cleanupTime);
                if (arList.isEmpty()) {
                    emptyListCount++;
                } else {
                    // reset
                    emptyListCount = 0;
                }
                long hitCount = ((EsResultList<EsAccessResult>) arList).getTotalHits();
                while (hitCount > 0) {
                    if (arList.isEmpty()) {
                        ThreadUtil.sleep(fessConfig.getIndexerWebfsCommitMarginTimeAsInteger().longValue());
                        cleanupTime = -1;
                    } else {
                        processAccessResults(docList, accessResultList, arList);
                        cleanupTime = cleanupAccessResults(accessResultList);
                    }
                    arList = getAccessResultList(cb, cleanupTime);
                    hitCount = ((EsResultList<EsAccessResult>) arList).getTotalHits();
                }
                if (!docList.isEmpty()) {
                    indexingHelper.sendDocuments(searchEngineClient, docList);
                }
                synchronized (finishedSessionIdList) {
                    if (sessionIdListSize != 0 && sessionIdListSize == finishedSessionIdList.size()) {
                        cleanupFinishedSessionData();
                    }
                }
                executeTime += System.currentTimeMillis() - updateTime;
                if (logger.isDebugEnabled()) {
                    logger.debug("Processed documents in IndexUpdater queue.");
                }
                // reset count
                errorCount = 0;
            } catch (final Exception e) {
                if (errorCount > maxErrorCount) {
                    throw e;
                }
                errorCount++;
                logger.warn("Failed to access data. Retry to access it {} times.", errorCount, e);
            } finally {
                if (systemHelper.isForceStop()) {
                    finishCrawling = true;
                    if (logger.isDebugEnabled()) {
                        logger.debug("Stopped indexUpdater.");
                    }
                }
            }
            if (emptyListCount >= maxEmptyListCount) {
                if (logger.isInfoEnabled()) {
                    logger.info("Terminating indexUpdater. emptyListCount is over {}.", maxEmptyListCount);
                }
                // terminate crawling
                finishCrawling = true;
                forceStop();
                if (fessConfig.getIndexerThreadDumpEnabledAsBoolean()) {
                    ThreadDumpUtil.printThreadDump();
                }
                org.codelibs.fess.exec.Crawler.addError("QueueTimeout");
            }
            if (!ComponentUtil.available()) {
                logger.info("IndexUpdater is terminated.");
                forceStop();
                break;
            }
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Finished indexUpdater.");
        }
    } catch (final ContainerNotAvailableException e) {
        if (logger.isDebugEnabled()) {
            logger.error("IndexUpdater is terminated.", e);
        } else if (logger.isInfoEnabled()) {
            logger.info("IndexUpdater is terminated.");
        }
        forceStop();
    } catch (final Throwable t) {
        if (ComponentUtil.available()) {
            logger.error("IndexUpdater is terminated.", t);
        } else if (logger.isDebugEnabled()) {
            logger.error("IndexUpdater is terminated.", t);
            org.codelibs.fess.exec.Crawler.addError(t.getClass().getSimpleName());
        } else if (logger.isInfoEnabled()) {
            logger.info("IndexUpdater is terminated.");
            org.codelibs.fess.exec.Crawler.addError(t.getClass().getSimpleName());
        }
        forceStop();
    } finally {
        intervalControlHelper.setCrawlerRunning(true);
    }
    if (logger.isInfoEnabled()) {
        logger.info("[EXEC TIME] index update time: {}ms", executeTime);
    }
}

Example 8 with SearchRequestBuilder

use of org.opensearch.action.search.SearchRequestBuilder in project fess by codelibs.

the class EsAbstractBehavior method delegateSelectList.

@Override
protected <RESULT extends ENTITY> List<RESULT> delegateSelectList(final ConditionBean cb, final Class<? extends RESULT> entityType) {
    // #pending check response
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex());
    final int from;
    final int size;
    if (cb.isFetchScopeEffective()) {
        from = cb.getPageStartIndex();
        size = cb.getFetchSize();
    } else {
        from = 0;
        size = 10;
    }
    builder.setFrom(from);
    builder.setSize(size);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        builder.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    final SearchResponse response = esCb.build(builder).execute().actionGet(searchTimeout);
    final EsPagingResultBean<RESULT> list = new EsPagingResultBean<>(builder);
    final SearchHits searchHits = getSearchHits(response);
    searchHits.forEach(hit -> {
        final Map<String, Object> source = hit.getSourceAsMap();
        final RESULT entity = createEntity(source, entityType);
        final DocMeta docMeta = ((EsAbstractEntity) entity).asDocMeta();
        docMeta.id(hit.getId());
        docMeta.version(hit.getVersion());
        docMeta.seqNo(hit.getSeqNo());
        docMeta.primaryTerm(hit.getPrimaryTerm());
        list.add(entity);
    });
    list.setPageSize(size);
    list.setAllRecordCount((int) searchHits.getTotalHits().value);
    list.setCurrentPageNumber(cb.getFetchPageNumber());
    list.setTook(response.getTook().getMillis());
    list.setTotalShards(response.getTotalShards());
    list.setSuccessfulShards(response.getSuccessfulShards());
    list.setFailedShards(response.getFailedShards());
    list.setTotalHits(searchHits.getTotalHits());
    list.setAggregation(response.getAggregations());
    return list;
}

Example 9 with SearchRequestBuilder

use of org.opensearch.action.search.SearchRequestBuilder in project fess by codelibs.

the class EsAbstractBehavior method delegateBulkRequest.

protected void delegateBulkRequest(final ConditionBean cb, Function<SearchHits, Boolean> handler) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForCursor).setSize(sizeForCursor);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        builder.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }
            if (!handler.apply(searchHits)) {
                break;
            }
            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
            scrollId = response.getScrollId();
        }
    } finally {
        deleteScrollContext(scrollId);
    }
}

Example 10 with SearchRequestBuilder

use of org.opensearch.action.search.SearchRequestBuilder in project fess by codelibs.

the class EsAbstractBehavior method delegateSelectList.

@Override
protected <RESULT extends ENTITY> List<RESULT> delegateSelectList(final ConditionBean cb, final Class<? extends RESULT> entityType) {
    // #pending check response
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex());
    final int from;
    final int size;
    if (cb.isFetchScopeEffective()) {
        from = cb.getPageStartIndex();
        size = cb.getFetchSize();
    } else {
        from = 0;
        size = 10;
    }
    builder.setFrom(from);
    builder.setSize(size);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        builder.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    final SearchResponse response = esCb.build(builder).execute().actionGet(searchTimeout);
    final EsPagingResultBean<RESULT> list = new EsPagingResultBean<>(builder);
    final SearchHits searchHits = getSearchHits(response);
    searchHits.forEach(hit -> {
        final Map<String, Object> source = hit.getSourceAsMap();
        final RESULT entity = createEntity(source, entityType);
        final DocMeta docMeta = ((EsAbstractEntity) entity).asDocMeta();
        docMeta.id(hit.getId());
        docMeta.version(hit.getVersion());
        docMeta.seqNo(hit.getSeqNo());
        docMeta.primaryTerm(hit.getPrimaryTerm());
        list.add(entity);
    });
    list.setPageSize(size);
    list.setAllRecordCount((int) searchHits.getTotalHits().value);
    list.setCurrentPageNumber(cb.getFetchPageNumber());
    list.setTook(response.getTook().getMillis());
    list.setTotalShards(response.getTotalShards());
    list.setSuccessfulShards(response.getSuccessfulShards());
    list.setFailedShards(response.getFailedShards());
    list.setTotalHits(searchHits.getTotalHits());
    list.setAggregation(response.getAggregations());
    return list;
}