Examples with FessConfig org.codelibs.fess.mylasta.direction.FessConfig used on opensource projects

Example 46 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class IndexingHelper method deleteByVirtualHost.

public long deleteByVirtualHost(final String virtualHost) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final String index = fessConfig.getIndexDocumentUpdateIndex();
    final QueryBuilder queryBuilder = QueryBuilders.termQuery(fessConfig.getIndexFieldVirtualHost(), virtualHost);
    return deleteByQueryBuilder(index, queryBuilder);
}

Example 47 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class IndexingHelper method sendDocuments.

public void sendDocuments(final SearchEngineClient searchEngineClient, final DocList docList) {
    if (docList.isEmpty()) {
        return;
    }
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final long execTime = System.currentTimeMillis();
    if (logger.isDebugEnabled()) {
        logger.debug("Sending {} documents to a server.", docList.size());
    }
    try {
        if (fessConfig.isThumbnailCrawlerEnabled()) {
            final ThumbnailManager thumbnailManager = ComponentUtil.getThumbnailManager();
            docList.stream().forEach(doc -> {
                if (!thumbnailManager.offer(doc)) {
                    if (logger.isDebugEnabled()) {
                        logger.debug("Removing {} from {}", doc.get(fessConfig.getIndexFieldThumbnail()), doc.get(fessConfig.getIndexFieldUrl()));
                    }
                    doc.remove(fessConfig.getIndexFieldThumbnail());
                }
            });
        }
        final CrawlingConfigHelper crawlingConfigHelper = ComponentUtil.getCrawlingConfigHelper();
        synchronized (searchEngineClient) {
            deleteOldDocuments(searchEngineClient, docList);
            searchEngineClient.addAll(fessConfig.getIndexDocumentUpdateIndex(), docList, (doc, builder) -> {
                final String configId = (String) doc.get(fessConfig.getIndexFieldConfigId());
                crawlingConfigHelper.getPipeline(configId).ifPresent(s -> builder.setPipeline(s));
            });
        }
        if (logger.isInfoEnabled()) {
            if (docList.getContentSize() > 0) {
                logger.info("Sent {} docs (Doc:{process {}ms, send {}ms, size {}}, {})", docList.size(), docList.getProcessingTime(), (System.currentTimeMillis() - execTime), MemoryUtil.byteCountToDisplaySize(docList.getContentSize()), MemoryUtil.getMemoryUsageLog());
            } else {
                logger.info("Sent {}  docs (Doc:{send {}ms}, {})", docList.size(), (System.currentTimeMillis() - execTime), MemoryUtil.getMemoryUsageLog());
            }
        }
    } finally {
        docList.clear();
    }
}

Example 48 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class SearchHelper method searchInternal.

protected List<Map<String, Object>> searchInternal(final String query, final SearchRequestParams params, final OptionalThing<FessUserBean> userBean) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
    return ComponentUtil.getSearchEngineClient().search(fessConfig.getIndexDocumentSearchIndex(), searchRequestBuilder -> {
        queryHelper.processSearchPreference(searchRequestBuilder, userBean, query);
        return SearchConditionBuilder.builder(searchRequestBuilder).query(query).offset(params.getStartPosition()).size(params.getPageSize()).facetInfo(params.getFacetInfo()).geoInfo(params.getGeoInfo()).highlightInfo(params.getHighlightInfo()).similarDocHash(params.getSimilarDocHash()).responseFields(queryHelper.getResponseFields()).searchRequestType(params.getType()).trackTotalHits(params.getTrackTotalHits()).build();
    }, (searchRequestBuilder, execTime, searchResponse) -> {
        searchResponse.ifPresent(r -> {
            if (r.getTotalShards() != r.getSuccessfulShards() && fessConfig.isQueryTimeoutLogging()) {
                // partial results
                final StringBuilder buf = new StringBuilder(1000);
                // 
                buf.append("[SEARCH TIMEOUT] {\"exec_time\":").append(execTime).append(",\"request\":").append(// 
                searchRequestBuilder.toString()).append(",\"response\":").append(r.toString()).append('}');
                logger.warn(buf.toString());
            }
        });
        final QueryResponseList queryResponseList = ComponentUtil.getQueryResponseList();
        queryResponseList.init(searchResponse, params.getStartPosition(), params.getPageSize());
        return queryResponseList;
    });
}

Example 49 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class SearchHelper method getDocumentByDocId.

public OptionalEntity<Map<String, Object>> getDocumentByDocId(final String docId, final String[] fields, final OptionalThing<FessUserBean> userBean) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    return ComponentUtil.getSearchEngineClient().getDocument(fessConfig.getIndexDocumentSearchIndex(), builder -> {
        final BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().must(QueryBuilders.termQuery(fessConfig.getIndexFieldDocId(), docId));
        // TODO SearchRequestType?
        final Set<String> roleSet = ComponentUtil.getRoleQueryHelper().build(SearchRequestType.JSON);
        final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
        if (!roleSet.isEmpty()) {
            queryHelper.buildRoleQuery(roleSet, boolQuery);
        }
        builder.setQuery(boolQuery);
        builder.setFetchSource(fields, null);
        queryHelper.processSearchPreference(builder, userBean, docId);
        return true;
    });
}

Example 50 with FessConfig

use of org.codelibs.fess.mylasta.direction.FessConfig in project fess by codelibs.

the class SearchHelper method scrollSearch.

public long scrollSearch(final SearchRequestParams params, final BooleanFunction<Map<String, Object>> cursor, final OptionalThing<FessUserBean> userBean) {
    LaRequestUtil.getOptionalRequest().ifPresent(request -> {
        request.setAttribute(Constants.REQUEST_LANGUAGES, params.getLanguages());
        request.setAttribute(Constants.REQUEST_QUERIES, params.getQuery());
    });
    final int pageSize = params.getPageSize();
    final String query = ComponentUtil.getQueryStringBuilder().params(params).sortField(params.getSort()).build();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    return ComponentUtil.getSearchEngineClient().<Map<String, Object>>scrollSearch(fessConfig.getIndexDocumentSearchIndex(), searchRequestBuilder -> {
        final QueryHelper queryHelper = ComponentUtil.getQueryHelper();
        queryHelper.processSearchPreference(searchRequestBuilder, userBean, query);
        return SearchConditionBuilder.builder(searchRequestBuilder).scroll().query(query).size(pageSize).responseFields(queryHelper.getScrollResponseFields()).searchRequestType(params.getType()).build();
    }, (searchResponse, hit) -> {
        final Map<String, Object> docMap = new HashMap<>();
        final Map<String, Object> source = hit.getSourceAsMap();
        if (source != null) {
            docMap.putAll(source);
        }
        final Map<String, DocumentField> fields = hit.getFields();
        if (fields != null) {
            docMap.putAll(fields.entrySet().stream().collect(Collectors.toMap(Entry::getKey, e -> (Object) e.getValue().getValues())));
        }
        final ViewHelper viewHelper = ComponentUtil.getViewHelper();
        if (viewHelper != null && !docMap.isEmpty()) {
            docMap.put(fessConfig.getResponseFieldContentTitle(), viewHelper.getContentTitle(docMap));
            docMap.put(fessConfig.getResponseFieldContentDescription(), viewHelper.getContentDescription(docMap));
            docMap.put(fessConfig.getResponseFieldUrlLink(), viewHelper.getUrlLink(docMap));
            docMap.put(fessConfig.getResponseFieldSitePath(), viewHelper.getSitePath(docMap));
        }
        if (!docMap.containsKey(Constants.SCORE)) {
            docMap.put(Constants.SCORE, hit.getScore());
        }
        docMap.put(fessConfig.getIndexFieldId(), hit.getId());
        docMap.put(fessConfig.getIndexFieldVersion(), hit.getVersion());
        docMap.put(fessConfig.getIndexFieldSeqNo(), hit.getSeqNo());
        docMap.put(fessConfig.getIndexFieldPrimaryTerm(), hit.getPrimaryTerm());
        return docMap;
    }, cursor);
}