Examples with EsAccessResultData org.codelibs.fess.crawler.entity.EsAccessResultData used on opensource projects

Example 1 with EsAccessResultData

use of org.codelibs.fess.crawler.entity.EsAccessResultData in project fess-crawler by codelibs.

the class AbstractCrawlerService method get.

protected <T> T get(final Class<T> clazz, final String sessionId, final String url) {
    final String id = getId(sessionId, url);
    final GetResponse response = getClient().get(c -> c.prepareGet(index, type, id).execute());
    if (response.isExists()) {
        final Map<String, Object> source = response.getSource();
        final T bean = BeanUtil.copyMapToNewBean(source, clazz, option -> {
            option.converter(new EsTimestampConverter(), timestampFields).excludeWhitespace();
            option.exclude(EsAccessResult.ACCESS_RESULT_DATA);
        });
        @SuppressWarnings("unchecked") final Map<String, Object> data = (Map<String, Object>) source.get(EsAccessResult.ACCESS_RESULT_DATA);
        if (data != null) {
            ((EsAccessResult) bean).setAccessResultData(new EsAccessResultData(data));
        }
        setId(bean, id);
        return bean;
    }
    return null;
}

Example 2 with EsAccessResultData

use of org.codelibs.fess.crawler.entity.EsAccessResultData in project fess-crawler by codelibs.

the class AbstractCrawlerService method getList.

protected <T> List<T> getList(final Class<T> clazz, final Consumer<SearchRequestBuilder> callback) {
    final SearchResponse response = getClient().get(c -> {
        final SearchRequestBuilder builder = c.prepareSearch(index).setTypes(type);
        callback.accept(builder);
        return builder.execute();
    });
    final EsResultList<T> targetList = new EsResultList<>();
    final SearchHits hits = response.getHits();
    targetList.setTotalHits(hits.getTotalHits());
    targetList.setTookInMillis(response.getTook().getMillis());
    if (hits.getTotalHits() != 0) {
        try {
            for (final SearchHit searchHit : hits.getHits()) {
                final Map<String, Object> source = searchHit.getSourceAsMap();
                final T target = BeanUtil.copyMapToNewBean(source, clazz, option -> {
                    option.converter(new EsTimestampConverter(), timestampFields).excludeWhitespace();
                    option.exclude(EsAccessResult.ACCESS_RESULT_DATA);
                });
                @SuppressWarnings("unchecked") final Map<String, Object> data = (Map<String, Object>) source.get(EsAccessResult.ACCESS_RESULT_DATA);
                if (data != null) {
                    ((EsAccessResult) target).setAccessResultData(new EsAccessResultData(data));
                }
                setId(target, searchHit.getId());
                targetList.add(target);
            }
        } catch (final Exception e) {
            throw new EsAccessException("response: " + response, e);
        }
    }
    return targetList;
}

Example 3 with EsAccessResultData

use of org.codelibs.fess.crawler.entity.EsAccessResultData in project fess-crawler by codelibs.

the class EsDataService method iterate.

@Override
public void iterate(final String sessionId, final AccessResultCallback<EsAccessResult> callback) {
    SearchResponse response = null;
    while (true) {
        if (response == null) {
            response = getClient().get(c -> c.prepareSearch(index).setTypes(type).setScroll(new TimeValue(scrollTimeout)).setQuery(QueryBuilders.boolQuery().filter(QueryBuilders.termQuery(SESSION_ID, sessionId))).setSize(scrollSize).execute());
        } else {
            final String scrollId = response.getScrollId();
            response = getClient().get(c -> c.prepareSearchScroll(scrollId).setScroll(new TimeValue(scrollTimeout)).execute());
        }
        final SearchHits searchHits = response.getHits();
        if (searchHits.getHits().length == 0) {
            break;
        }
        for (final SearchHit searchHit : searchHits) {
            final Map<String, Object> source = searchHit.getSourceAsMap();
            final EsAccessResult accessResult = BeanUtil.copyMapToNewBean(source, EsAccessResult.class, option -> {
                option.converter(new EsTimestampConverter(), timestampFields).excludeWhitespace();
                option.exclude(EsAccessResult.ACCESS_RESULT_DATA);
            });
            @SuppressWarnings("unchecked") final Map<String, Object> data = (Map<String, Object>) source.get(EsAccessResult.ACCESS_RESULT_DATA);
            if (data != null) {
                accessResult.setAccessResultData(new EsAccessResultData(data));
            }
            callback.iterate(accessResult);
        }
    }
}