Search in sources :

Example 1 with Terms

use of org.opensearch.search.aggregations.bucket.terms.Terms in project fess by codelibs.

the class CrawlingInfoHelper method getSessionIdList.

public List<Map<String, String>> getSessionIdList(final SearchEngineClient searchEngineClient) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    return searchEngineClient.search(fessConfig.getIndexDocumentSearchIndex(), queryRequestBuilder -> {
        queryRequestBuilder.setQuery(QueryBuilders.matchAllQuery());
        final TermsAggregationBuilder termsBuilder = AggregationBuilders.terms(fessConfig.getIndexFieldSegment()).field(fessConfig.getIndexFieldSegment()).size(maxSessionIdsInList).order(BucketOrder.key(false));
        queryRequestBuilder.addAggregation(termsBuilder);
        queryRequestBuilder.setPreference(Constants.SEARCH_PREFERENCE_LOCAL);
        return true;
    }, (queryRequestBuilder, execTime, searchResponse) -> {
        final List<Map<String, String>> sessionIdList = new ArrayList<>();
        searchResponse.ifPresent(response -> {
            final Terms terms = response.getAggregations().get(fessConfig.getIndexFieldSegment());
            for (final Bucket bucket : terms.getBuckets()) {
                final Map<String, String> map = new HashMap<>(2);
                map.put(fessConfig.getIndexFieldSegment(), bucket.getKey().toString());
                map.put(FACET_COUNT_KEY, Long.toString(bucket.getDocCount()));
                sessionIdList.add(map);
            }
        });
        return sessionIdList;
    });
}
Also used : TermsAggregationBuilder(org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder) Bucket(org.opensearch.search.aggregations.bucket.terms.Terms.Bucket) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) ArrayList(java.util.ArrayList) Terms(org.opensearch.search.aggregations.bucket.terms.Terms) FessConfig(org.codelibs.fess.mylasta.direction.FessConfig) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map)

Example 2 with Terms

use of org.opensearch.search.aggregations.bucket.terms.Terms in project fess by codelibs.

the class SearchLogService method getSearchLogList.

public List<?> getSearchLogList(final SearchLogPager pager) {
    final EsPagingResultBean<?> list;
    if (SearchLogPager.LOG_TYPE_USERINFO.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) userInfoBhv.selectPage(cb -> {
            cb.paging(pager.getPageSize(), pager.getCurrentPageNumber());
            cb.query().addOrderBy_UpdatedAt_Desc();
            createUserInfoCondition(pager, cb);
        });
    } else if (SearchLogPager.LOG_TYPE_CLICK.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) clickLogBhv.selectPage(cb -> {
            cb.paging(pager.getPageSize(), pager.getCurrentPageNumber());
            cb.query().addOrderBy_RequestedAt_Desc();
            createClickLogCondition(pager, cb);
        });
    } else if (SearchLogPager.LOG_TYPE_CLICK_COUNT.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) clickLogBhv.selectPage(cb -> {
            cb.fetchFirst(0);
            createClickLogCondition(pager, cb);
            cb.aggregation().setUrl_Terms(SearchLogPager.LOG_TYPE_CLICK_COUNT, op -> {
                op.size(pager.getPageSize());
                if (fessConfig.getSearchlogAggShardSizeAsInteger() >= 0) {
                    op.shardSize(fessConfig.getSearchlogAggShardSizeAsInteger());
                }
            }, null);
        });
        final Terms agg = list.getAggregations().get(SearchLogPager.LOG_TYPE_CLICK_COUNT);
        final List<? extends Terms.Bucket> buckets = agg.getBuckets();
        updatePagerByAgg(pager, buckets.size());
        return buckets.stream().map(e -> {
            final Map<String, Object> map = new HashMap<>();
            map.put(ID, Base64.getUrlEncoder().encodeToString(e.getKeyAsString().getBytes(StandardCharsets.UTF_8)));
            map.put(KEY, e.getKeyAsString());
            map.put(COUNT, e.getDocCount());
            return map;
        }).collect(Collectors.toList());
    } else if (SearchLogPager.LOG_TYPE_FAVORITE.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) favoriteLogBhv.selectPage(cb -> {
            cb.paging(pager.getPageSize(), pager.getCurrentPageNumber());
            cb.query().addOrderBy_CreatedAt_Desc();
            createFavoriteLogCondition(pager, cb);
        });
    } else if (SearchLogPager.LOG_TYPE_FAVORITE_COUNT.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) favoriteLogBhv.selectPage(cb -> {
            cb.fetchFirst(0);
            createFavoriteLogCondition(pager, cb);
            cb.aggregation().setUrl_Terms(SearchLogPager.LOG_TYPE_FAVORITE_COUNT, op -> {
                op.size(pager.getPageSize());
                if (fessConfig.getSearchlogAggShardSizeAsInteger() >= 0) {
                    op.shardSize(fessConfig.getSearchlogAggShardSizeAsInteger());
                }
            }, null);
        });
        final Terms agg = list.getAggregations().get(SearchLogPager.LOG_TYPE_FAVORITE_COUNT);
        final List<? extends Terms.Bucket> buckets = agg.getBuckets();
        updatePagerByAgg(pager, buckets.size());
        return buckets.stream().map(e -> {
            final Map<String, Object> map = new HashMap<>();
            map.put(ID, Base64.getUrlEncoder().encodeToString(e.getKeyAsString().getBytes(StandardCharsets.UTF_8)));
            map.put(KEY, e.getKeyAsString());
            map.put(COUNT, e.getDocCount());
            return map;
        }).collect(Collectors.toList());
    } else if (SearchLogPager.LOG_TYPE_SEARCH_COUNT_HOUR.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) searchLogBhv.selectPage(cb -> {
            cb.fetchFirst(0);
            createSearchLogCondition(pager, cb);
            cb.aggregation().setRequestedAt_DateHistogram(SearchLogPager.LOG_TYPE_SEARCH_COUNT_HOUR, op -> {
                op.calendarInterval(DateHistogramInterval.HOUR);
                op.minDocCount(0);
                op.order(BucketOrder.key(true));
            }, null);
        });
        final Histogram agg = list.getAggregations().get(SearchLogPager.LOG_TYPE_SEARCH_COUNT_HOUR);
        final List<? extends Histogram.Bucket> buckets = agg.getBuckets();
        updatePagerByAgg(pager, buckets.size());
        return buckets.stream().map(e -> {
            final Map<String, Object> map = new HashMap<>();
            map.put(ID, Base64.getUrlEncoder().encodeToString(e.getKeyAsString().getBytes(StandardCharsets.UTF_8)));
            map.put(KEY, e.getKeyAsString());
            map.put(COUNT, e.getDocCount());
            return map;
        }).collect(Collectors.toList());
    } else if (SearchLogPager.LOG_TYPE_SEARCH_COUNT_DAY.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) searchLogBhv.selectPage(cb -> {
            cb.fetchFirst(0);
            createSearchLogCondition(pager, cb);
            cb.aggregation().setRequestedAt_DateHistogram(SearchLogPager.LOG_TYPE_SEARCH_COUNT_DAY, op -> {
                op.calendarInterval(DateHistogramInterval.DAY);
                op.minDocCount(0);
                op.order(BucketOrder.key(true));
            }, null);
        });
        final Histogram agg = list.getAggregations().get(SearchLogPager.LOG_TYPE_SEARCH_COUNT_DAY);
        final List<? extends Histogram.Bucket> buckets = agg.getBuckets();
        updatePagerByAgg(pager, buckets.size());
        return buckets.stream().map(e -> {
            final Map<String, Object> map = new HashMap<>();
            map.put(ID, Base64.getUrlEncoder().encodeToString(e.getKeyAsString().getBytes(StandardCharsets.UTF_8)));
            map.put(KEY, e.getKeyAsString());
            map.put(COUNT, e.getDocCount());
            return map;
        }).collect(Collectors.toList());
    } else if (SearchLogPager.LOG_TYPE_SEARCH_USER_HOUR.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) searchLogBhv.selectPage(cb -> {
            cb.fetchFirst(0);
            createSearchLogCondition(pager, cb);
            cb.aggregation().setRequestedAt_DateHistogram(SearchLogPager.LOG_TYPE_SEARCH_USER_HOUR, op -> {
                op.calendarInterval(DateHistogramInterval.HOUR);
                op.subAggregation(AggregationBuilders.cardinality(USER_INFO_ID).field(USER_INFO_ID));
                op.minDocCount(0);
                op.order(BucketOrder.key(true));
            }, null);
        });
        final Histogram agg = list.getAggregations().get(SearchLogPager.LOG_TYPE_SEARCH_USER_HOUR);
        final List<? extends Histogram.Bucket> buckets = agg.getBuckets();
        updatePagerByAgg(pager, buckets.size());
        return buckets.stream().map(e -> {
            final Map<String, Object> map = new HashMap<>();
            map.put(ID, Base64.getUrlEncoder().encodeToString(e.getKeyAsString().getBytes(StandardCharsets.UTF_8)));
            map.put(KEY, e.getKeyAsString());
            final Cardinality value = e.getAggregations().get(USER_INFO_ID);
            map.put(COUNT, value.getValue());
            return map;
        }).collect(Collectors.toList());
    } else if (SearchLogPager.LOG_TYPE_SEARCH_USER_DAY.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) searchLogBhv.selectPage(cb -> {
            cb.fetchFirst(0);
            createSearchLogCondition(pager, cb);
            cb.aggregation().setRequestedAt_DateHistogram(SearchLogPager.LOG_TYPE_SEARCH_USER_DAY, op -> {
                op.calendarInterval(DateHistogramInterval.DAY);
                op.subAggregation(AggregationBuilders.cardinality(USER_INFO_ID).field(USER_INFO_ID));
                op.minDocCount(0);
                op.order(BucketOrder.key(true));
            }, null);
        });
        final Histogram agg = list.getAggregations().get(SearchLogPager.LOG_TYPE_SEARCH_USER_DAY);
        final List<? extends Histogram.Bucket> buckets = agg.getBuckets();
        updatePagerByAgg(pager, buckets.size());
        return buckets.stream().map(e -> {
            final Map<String, Object> map = new HashMap<>();
            map.put(ID, Base64.getUrlEncoder().encodeToString(e.getKeyAsString().getBytes(StandardCharsets.UTF_8)));
            map.put(KEY, e.getKeyAsString());
            final Cardinality value = e.getAggregations().get(USER_INFO_ID);
            map.put(COUNT, value.getValue());
            return map;
        }).collect(Collectors.toList());
    } else if (SearchLogPager.LOG_TYPE_SEARCH_REQTIMEAVG_HOUR.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) searchLogBhv.selectPage(cb -> {
            cb.fetchFirst(0);
            createSearchLogCondition(pager, cb);
            cb.aggregation().setRequestedAt_DateHistogram(SearchLogPager.LOG_TYPE_SEARCH_REQTIMEAVG_HOUR, op -> {
                op.calendarInterval(DateHistogramInterval.HOUR);
                op.subAggregation(AggregationBuilders.avg(QUERY_TIME).field(QUERY_TIME));
                op.minDocCount(0);
                op.order(BucketOrder.key(true));
            }, null);
        });
        final Histogram agg = list.getAggregations().get(SearchLogPager.LOG_TYPE_SEARCH_REQTIMEAVG_HOUR);
        final List<? extends Histogram.Bucket> buckets = agg.getBuckets();
        updatePagerByAgg(pager, buckets.size());
        return buckets.stream().map(e -> {
            final Map<String, Object> map = new HashMap<>();
            map.put(ID, Base64.getUrlEncoder().encodeToString(e.getKeyAsString().getBytes(StandardCharsets.UTF_8)));
            map.put(KEY, e.getKeyAsString());
            final Avg value = e.getAggregations().get(QUERY_TIME);
            map.put(COUNT, value.getValueAsString());
            return map;
        }).collect(Collectors.toList());
    } else if (SearchLogPager.LOG_TYPE_SEARCH_REQTIMEAVG_DAY.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) searchLogBhv.selectPage(cb -> {
            cb.fetchFirst(0);
            createSearchLogCondition(pager, cb);
            cb.aggregation().setRequestedAt_DateHistogram(SearchLogPager.LOG_TYPE_SEARCH_REQTIMEAVG_DAY, op -> {
                op.calendarInterval(DateHistogramInterval.DAY);
                op.subAggregation(AggregationBuilders.avg(QUERY_TIME).field(QUERY_TIME));
                op.minDocCount(0);
                op.order(BucketOrder.key(true));
            }, null);
        });
        final Histogram agg = list.getAggregations().get(SearchLogPager.LOG_TYPE_SEARCH_REQTIMEAVG_DAY);
        final List<? extends Histogram.Bucket> buckets = agg.getBuckets();
        updatePagerByAgg(pager, buckets.size());
        return buckets.stream().map(e -> {
            final Map<String, Object> map = new HashMap<>();
            map.put(ID, Base64.getUrlEncoder().encodeToString(e.getKeyAsString().getBytes(StandardCharsets.UTF_8)));
            map.put(KEY, e.getKeyAsString());
            final Avg value = e.getAggregations().get(QUERY_TIME);
            map.put(COUNT, value.getValueAsString());
            return map;
        }).collect(Collectors.toList());
    } else if (SearchLogPager.LOG_TYPE_SEARCH_KEYWORD.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) searchLogBhv.selectPage(cb -> {
            cb.fetchFirst(0);
            createSearchLogCondition(pager, cb);
            cb.aggregation().setSearchWord_Terms(SearchLogPager.LOG_TYPE_SEARCH_KEYWORD, op -> {
                op.size(pager.getPageSize());
                if (fessConfig.getSearchlogAggShardSizeAsInteger() >= 0) {
                    op.shardSize(fessConfig.getSearchlogAggShardSizeAsInteger());
                }
            }, null);
        });
        final Terms agg = list.getAggregations().get(SearchLogPager.LOG_TYPE_SEARCH_KEYWORD);
        final List<? extends Terms.Bucket> buckets = agg.getBuckets();
        updatePagerByAgg(pager, buckets.size());
        return buckets.stream().map(e -> {
            final Map<String, Object> map = new HashMap<>();
            map.put(ID, Base64.getUrlEncoder().encodeToString(e.getKeyAsString().getBytes(StandardCharsets.UTF_8)));
            map.put(KEY, e.getKeyAsString());
            map.put(COUNT, e.getDocCount());
            return map;
        }).collect(Collectors.toList());
    } else if (SearchLogPager.LOG_TYPE_SEARCH_ZEROHIT.equalsIgnoreCase(pager.logType)) {
        list = (EsPagingResultBean<?>) searchLogBhv.selectPage(cb -> {
            cb.fetchFirst(0);
            createSearchLogCondition(pager, cb);
            cb.query().setHitCount_Equal(0L);
            cb.aggregation().setSearchWord_Terms(SearchLogPager.LOG_TYPE_SEARCH_ZEROHIT, op -> {
                op.size(pager.getPageSize());
                if (fessConfig.getSearchlogAggShardSizeAsInteger() >= 0) {
                    op.shardSize(fessConfig.getSearchlogAggShardSizeAsInteger());
                }
            }, null);
        });
        final Terms agg = list.getAggregations().get(SearchLogPager.LOG_TYPE_SEARCH_ZEROHIT);
        final List<? extends Terms.Bucket> buckets = agg.getBuckets();
        updatePagerByAgg(pager, buckets.size());
        return buckets.stream().map(e -> {
            final Map<String, Object> map = new HashMap<>();
            map.put(ID, Base64.getUrlEncoder().encodeToString(e.getKeyAsString().getBytes(StandardCharsets.UTF_8)));
            map.put(KEY, e.getKeyAsString());
            map.put(COUNT, e.getDocCount());
            return map;
        }).collect(Collectors.toList());
    // } else if (SearchLogPager.LOG_TYPE_SEARCH_ZEROCLICK.equalsIgnoreCase(pager.logType)) {
    // list = (EsPagingResultBean<?>) searchLogBhv.selectPage(cb -> {
    // cb.fetchFirst(0);
    // createSearchLogCondition(pager, cb);
    // // TODO 0 clicked
    // });
    } else {
        list = (EsPagingResultBean<?>) searchLogBhv.selectPage(cb -> {
            cb.paging(pager.getPageSize(), pager.getCurrentPageNumber());
            cb.query().addOrderBy_RequestedAt_Desc();
            createSearchLogCondition(pager, cb);
        });
    }
    // update pager
    BeanUtil.copyBeanToBean(list, pager, option -> option.include(Constants.PAGER_CONVERSION_RULE));
    pager.setPageNumberList(list.pageRange(op -> {
        op.rangeSize(fessConfig.getPagingPageRangeSizeAsInteger());
    }).createPageNumberList());
    return list;
}
Also used : BeanUtil(org.codelibs.core.beans.util.BeanUtil) FavoriteLog(org.codelibs.fess.es.log.exentity.FavoriteLog) Constants(org.codelibs.fess.Constants) BucketOrder(org.opensearch.search.aggregations.BucketOrder) FessSystemException(org.codelibs.fess.exception.FessSystemException) LocalDateTime(java.time.LocalDateTime) HashMap(java.util.HashMap) ClickLog(org.codelibs.fess.es.log.exentity.ClickLog) DateHistogramInterval(org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval) SearchLogPager(org.codelibs.fess.app.pager.SearchLogPager) LinkedHashMap(java.util.LinkedHashMap) ClickLogCB(org.codelibs.fess.es.log.cbean.ClickLogCB) FessConfig(org.codelibs.fess.mylasta.direction.FessConfig) Cardinality(org.opensearch.search.aggregations.metrics.Cardinality) FavoriteLogCB(org.codelibs.fess.es.log.cbean.FavoriteLogCB) Map(java.util.Map) EsPagingResultBean(org.codelibs.fess.es.log.allcommon.EsPagingResultBean) FavoriteLogBhv(org.codelibs.fess.es.log.exbhv.FavoriteLogBhv) ZoneOffset(java.time.ZoneOffset) UserInfo(org.codelibs.fess.es.log.exentity.UserInfo) UserInfoBhv(org.codelibs.fess.es.log.exbhv.UserInfoBhv) ClickLogBhv(org.codelibs.fess.es.log.exbhv.ClickLogBhv) SearchLog(org.codelibs.fess.es.log.exentity.SearchLog) OptionalEntity(org.dbflute.optional.OptionalEntity) Resource(javax.annotation.Resource) StringUtil(org.codelibs.core.lang.StringUtil) Terms(org.opensearch.search.aggregations.bucket.terms.Terms) Avg(org.opensearch.search.aggregations.metrics.Avg) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) ZoneId(java.time.ZoneId) AggregationBuilders(org.opensearch.search.aggregations.AggregationBuilders) Base64(java.util.Base64) List(java.util.List) Logger(org.apache.logging.log4j.Logger) UserInfoCB(org.codelibs.fess.es.log.cbean.UserInfoCB) SearchLogBhv(org.codelibs.fess.es.log.exbhv.SearchLogBhv) SearchLogCB(org.codelibs.fess.es.log.cbean.SearchLogCB) SystemHelper(org.codelibs.fess.helper.SystemHelper) DateTimeFormatter(java.time.format.DateTimeFormatter) LogManager(org.apache.logging.log4j.LogManager) Histogram(org.opensearch.search.aggregations.bucket.histogram.Histogram) FessFunctions(org.codelibs.fess.taglib.FessFunctions) Histogram(org.opensearch.search.aggregations.bucket.histogram.Histogram) Cardinality(org.opensearch.search.aggregations.metrics.Cardinality) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Terms(org.opensearch.search.aggregations.bucket.terms.Terms) Avg(org.opensearch.search.aggregations.metrics.Avg) EsPagingResultBean(org.codelibs.fess.es.log.allcommon.EsPagingResultBean) List(java.util.List) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) Map(java.util.Map)

Aggregations

HashMap (java.util.HashMap)2 LinkedHashMap (java.util.LinkedHashMap)2 Map (java.util.Map)2 FessConfig (org.codelibs.fess.mylasta.direction.FessConfig)2 StandardCharsets (java.nio.charset.StandardCharsets)1 LocalDateTime (java.time.LocalDateTime)1 ZoneId (java.time.ZoneId)1 ZoneOffset (java.time.ZoneOffset)1 DateTimeFormatter (java.time.format.DateTimeFormatter)1 ArrayList (java.util.ArrayList)1 Base64 (java.util.Base64)1 List (java.util.List)1 Collectors (java.util.stream.Collectors)1 Resource (javax.annotation.Resource)1 LogManager (org.apache.logging.log4j.LogManager)1 Logger (org.apache.logging.log4j.Logger)1 BeanUtil (org.codelibs.core.beans.util.BeanUtil)1 StringUtil (org.codelibs.core.lang.StringUtil)1 Constants (org.codelibs.fess.Constants)1 SearchLogPager (org.codelibs.fess.app.pager.SearchLogPager)1