Examples with SearchLog org.codelibs.fess.es.log.exentity.SearchLog used on opensource projects

Example 1 with SearchLog

use of org.codelibs.fess.es.log.exentity.SearchLog in project fess by codelibs.

the class SuggestHelper method indexFromSearchLog.

public void indexFromSearchLog(final List<SearchLog> searchLogList) {
    final Map<String, LocalDateTime> duplicateSessionMap = new HashMap<>();
    searchLogList.stream().forEach(searchLog -> {
        if (searchLog.getHitCount() == null || searchLog.getHitCount().longValue() < fessConfig.getSuggestMinHitCountAsInteger().longValue()) {
            return;
        }
        final String sessionId;
        if (searchLog.getUserSessionId() != null) {
            sessionId = searchLog.getUserSessionId();
        } else if (Constants.SEARCH_LOG_ACCESS_TYPE_WEB.equals(searchLog.getAccessType())) {
            sessionId = searchLog.getClientIp();
        } else {
            sessionId = searchLog.getClientIp() + '_' + searchLog.getSearchWord();
        }
        final LocalDateTime requestedAt = searchLog.getRequestedAt();
        if ((sessionId == null) || (duplicateSessionMap.containsKey(sessionId) && duplicateSessionMap.get(sessionId).plusMinutes(searchStoreIntervalMinute).isAfter(requestedAt))) {
            return;
        }
        final StringBuilder sb = new StringBuilder(100);
        final List<String> fields = new ArrayList<>();
        final List<String> tags = new ArrayList<>();
        final List<String> roles = new ArrayList<>();
        for (final Pair<String, String> searchFieldLog : searchLog.getSearchFieldLogList()) {
            final String name = searchFieldLog.getFirst();
            if (contentFieldNameSet.contains(name)) {
                if (sb.length() > 0) {
                    sb.append(TEXT_SEP);
                }
                sb.append(searchFieldLog.getSecond());
                fields.add(name);
            } else if (tagFieldNameSet.contains(name)) {
                tags.add(searchFieldLog.getSecond());
            } else if (roleFieldNameSet.contains(name)) {
                roles.add(searchFieldLog.getSecond());
            }
        }
        final String virtualHost = searchLog.getVirtualHost();
        if (virtualHost != null) {
            tags.add(virtualHost);
        }
        if (sb.length() > 0) {
            final String[] langs = searchLog.getLanguages() == null ? new String[] {} : searchLog.getLanguages().split(",");
            stream(searchLog.getRoles()).of(stream -> stream.forEach(role -> roles.add(role)));
            if (fessConfig.isValidSearchLogPermissions(roles.toArray(new String[roles.size()]))) {
                suggester.indexer().indexFromSearchWord(sb.toString(), fields.toArray(new String[fields.size()]), tags.toArray(new String[tags.size()]), roles.toArray(new String[roles.size()]), 1, langs);
                duplicateSessionMap.put(sessionId, requestedAt);
            }
        }
    });
    refresh();
}

Example 2 with SearchLog

use of org.codelibs.fess.es.log.exentity.SearchLog in project fess by codelibs.

the class SearchLogHelper method addSearchLog.

public void addSearchLog(final SearchRequestParams params, final LocalDateTime requestedTime, final String queryId, final String query, final int pageStart, final int pageSize, final QueryResponseList queryResponseList) {
    final RoleQueryHelper roleQueryHelper = ComponentUtil.getRoleQueryHelper();
    final UserInfoHelper userInfoHelper = ComponentUtil.getUserInfoHelper();
    final SearchLog searchLog = new SearchLog();
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    if (fessConfig.isUserInfo()) {
        final String userCode = userInfoHelper.getUserCode();
        if (userCode != null) {
            searchLog.setUserSessionId(userCode);
            searchLog.setUserInfo(getUserInfo(userCode));
        }
    }
    searchLog.setRoles(roleQueryHelper.build(params.getType()).stream().toArray(n -> new String[n]));
    searchLog.setQueryId(queryId);
    searchLog.setHitCount(queryResponseList.getAllRecordCount());
    searchLog.setHitCountRelation(queryResponseList.getAllRecordCountRelation());
    searchLog.setResponseTime(queryResponseList.getExecTime());
    searchLog.setQueryTime(queryResponseList.getQueryTime());
    searchLog.setSearchWord(StringUtils.abbreviate(query, 1000));
    searchLog.setRequestedAt(requestedTime);
    searchLog.setSearchQuery(StringUtils.abbreviate(queryResponseList.getSearchQuery(), 1000));
    searchLog.setQueryOffset(pageStart);
    searchLog.setQueryPageSize(pageSize);
    ComponentUtil.getRequestManager().findUserBean(FessUserBean.class).ifPresent(user -> {
        searchLog.setUser(user.getUserId());
    });
    final HttpServletRequest request = LaRequestUtil.getRequest();
    searchLog.setClientIp(StringUtils.abbreviate(ComponentUtil.getViewHelper().getClientIp(request), 100));
    searchLog.setReferer(StringUtils.abbreviate(request.getHeader("referer"), 1000));
    searchLog.setUserAgent(StringUtils.abbreviate(request.getHeader("user-agent"), 255));
    final Object accessType = request.getAttribute(Constants.SEARCH_LOG_ACCESS_TYPE);
    if (Constants.SEARCH_LOG_ACCESS_TYPE_JSON.equals(accessType)) {
        searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_JSON);
    } else if (Constants.SEARCH_LOG_ACCESS_TYPE_GSA.equals(accessType)) {
        searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_GSA);
    } else if (Constants.SEARCH_LOG_ACCESS_TYPE_OTHER.equals(accessType)) {
        searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_OTHER);
    } else if (Constants.SEARCH_LOG_ACCESS_TYPE_ADMIN.equals(accessType)) {
        searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_ADMIN);
    } else {
        searchLog.setAccessType(Constants.SEARCH_LOG_ACCESS_TYPE_WEB);
    }
    final Object languages = request.getAttribute(Constants.REQUEST_LANGUAGES);
    if (languages != null) {
        searchLog.setLanguages(StringUtils.join((String[]) languages, ","));
    } else {
        searchLog.setLanguages(StringUtil.EMPTY);
    }
    final String virtualHostKey = ComponentUtil.getVirtualHostHelper().getVirtualHostKey();
    if (StringUtil.isNotBlank(virtualHostKey)) {
        searchLog.setVirtualHost(virtualHostKey);
    } else {
        searchLog.setVirtualHost(StringUtil.EMPTY);
    }
    @SuppressWarnings("unchecked") final Map<String, List<String>> fieldLogMap = (Map<String, List<String>>) request.getAttribute(Constants.FIELD_LOGS);
    if (fieldLogMap != null) {
        final int queryMaxLength = fessConfig.getQueryMaxLengthAsInteger();
        for (final Map.Entry<String, List<String>> logEntry : fieldLogMap.entrySet()) {
            for (final String value : logEntry.getValue()) {
                searchLog.addSearchFieldLogValue(logEntry.getKey(), StringUtils.abbreviate(value, queryMaxLength));
            }
        }
    }
    LaRequestUtil.getOptionalRequest().ifPresent(req -> {
        for (final String s : fessConfig.getSearchlogRequestHeadersAsArray()) {
            final String key = s.replace('-', '_').toLowerCase(Locale.ENGLISH);
            Collections.list(req.getHeaders(s)).stream().forEach(v -> {
                searchLog.addRequestHeaderValue(key, v);
            });
        }
    });
    addDocumentsInResponse(queryResponseList, searchLog);
    searchLogQueue.add(searchLog);
}

Example 3 with SearchLog

use of org.codelibs.fess.es.log.exentity.SearchLog in project fess by codelibs.

the class SearchLogHelper method processUserInfoLog.

protected void processUserInfoLog(final List<SearchLog> searchLogList, final Map<String, UserInfo> userInfoMap) {
    if (!userInfoMap.isEmpty()) {
        final FessConfig fessConfig = ComponentUtil.getFessConfig();
        final List<UserInfo> insertList = new ArrayList<>(userInfoMap.values());
        final List<UserInfo> updateList = new ArrayList<>();
        final UserInfoBhv userInfoBhv = ComponentUtil.getComponent(UserInfoBhv.class);
        userInfoBhv.selectList(cb -> {
            cb.query().setId_InScope(userInfoMap.keySet());
            cb.fetchFirst(userInfoMap.size());
        }).forEach(userInfo -> {
            final String code = userInfo.getId();
            final UserInfo entity = userInfoMap.get(code);
            entity.setId(userInfo.getId());
            entity.setCreatedAt(userInfo.getCreatedAt());
            updateList.add(entity);
            insertList.remove(entity);
        });
        userInfoBhv.batchInsert(insertList);
        userInfoBhv.batchUpdate(updateList);
        searchLogList.stream().forEach(searchLog -> {
            searchLog.getUserInfo().ifPresent(userInfo -> {
                searchLog.setUserInfoId(userInfo.getId());
            });
        });
        if (fessConfig.isLoggingSearchUseLogfile()) {
            insertList.forEach(this::writeSearchLogEvent);
            updateList.forEach(this::writeSearchLogEvent);
        }
    }
}

Example 4 with SearchLog

use of org.codelibs.fess.es.log.exentity.SearchLog in project fess by codelibs.

the class SearchLogHelper method processClickLogQueue.

protected void processClickLogQueue(final Queue<ClickLog> queue) {
    final Map<String, Integer> clickCountMap = new HashMap<>();
    final List<ClickLog> clickLogList = new ArrayList<>();
    for (final ClickLog clickLog : queue) {
        try {
            final SearchLogBhv searchLogBhv = ComponentUtil.getComponent(SearchLogBhv.class);
            searchLogBhv.selectEntity(cb -> {
                cb.query().setQueryId_Equal(clickLog.getQueryId());
            }).ifPresent(entity -> {
                clickLogList.add(clickLog);
                final String docId = clickLog.getDocId();
                Integer countObj = clickCountMap.get(docId);
                if (countObj == null) {
                    countObj = 1;
                } else {
                    countObj = countObj.intValue() + 1;
                }
                clickCountMap.put(docId, countObj);
            }).orElse(() -> {
                logger.warn("Not Found for SearchLog: {}", clickLog);
            });
        } catch (final Exception e) {
            logger.warn("Failed to process: {}", clickLog, e);
        }
    }
    processClickLog(clickLogList);
    updateClickFieldInIndex(clickCountMap);
}

Example 5 with SearchLog

use of org.codelibs.fess.es.log.exentity.SearchLog in project fess by codelibs.

the class SearchLogHelper method processSearchLogQueue.

protected void processSearchLogQueue(final Queue<SearchLog> queue) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final String value = fessConfig.getPurgeByBots();
    String[] botNames;
    if (StringUtil.isBlank(value)) {
        botNames = StringUtil.EMPTY_STRINGS;
    } else {
        botNames = value.split(",");
    }
    final List<SearchLog> searchLogList = new ArrayList<>();
    final Map<String, UserInfo> userInfoMap = new HashMap<>();
    queue.stream().forEach(searchLog -> {
        final String userAgent = searchLog.getUserAgent();
        final boolean isBot = userAgent != null && stream(botNames).get(stream -> stream.anyMatch(botName -> userAgent.indexOf(botName) >= 0));
        if (!isBot) {
            searchLog.getUserInfo().ifPresent(userInfo -> {
                final String code = userInfo.getId();
                final UserInfo oldUserInfo = userInfoMap.get(code);
                if (oldUserInfo != null) {
                    userInfo.setCreatedAt(oldUserInfo.getCreatedAt());
                }
                userInfoMap.put(code, userInfo);
            });
            searchLogList.add(searchLog);
        }
    });
    processUserInfoLog(searchLogList, userInfoMap);
    processSearchLog(searchLogList);
}