Examples with BoolQueryBuilder org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.BoolQueryBuilder used on opensource projects

Example 41 with BoolQueryBuilder

use of org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.BoolQueryBuilder in project herd by FINRAOS.

the class IndexSearchDaoImpl method indexSearch.

@Override
public IndexSearchResponse indexSearch(final IndexSearchRequest indexSearchRequest, final Set<String> fields, final Set<String> match, final String bdefActiveIndex, final String tagActiveIndex) {
    boolean negationTermsExist = herdSearchQueryHelper.determineNegationTermsPresent(indexSearchRequest);
    // Build a basic Boolean query upon which add all the necessary clauses as needed
    BoolQueryBuilder indexSearchQueryBuilder = QueryBuilders.boolQuery();
    String searchPhrase = indexSearchRequest.getSearchTerm();
    // Add the negation queries builder within a 'must-not' clause to the parent bool query if negation terms exist
    if (negationTermsExist) {
        // Build negation queries- each term is added to the query with a 'must-not' clause,
        List<String> negationTerms = herdSearchQueryHelper.extractNegationTerms(indexSearchRequest);
        if (CollectionUtils.isNotEmpty(negationTerms)) {
            negationTerms.forEach(term -> {
                indexSearchQueryBuilder.mustNot(buildMultiMatchQuery(term, PHRASE, 100f, FIELD_TYPE_STEMMED, match));
            });
        }
        // Remove the negation terms from the search phrase
        searchPhrase = herdSearchQueryHelper.extractSearchPhrase(indexSearchRequest);
    }
    // Build a Dismax query with three primary components (multi-match queries) with boost values, these values can be configured in the
    // DB which provides a way to dynamically tune search behavior at runtime:
    // 1. Phrase match query on shingles fields.
    // 2. Phrase prefix query on stemmed fields.
    // 3. Best fields query on ngrams fields.
    final MultiMatchQueryBuilder phrasePrefixMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, PHRASE_PREFIX, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_PREFIX_QUERY_BOOST, Float.class), FIELD_TYPE_STEMMED, match);
    final MultiMatchQueryBuilder bestFieldsMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, BEST_FIELDS, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_BEST_FIELDS_QUERY_BOOST, Float.class), FIELD_TYPE_NGRAMS, match);
    final MultiMatchQueryBuilder phraseMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, PHRASE, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class), FIELD_TYPE_SHINGLES, match);
    final MultiMatchQueryBuilder phraseStemmedMultiMatchQueryBuilder = buildMultiMatchQuery(searchPhrase, PHRASE, configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class), FIELD_TYPE_STEMMED, match);
    // Add the multi match queries to a dis max query and add to the parent bool query within a 'must' clause
    indexSearchQueryBuilder.must(disMaxQuery().add(phrasePrefixMultiMatchQueryBuilder).add(bestFieldsMultiMatchQueryBuilder).add(phraseMultiMatchQueryBuilder).add(phraseStemmedMultiMatchQueryBuilder));
    // Add filter clauses if index search filters are specified in the request
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getIndexSearchFilters())) {
        indexSearchQueryBuilder.filter(elasticsearchHelper.addIndexSearchFilterBooleanClause(indexSearchRequest.getIndexSearchFilters(), bdefActiveIndex, tagActiveIndex));
    }
    // Get function score query builder
    FunctionScoreQueryBuilder functionScoreQueryBuilder = getFunctionScoreQueryBuilder(indexSearchQueryBuilder, bdefActiveIndex);
    // The fields in the search indexes to return
    final String[] searchSources = { NAME_SOURCE, NAMESPACE_CODE_SOURCE, TAG_CODE_SOURCE, TAG_TYPE_CODE_SOURCE, DISPLAY_NAME_SOURCE, DESCRIPTION_SOURCE, BDEF_TAGS_SOURCE, BDEF_TAGS_SEARCH_SCORE_MULTIPLIER };
    // Create a new indexSearch source builder
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    // Fetch only the required fields
    searchSourceBuilder.fetchSource(searchSources, null);
    searchSourceBuilder.query(functionScoreQueryBuilder);
    // Create a indexSearch request builder
    SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(new ElasticsearchClientImpl(), SearchAction.INSTANCE);
    searchRequestBuilder.setIndices(bdefActiveIndex, tagActiveIndex);
    searchRequestBuilder.setSource(searchSourceBuilder).setSize(SEARCH_RESULT_SIZE).addSort(SortBuilders.scoreSort());
    // Add highlighting if specified in the request
    if (BooleanUtils.isTrue(indexSearchRequest.isEnableHitHighlighting())) {
        // Fetch configured 'tag' values for highlighting
        String preTag = configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_PRETAGS);
        String postTag = configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_POSTTAGS);
        searchRequestBuilder.highlighter(buildHighlightQuery(preTag, postTag, match));
    }
    // Add facet aggregations if specified in the request
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getFacetFields())) {
        searchRequestBuilder = elasticsearchHelper.addFacetFieldAggregations(new HashSet<>(indexSearchRequest.getFacetFields()), searchRequestBuilder);
    }
    // Log the actual elasticsearch query when debug is enabled
    LOGGER.debug("indexSearchRequest={}", searchRequestBuilder.toString());
    // Retrieve the indexSearch response
    final Search.Builder searchBuilder = new Search.Builder(searchRequestBuilder.toString()).addIndices(Arrays.asList(bdefActiveIndex, tagActiveIndex));
    final SearchResult searchResult = jestClientHelper.searchExecute(searchBuilder.build());
    final List<IndexSearchResult> indexSearchResults = buildIndexSearchResults(fields, tagActiveIndex, bdefActiveIndex, searchResult, indexSearchRequest.isEnableHitHighlighting());
    List<Facet> facets = null;
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getFacetFields())) {
        // Extract facets from the search response
        facets = new ArrayList<>(extractFacets(indexSearchRequest, searchResult, bdefActiveIndex, tagActiveIndex));
    }
    return new IndexSearchResponse(searchResult.getTotal(), indexSearchResults, facets);
}

Example 42 with BoolQueryBuilder

use of org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.BoolQueryBuilder in project herd by FINRAOS.

the class ElasticsearchHelper method applySearchFilterClause.

/**
 * Resolves the search filters into an Elasticsearch {@link BoolQueryBuilder}
 *
 * @param indexSearchFilter the specified search filter
 * @param bdefActiveIndex the name of the bdef active index
 * @param tagActiveIndex the name of the tag active index
 *
 * @return {@link BoolQueryBuilder} the resolved filter query
 */
private BoolQueryBuilder applySearchFilterClause(IndexSearchFilter indexSearchFilter, String bdefActiveIndex, String tagActiveIndex) {
    BoolQueryBuilder indexSearchFilterClauseBuilder = new BoolQueryBuilder();
    for (IndexSearchKey indexSearchKey : indexSearchFilter.getIndexSearchKeys()) {
        if (indexSearchKey.getTagKey() != null) {
            // Add constant-score term queries for tagType-code and tag-code from the tag-key.
            ConstantScoreQueryBuilder searchKeyQueryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.boolQuery().should(QueryBuilders.boolQuery().must(QueryBuilders.termQuery(BDEF_TAGTYPE_CODE_FIELD, indexSearchKey.getTagKey().getTagTypeCode())).must(QueryBuilders.termQuery(BDEF_TAG_CODE_FIELD, indexSearchKey.getTagKey().getTagCode()))).should(QueryBuilders.boolQuery().must(QueryBuilders.termQuery(TAG_TAGTYPE_CODE_FIELD, indexSearchKey.getTagKey().getTagTypeCode())).must(QueryBuilders.termQuery(TAG_TAG_CODE_FIELD, indexSearchKey.getTagKey().getTagCode()))));
            // Individual index search keys are OR-ed
            indexSearchFilterClauseBuilder.should(searchKeyQueryBuilder);
        }
        if (indexSearchKey.getIndexSearchResultTypeKey() != null) {
            String indexSearchResultType = indexSearchKey.getIndexSearchResultTypeKey().getIndexSearchResultType();
            String indexName = indexSearchResultType.equalsIgnoreCase(SearchIndexTypeEntity.SearchIndexTypes.TAG.name()) ? tagActiveIndex : bdefActiveIndex;
            // Add constant-score term queries for tagType-code and tag-code from the tag-key.
            ConstantScoreQueryBuilder searchKeyQueryBuilder = QueryBuilders.constantScoreQuery(QueryBuilders.boolQuery().must(QueryBuilders.termQuery(RESULT_TYPE_FIELD, indexName)));
            // Individual index search keys are OR-ed
            indexSearchFilterClauseBuilder.should(searchKeyQueryBuilder);
        }
    }
    return indexSearchFilterClauseBuilder;
}

Example 43 with BoolQueryBuilder

use of org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.BoolQueryBuilder in project incubator-sdap-mudrod by apache.

the class CrawlerDetection method checkByRate.

private int checkByRate(ESDriver es, String user) {
    int rate = Integer.parseInt(props.getProperty(MudrodConstants.REQUEST_RATE));
    Pattern pattern = Pattern.compile("get (.*?) http/*");
    Matcher matcher;
    BoolQueryBuilder filterSearch = new BoolQueryBuilder();
    filterSearch.must(QueryBuilders.termQuery("IP", user));
    AggregationBuilder aggregation = AggregationBuilders.dateHistogram("by_minute").field("Time").dateHistogramInterval(DateHistogramInterval.MINUTE).order(Order.COUNT_DESC);
    SearchResponse checkRobot = es.getClient().prepareSearch(logIndex).setTypes(httpType, ftpType).setQuery(filterSearch).setSize(0).addAggregation(aggregation).execute().actionGet();
    Histogram agg = checkRobot.getAggregations().get("by_minute");
    List<? extends Histogram.Bucket> botList = agg.getBuckets();
    long maxCount = botList.get(0).getDocCount();
    if (maxCount >= rate) {
        return 0;
    } else {
        DateTime dt1 = null;
        int toLast = 0;
        SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(httpType, ftpType).setScroll(new TimeValue(60000)).setQuery(filterSearch).setSize(100).execute().actionGet();
        while (true) {
            for (SearchHit hit : scrollResp.getHits().getHits()) {
                Map<String, Object> result = hit.getSource();
                String logtype = (String) result.get("LogType");
                if (logtype.equals(MudrodConstants.HTTP_LOG)) {
                    String request = (String) result.get("Request");
                    matcher = pattern.matcher(request.trim().toLowerCase());
                    boolean find = false;
                    while (matcher.find()) {
                        request = matcher.group(1);
                        result.put("RequestUrl", props.getProperty(MudrodConstants.BASE_URL) + request);
                        find = true;
                    }
                    if (!find) {
                        result.put("RequestUrl", request);
                    }
                } else {
                    result.put("RequestUrl", result.get("Request"));
                }
                DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
                DateTime dt2 = fmt.parseDateTime((String) result.get("Time"));
                if (dt1 == null) {
                    toLast = 0;
                } else {
                    toLast = Math.abs(Seconds.secondsBetween(dt1, dt2).getSeconds());
                }
                result.put("ToLast", toLast);
                IndexRequest ir = new IndexRequest(logIndex, cleanupType).source(result);
                es.getBulkProcessor().add(ir);
                dt1 = dt2;
            }
            scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
            if (scrollResp.getHits().getHits().length == 0) {
                break;
            }
        }
    }
    return 1;
}

Example 44 with BoolQueryBuilder

use of org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.BoolQueryBuilder in project incubator-sdap-mudrod by apache.

the class SessionGenerator method combineShortSessions.

public void combineShortSessions(ESDriver es, String user, int timeThres) throws ElasticsearchException, IOException {
    BoolQueryBuilder filterSearch = new BoolQueryBuilder();
    filterSearch.must(QueryBuilders.termQuery("IP", user));
    String[] indexArr = new String[] { logIndex };
    String[] typeArr = new String[] { cleanupType };
    int docCount = es.getDocCount(indexArr, typeArr, filterSearch);
    if (docCount < 3) {
        deleteInvalid(es, user);
        return;
    }
    BoolQueryBuilder filterCheck = new BoolQueryBuilder();
    filterCheck.must(QueryBuilders.termQuery("IP", user)).must(QueryBuilders.termQuery("Referer", "-"));
    SearchResponse checkReferer = es.getClient().prepareSearch(logIndex).setTypes(this.cleanupType).setScroll(new TimeValue(60000)).setQuery(filterCheck).setSize(0).execute().actionGet();
    long numInvalid = checkReferer.getHits().getTotalHits();
    double invalidRate = numInvalid / docCount;
    if (invalidRate >= 0.8) {
        deleteInvalid(es, user);
        return;
    }
    StatsAggregationBuilder statsAgg = AggregationBuilders.stats("Stats").field("Time");
    SearchResponse srSession = es.getClient().prepareSearch(logIndex).setTypes(this.cleanupType).setScroll(new TimeValue(60000)).setQuery(filterSearch).addAggregation(AggregationBuilders.terms("Sessions").field("SessionID").size(docCount).subAggregation(statsAgg)).execute().actionGet();
    Terms sessions = srSession.getAggregations().get("Sessions");
    List<Session> sessionList = new ArrayList<>();
    for (Terms.Bucket session : sessions.getBuckets()) {
        Stats agg = session.getAggregations().get("Stats");
        Session sess = new Session(props, es, agg.getMinAsString(), agg.getMaxAsString(), session.getKey().toString());
        sessionList.add(sess);
    }
    Collections.sort(sessionList);
    DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
    String last = null;
    String lastnewID = null;
    String lastoldID = null;
    String current = null;
    for (Session s : sessionList) {
        current = s.getEndTime();
        if (last != null) {
            if (Seconds.secondsBetween(fmt.parseDateTime(last), fmt.parseDateTime(current)).getSeconds() < timeThres) {
                if (lastnewID == null) {
                    s.setNewID(lastoldID);
                } else {
                    s.setNewID(lastnewID);
                }
                QueryBuilder fs = QueryBuilders.boolQuery().filter(QueryBuilders.termQuery("SessionID", s.getID()));
                SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(this.cleanupType).setScroll(new TimeValue(60000)).setQuery(fs).setSize(100).execute().actionGet();
                while (true) {
                    for (SearchHit hit : scrollResp.getHits().getHits()) {
                        if (lastnewID == null) {
                            update(es, logIndex, this.cleanupType, hit.getId(), "SessionID", lastoldID);
                        } else {
                            update(es, logIndex, this.cleanupType, hit.getId(), "SessionID", lastnewID);
                        }
                    }
                    scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
                    if (scrollResp.getHits().getHits().length == 0) {
                        break;
                    }
                }
            }
        }
        lastoldID = s.getID();
        lastnewID = s.getNewID();
        last = current;
    }
}

Example 45 with BoolQueryBuilder

use of org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.BoolQueryBuilder in project incubator-sdap-mudrod by apache.

the class SessionStatistic method processSession.

public int processSession(ESDriver es, String sessionId) throws IOException, InterruptedException, ExecutionException {
    String inputType = cleanupType;
    String outputType = sessionStats;
    DateTimeFormatter fmt = ISODateTimeFormat.dateTime();
    String min = null;
    String max = null;
    DateTime start = null;
    DateTime end = null;
    int duration = 0;
    float requestRate = 0;
    int sessionCount = 0;
    Pattern pattern = Pattern.compile("get (.*?) http/*");
    StatsAggregationBuilder statsAgg = AggregationBuilders.stats("Stats").field("Time");
    BoolQueryBuilder filterSearch = new BoolQueryBuilder();
    filterSearch.must(QueryBuilders.termQuery("SessionID", sessionId));
    SearchResponse sr = es.getClient().prepareSearch(logIndex).setTypes(inputType).setQuery(filterSearch).addAggregation(statsAgg).execute().actionGet();
    Stats agg = sr.getAggregations().get("Stats");
    min = agg.getMinAsString();
    max = agg.getMaxAsString();
    start = fmt.parseDateTime(min);
    end = fmt.parseDateTime(max);
    duration = Seconds.secondsBetween(start, end).getSeconds();
    int searchDataListRequestCount = 0;
    int searchDataRequestCount = 0;
    int searchDataListRequestByKeywordsCount = 0;
    int ftpRequestCount = 0;
    int keywordsNum = 0;
    String iP = null;
    String keywords = "";
    String views = "";
    String downloads = "";
    SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(inputType).setScroll(new TimeValue(60000)).setQuery(filterSearch).setSize(100).execute().actionGet();
    while (true) {
        for (SearchHit hit : scrollResp.getHits().getHits()) {
            Map<String, Object> result = hit.getSource();
            String request = (String) result.get("Request");
            String logType = (String) result.get("LogType");
            iP = (String) result.get("IP");
            Matcher matcher = pattern.matcher(request.trim().toLowerCase());
            while (matcher.find()) {
                request = matcher.group(1);
            }
            String datasetlist = props.getProperty(MudrodConstants.SEARCH_MARKER);
            String dataset = props.getProperty(MudrodConstants.VIEW_MARKER);
            if (request.contains(datasetlist)) {
                searchDataListRequestCount++;
                RequestUrl requestURL = new RequestUrl();
                String infoStr = requestURL.getSearchInfo(request) + ",";
                String info = es.customAnalyzing(props.getProperty(MudrodConstants.ES_INDEX_NAME), infoStr);
                if (!",".equals(info)) {
                    if ("".equals(keywords)) {
                        keywords = keywords + info;
                    } else {
                        String[] items = info.split(",");
                        String[] keywordList = keywords.split(",");
                        for (String item : items) {
                            if (!Arrays.asList(keywordList).contains(item)) {
                                keywords = keywords + item + ",";
                            }
                        }
                    }
                }
            }
            if (request.startsWith(dataset)) {
                searchDataRequestCount++;
                if (findDataset(request) != null) {
                    String view = findDataset(request);
                    if ("".equals(views))
                        views = view;
                    else if (!views.contains(view))
                        views = views + "," + view;
                }
            }
            if (MudrodConstants.FTP_LOG.equals(logType)) {
                ftpRequestCount++;
                String download = "";
                String requestLowercase = request.toLowerCase();
                if (!requestLowercase.endsWith(".jpg") && !requestLowercase.endsWith(".pdf") && !requestLowercase.endsWith(".txt") && !requestLowercase.endsWith(".gif")) {
                    download = request;
                }
                if ("".equals(downloads)) {
                    downloads = download;
                } else {
                    if (!downloads.contains(download)) {
                        downloads = downloads + "," + download;
                    }
                }
            }
        }
        scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
        // Break condition: No hits are returned
        if (scrollResp.getHits().getHits().length == 0) {
            break;
        }
    }
    if (!"".equals(keywords)) {
        keywordsNum = keywords.split(",").length;
    }
    if (searchDataListRequestCount != 0 && searchDataListRequestCount <= Integer.parseInt(props.getProperty(MudrodConstants.SEARCH_F)) && searchDataRequestCount != 0 && searchDataRequestCount <= Integer.parseInt(props.getProperty(MudrodConstants.VIEW_F)) && ftpRequestCount <= Integer.parseInt(props.getProperty(MudrodConstants.DOWNLOAD_F))) {
        String sessionURL = props.getProperty(MudrodConstants.SESSION_PORT) + props.getProperty(MudrodConstants.SESSION_URL) + "?sessionid=" + sessionId + "&sessionType=" + outputType + "&requestType=" + inputType;
        sessionCount = 1;
        IndexRequest ir = new IndexRequest(logIndex, outputType).source(jsonBuilder().startObject().field("SessionID", sessionId).field("SessionURL", sessionURL).field("Duration", duration).field("Number of Keywords", keywordsNum).field("Time", min).field("End_time", max).field("searchDataListRequest_count", searchDataListRequestCount).field("searchDataListRequest_byKeywords_count", searchDataListRequestByKeywordsCount).field("searchDataRequest_count", searchDataRequestCount).field("keywords", es.customAnalyzing(logIndex, keywords)).field("views", views).field("downloads", downloads).field("request_rate", requestRate).field("Comments", "").field("Validation", 0).field("Produceby", 0).field("Correlation", 0).field("IP", iP).endObject());
        es.getBulkProcessor().add(ir);
    }
    return sessionCount;
}