Examples with SearchHit org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit used on opensource projects

Example 81 with SearchHit

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit in project topcom-cloud by 545314690.

the class ElasticSearchService method search.

/**
 * 功能描述：查询
 * @param index 索引名
 * @param type 类型
 * @param constructor 查询构造
 */
public Page search(String index, String type, ESQueryBuilderConstructor constructor) {
    List<Map<String, Object>> result = new ArrayList<>();
    SearchRequestBuilder searchRequestBuilder = client.prepareSearch(index).setTypes(type);
    // 排序
    if (StringUtils.isNotEmpty(constructor.getAsc()))
        searchRequestBuilder.addSort(constructor.getAsc(), SortOrder.ASC);
    if (StringUtils.isNotEmpty(constructor.getDesc()))
        searchRequestBuilder.addSort(constructor.getDesc(), SortOrder.DESC);
    // 设置查询体
    searchRequestBuilder.setQuery(constructor.listBuilders());
    // 返回条目数
    int size = constructor.getSize();
    if (size < 0) {
        size = 0;
    }
    if (size > MAX) {
        size = MAX;
    }
    // 返回条目数
    searchRequestBuilder.setSize(size);
    searchRequestBuilder.setFrom(constructor.getFrom() < 0 ? 0 : constructor.getFrom());
    SearchResponse searchResponse = searchRequestBuilder.execute().actionGet();
    SearchHits hits = searchResponse.getHits();
    SearchHit[] searchHists = hits.getHits();
    for (SearchHit sh : searchHists) {
        Map<String, Object> source = sh.getSource();
        source.put("_id", sh.getId());
        result.add(sh.getSource());
    }
    return new Page(result, hits.getTotalHits());
}

Example 82 with SearchHit

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit in project topcom-cloud by 545314690.

the class EtlController method weiboEtl.

/**
 * 微博id->weiboId
 * md5（id）->微博id
 * @param limit
 * @return
 */
@ApiOperation("weiboEtl")
@RequestMapping(value = "/weiboEtl", method = RequestMethod.GET, produces = "application/json")
@ResponseBody
public Object weiboEtl(@RequestParam Integer limit) {
    long count = 0;
    SearchResponse searchResponse = scanWeibo("", limit);
    while (searchResponse.getHits().getHits().length > 0) {
        SearchHit[] hits = searchResponse.getHits().hits();
        List<JSONObject> commentsList = new ArrayList<>();
        for (int i = 0; i < hits.length; i++) {
            JSONObject jsonObject = JSONObject.fromObject(hits[i].sourceAsMap());
            jsonObject.put("weiboId", jsonObject.get("id"));
            jsonObject.put("id", MD5Utils.md5(jsonObject.get("id").toString()));
            commentsList.add(jsonObject);
        }
        updateJson("yuqing", "article", commentsList);
        count = count + searchResponse.getHits().getHits().length;
        logger.error(searchResponse.getHits().getTotalHits());
        logger.error("处理微博条数：" + count + "\n 完成度" + (double) count * 100 / searchResponse.getHits().getTotalHits() + "%");
        searchResponse = scanWeibo(searchResponse.getScrollId(), limit);
    }
    return "done";
}

Example 83 with SearchHit

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit in project topcom-cloud by 545314690.

the class EtlController method bbsEtl.

/**
 * 微博id->weiboId
 * md5（id）->微博id
 * @param limit
 * @return
 */
@ApiOperation("bbsEtl")
@RequestMapping(value = "/bbsEtl", method = RequestMethod.GET, produces = "application/json")
@ResponseBody
public Object bbsEtl(@RequestParam Integer limit) {
    long count = 0;
    long count1 = 0;
    SearchResponse searchResponse = scanBbs("", limit);
    while (searchResponse.getHits().getHits().length > 0) {
        SearchHit[] hits = searchResponse.getHits().hits();
        List<JSONObject> commentsList = new ArrayList<>();
        for (int i = 0; i < hits.length; i++) {
            JSONObject jsonObject = JSONObject.fromObject(hits[i].sourceAsMap());
            String type = jsonObject.get("type").toString();
            if (type.equals("Bbs")) {
                Object object = jsonObject.get("floorNum");
                if (object != null && Integer.valueOf(object.toString()) > 0) {
                    commentsList.add(jsonObject);
                }
            }
        }
        // String path = "D:\\data\\bbs\\"+"bbs"+(count1/50000)+".txt";
        String path = "/home/yuqing/data/esFile/" + "bbs" + (count1 / 50000) + ".txt";
        save2File(commentsList, path);
        dleteEsDoc("yuqing", "article", commentsList);
        count1 = count1 + commentsList.size();
        count = count + searchResponse.getHits().getHits().length;
        logger.error("删除BBS条数：" + count1);
        logger.error("处理BBS条数：" + count + "\n 完成度" + (double) count * 100 / searchResponse.getHits().getTotalHits() + "%");
        searchResponse = scanBbs(searchResponse.getScrollId(), limit);
    }
    return "done";
}

Example 84 with SearchHit

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit in project topcom-cloud by 545314690.

the class EtlController method commentsEtl.

@ApiOperation("commentsEtl")
@RequestMapping(value = "/commentsEtl", method = RequestMethod.GET, produces = "application/json")
@ResponseBody
public Object commentsEtl(@RequestParam Integer limit) {
    long count = 0;
    long count1 = 0;
    SearchResponse searchResponse = scanComments("", limit);
    while (searchResponse.getHits().getHits().length > 0) {
        SearchHit[] hits = searchResponse.getHits().hits();
        List<JSONObject> commentsList = new ArrayList<>();
        for (int i = 0; i < hits.length; i++) {
            JSONObject jsonObject = JSONObject.fromObject(hits[i].sourceAsMap());
            commentsList.add(jsonObject);
        }
        // String path = "D:\\data\\comments\\"+"comments"+(count1/50000)+".txt";
        String path = "/home/yuqing/data/esFile/comments/" + "comments" + (count1 / 50000) + ".txt";
        save2File(commentsList, path);
        dleteCommentsDoc("yuqing", "comment", commentsList);
        count1 = count1 + commentsList.size();
        count = count + searchResponse.getHits().getHits().length;
        logger.error("删除comments条数：" + count1);
        logger.error("处理comments条数：" + count + "\n 完成度" + (double) count * 100 / searchResponse.getHits().getTotalHits() + "%");
        searchResponse = scanComments(searchResponse.getScrollId(), limit);
    }
    return "done";
}

Example 85 with SearchHit

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit in project herd by FINRAOS.

the class IndexSearchDaoTest method testIndexSearch.

private void testIndexSearch(Set<String> fields, Set<String> match, List<IndexSearchFilter> searchFilters, List<String> facetList, boolean isHitHighlightingEnabled, boolean testExceptions, boolean setInvalidSearchResultIndexName, boolean isColumnFields) throws IOException {
    // Build the mocks
    SearchRequestBuilder searchRequestBuilder = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithSource = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithSize = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithSorting = mock(SearchRequestBuilder.class);
    SearchRequestBuilder searchRequestBuilderWithHighlighting = mock(SearchRequestBuilder.class);
    SearchResponse searchResponse = mock(SearchResponse.class);
    SearchHits searchHits = mock(SearchHits.class);
    SearchHit searchHit1 = mock(SearchHit.class);
    SearchHit searchHit2 = mock(SearchHit.class);
    SearchShardTarget searchShardTarget1 = mock(SearchShardTarget.class);
    SearchShardTarget searchShardTarget2 = mock(SearchShardTarget.class);
    SearchHit[] searchHitArray = new SearchHit[2];
    searchHitArray[0] = searchHit1;
    searchHitArray[1] = searchHit2;
    HighlightField highlightField = mock(HighlightField.class);
    when(highlightField.getName()).thenReturn("displayName");
    Text[] value = { new Text("match <hlt>fragment</hlt class=\"highlight\">"), new Text("<hlt class=\"highlight\">match</hlt>") };
    when(highlightField.getFragments()).thenReturn(value);
    @SuppressWarnings("unchecked") ListenableActionFuture<SearchResponse> listenableActionFuture = mock(ListenableActionFuture.class);
    final String highlightFieldsConfigValue = "{\"highlightFields\":[{\"fieldName\":\"displayName\",\"fragmentSize\":100,\"matchedFields\":[\"displayName\",\"displayName.stemmed\",\"displayName.ngrams\"],\"numOfFragments\":5}]}";
    final String highlightFieldsColumnMatchConfigValue = "{\"highlightFields\":[{\"fieldName\":\"columnName\",\"fragmentSize\":100,\"matchedFields\":[\"columnName\",\"columnName.stemmed\",\"columnName.ngrams\"],\"numOfFragments\":5}]}";
    // Mock the call to external methods
    when(configurationHelper.getProperty(ConfigurationValue.TAG_SHORT_DESCRIPTION_LENGTH, Integer.class)).thenReturn(300);
    when(configurationHelper.getProperty(ConfigurationValue.BUSINESS_OBJECT_DEFINITION_SHORT_DESCRIPTION_LENGTH, Integer.class)).thenReturn(300);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_NGRAMS)).thenReturn("{\"displayName\":\"1.0\"}");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_STEMMED)).thenReturn("{\"displayName\":\"1.0\"}");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_SHINGLES)).thenReturn("{\"displayName\":\"1.0\"}");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_PRETAGS)).thenReturn("<hlt class=\"highlight\">");
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_POSTTAGS)).thenReturn("</hlt>");
    if (match != null && match.contains(MATCH_COLUMN)) {
        when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_COLUMN_MATCH_HIGHLIGHT_FIELDS)).thenReturn(highlightFieldsColumnMatchConfigValue);
    } else {
        when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_HIGHLIGHT_FIELDS)).thenReturn(highlightFieldsConfigValue);
    }
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_BEST_FIELDS_QUERY_BOOST, Float.class)).thenReturn(1f);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_PREFIX_QUERY_BOOST, Float.class)).thenReturn(1f);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class)).thenReturn(1f);
    when(configurationHelper.getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_SLOP, Integer.class)).thenReturn(5);
    Map<String, String> fieldsBoostMap = new HashMap<>();
    fieldsBoostMap.put("displayName", "1.0");
    if (isColumnFields) {
        fieldsBoostMap.put(COLUMNS_NAME_FIELD, "1.0");
        fieldsBoostMap.put(SCHEMA_COLUMNS_NAME_FIELD, "1.0");
    }
    if (testExceptions) {
        when(jsonHelper.unmarshallJsonToObject(Map.class, "{\"displayName\":\"1.0\"}")).thenThrow(new IOException());
    } else {
        when(jsonHelper.unmarshallJsonToObject(Map.class, "{\"displayName\":\"1.0\"}")).thenReturn(fieldsBoostMap);
    }
    IndexSearchHighlightField indexSearchHighlightField = new IndexSearchHighlightField("displayName", 100, Arrays.asList("displayName", "displayName.stemmed", "displayName.ngrams"), 5);
    IndexSearchHighlightFields highlightFields = new IndexSearchHighlightFields(Collections.singletonList(indexSearchHighlightField));
    IndexSearchHighlightField indexSearchHighlightFieldColumnOnly = new IndexSearchHighlightField("columnName", 100, Arrays.asList("columnName", "columnName.stemmed", "columnName.ngrams"), 5);
    IndexSearchHighlightFields highlightFieldsColumnOnly = new IndexSearchHighlightFields(Collections.singletonList(indexSearchHighlightFieldColumnOnly));
    if (testExceptions) {
        when(jsonHelper.unmarshallJsonToObject(IndexSearchHighlightFields.class, highlightFieldsConfigValue)).thenThrow(new IOException());
    } else {
        if (match != null && match.contains(MATCH_COLUMN)) {
            when(jsonHelper.unmarshallJsonToObject(IndexSearchHighlightFields.class, highlightFieldsColumnMatchConfigValue)).thenReturn(highlightFieldsColumnOnly);
        } else {
            when(jsonHelper.unmarshallJsonToObject(IndexSearchHighlightFields.class, highlightFieldsConfigValue)).thenReturn(highlightFields);
        }
    }
    when(searchRequestBuilder.setSource(any(SearchSourceBuilder.class))).thenReturn(searchRequestBuilderWithSource);
    when(searchRequestBuilderWithSource.setSize(SEARCH_RESULT_SIZE)).thenReturn(searchRequestBuilderWithSize);
    when(searchRequestBuilderWithSize.addSort(any(SortBuilder.class))).thenReturn(searchRequestBuilderWithSorting);
    when(searchRequestBuilderWithSorting.highlighter(any(HighlightBuilder.class))).thenReturn(searchRequestBuilderWithHighlighting);
    when(searchRequestBuilder.execute()).thenReturn(listenableActionFuture);
    when(listenableActionFuture.actionGet()).thenReturn(searchResponse);
    when(searchResponse.getHits()).thenReturn(searchHits);
    when(searchHits.hits()).thenReturn(searchHitArray);
    Map<String, Object> sourceMap1 = new HashMap<>();
    Map<String, Object> tagTypeMap = new HashMap<>();
    tagTypeMap.put(CODE, TAG_TYPE_CODE);
    sourceMap1.put(TAG_TYPE, tagTypeMap);
    when(searchHit1.sourceAsMap()).thenReturn(sourceMap1);
    Map<String, Object> sourceMap2 = new HashMap<>();
    Map<String, Object> businessObjectDefinitionMap = new HashMap<>();
    businessObjectDefinitionMap.put(CODE, NAMESPACE_CODE);
    sourceMap2.put(NAMESPACE, businessObjectDefinitionMap);
    when(searchHit2.sourceAsMap()).thenReturn(sourceMap2);
    when(searchHit1.getShard()).thenReturn(searchShardTarget1);
    when(searchHit2.getShard()).thenReturn(searchShardTarget2);
    when(searchShardTarget1.getIndex()).thenReturn(TAG_SEARCH_INDEX_NAME);
    when(searchShardTarget2.getIndex()).thenReturn(BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME);
    when(searchHits.getTotalHits()).thenReturn(200L);
    Map<String, HighlightField> highlightFieldMap = new HashMap<>();
    highlightFieldMap.put("displayName", highlightField);
    when(searchHit1.getHighlightFields()).thenReturn(highlightFieldMap);
    when(searchHit2.getHighlightFields()).thenReturn(highlightFieldMap);
    // Create index search request
    final IndexSearchRequest indexSearchRequest = new IndexSearchRequest(SEARCH_TERM, searchFilters, facetList, isHitHighlightingEnabled);
    List<TagTypeIndexSearchResponseDto> tagTypeIndexSearchResponseDtos = Collections.singletonList(new TagTypeIndexSearchResponseDto("code", Collections.singletonList(new TagIndexSearchResponseDto("tag1", 1, null)), null));
    List<ResultTypeIndexSearchResponseDto> resultTypeIndexSearchResponseDto = Collections.singletonList(new ResultTypeIndexSearchResponseDto("type", 1, null));
    when(elasticsearchHelper.getNestedTagTagIndexSearchResponseDto(searchResponse)).thenReturn(tagTypeIndexSearchResponseDtos);
    when(elasticsearchHelper.getResultTypeIndexSearchResponseDto(searchResponse)).thenReturn(resultTypeIndexSearchResponseDto);
    when(elasticsearchHelper.getFacetsResponse(any(ElasticsearchResponseDto.class), eq(BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME), eq(TAG_SEARCH_INDEX_NAME))).thenCallRealMethod();
    when(elasticsearchHelper.addIndexSearchFilterBooleanClause(any(), any(), any())).thenCallRealMethod();
    when(elasticsearchHelper.addFacetFieldAggregations(any(), any(SearchRequestBuilder.class))).thenReturn(searchRequestBuilder);
    when(herdSearchQueryHelper.determineNegationTermsPresent(any(IndexSearchRequest.class))).thenCallRealMethod();
    when(herdSearchQueryHelper.extractNegationTerms(any(IndexSearchRequest.class))).thenCallRealMethod();
    when(herdSearchQueryHelper.extractSearchPhrase(any(IndexSearchRequest.class))).thenCallRealMethod();
    SearchResult searchResult = mock(SearchResult.class);
    when(jestClientHelper.searchExecute(any())).thenReturn(searchResult);
    List<SearchResult.Hit<Map, Void>> searchHitList = new ArrayList<>();
    Gson gson = new Gson();
    Map<String, Object> map = new HashMap<>();
    map.put(DISPLAY_NAME_SOURCE, "Display Name");
    Map<String, Object> codeMap = new HashMap<>();
    codeMap.put(CODE, "Code");
    map.put(TAG_TYPE, codeMap);
    map.put(NAMESPACE, codeMap);
    JsonElement element = gson.toJsonTree(map);
    List<String> highlightList = new ArrayList<>();
    highlightList.add("Highlight 1");
    Map<String, List<String>> highlightMap = new HashMap<>();
    highlightMap.put("field", highlightList);
    SearchResult.Hit<Map, Void> hit1 = new SearchResult(gson).new Hit(HashMap.class, element, HashMap.class, null, highlightMap, null, setInvalidSearchResultIndexName ? INVALID_VALUE : TAG_SEARCH_INDEX_NAME, TAG_SEARCH_INDEX_NAME, "type", 1.0);
    SearchResult.Hit<Map, Void> hit2 = new SearchResult(gson).new Hit(HashMap.class, element, HashMap.class, null, highlightMap, null, BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, "type", 2.0);
    searchHitList.add(hit1);
    searchHitList.add(hit2);
    when(searchResult.getHits(Map.class)).thenReturn(searchHitList);
    when(searchResult.getTotal()).thenReturn(200L);
    MetricAggregation metricAggregation = mock(MetricAggregation.class);
    TermsAggregation termsAggregation = mock(TermsAggregation.class);
    when(searchResult.getAggregations()).thenReturn(metricAggregation);
    when(metricAggregation.getTermsAggregation(TAG_TYPE_FACET_AGGS)).thenReturn(termsAggregation);
    List<TermsAggregation.Entry> buckets = new ArrayList<>();
    TermsAggregation.Entry entry1 = mock(TermsAggregation.Entry.class);
    TermsAggregation.Entry entry2 = mock(TermsAggregation.Entry.class);
    buckets.add(entry1);
    buckets.add(entry2);
    when(termsAggregation.getBuckets()).thenReturn(buckets);
    // Call the method under test
    IndexSearchResponse indexSearchResponse = indexSearchDao.indexSearch(indexSearchRequest, fields, match, BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, TAG_SEARCH_INDEX_NAME);
    List<IndexSearchResult> indexSearchResults = indexSearchResponse.getIndexSearchResults();
    assertThat("Index search results list is null.", indexSearchResults, not(nullValue()));
    assertThat(indexSearchResponse.getTotalIndexSearchResults(), is(200L));
    // Verify external method calls.
    verify(herdSearchQueryHelper).determineNegationTermsPresent(indexSearchRequest);
    verify(herdSearchQueryHelper).extractNegationTerms(indexSearchRequest);
    verify(herdSearchQueryHelper).extractSearchPhrase(indexSearchRequest);
    verify(configurationHelper).getProperty(ConfigurationValue.TAG_SHORT_DESCRIPTION_LENGTH, Integer.class);
    verify(configurationHelper).getProperty(ConfigurationValue.BUSINESS_OBJECT_DEFINITION_SHORT_DESCRIPTION_LENGTH, Integer.class);
    verify(configurationHelper, times(3)).getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_STEMMED);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_NGRAMS);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_SEARCHABLE_FIELDS_SHINGLES);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_PREFIX_QUERY_BOOST, Float.class);
    verify(configurationHelper).getProperty(ConfigurationValue.ELASTICSEARCH_BEST_FIELDS_QUERY_BOOST, Float.class);
    verify(configurationHelper, times(2)).getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_BOOST, Float.class);
    verify(configurationHelper, times(5)).getProperty(ConfigurationValue.ELASTICSEARCH_PHRASE_QUERY_SLOP, Integer.class);
    verify(jsonHelper, times(5)).unmarshallJsonToObject(Map.class, "{\"displayName\":\"1.0\"}");
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getIndexSearchFilters())) {
        verify(elasticsearchHelper).addIndexSearchFilterBooleanClause(indexSearchRequest.getIndexSearchFilters(), BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME, TAG_SEARCH_INDEX_NAME);
    }
    if (indexSearchRequest.isEnableHitHighlighting() != null) {
        verifyHitHighlightingInteractions(searchRequestBuilder, indexSearchRequest.isEnableHitHighlighting(), match);
    }
    if (CollectionUtils.isNotEmpty(indexSearchRequest.getFacetFields())) {
        verify(elasticsearchHelper).addFacetFieldAggregations(any(), any(SearchRequestBuilder.class));
        if (indexSearchRequest.getFacetFields().contains(ElasticsearchHelper.TAG_FACET)) {
            verify(elasticsearchHelper).getNestedTagTagIndexSearchResponseDto(searchResult);
            verify(elasticsearchHelper).getTagTagIndexSearchResponseDto(searchResult);
        }
        if (indexSearchRequest.getFacetFields().contains(ElasticsearchHelper.RESULT_TYPE_FACET)) {
            verify(elasticsearchHelper).getResultTypeIndexSearchResponseDto(searchResult);
        }
        verify(elasticsearchHelper).getFacetsResponse(any(ElasticsearchResponseDto.class), eq(BUSINESS_OBJECT_DEFINITION_SEARCH_INDEX_NAME), eq(TAG_SEARCH_INDEX_NAME));
    }
    verify(jestClientHelper).searchExecute(any());
    verify(searchResult).getTotal();
    verify(searchResult).getHits(Map.class);
    verifyNoMoreInteractions(searchRequestBuilder, searchRequestBuilderWithSource, searchRequestBuilderWithSize, searchRequestBuilderWithSorting, searchRequestBuilderWithHighlighting, searchResponse, searchHits, searchHit1, searchHit2, searchShardTarget1, searchShardTarget2, highlightField, listenableActionFuture, searchResult, metricAggregation, termsAggregation, entry1, entry2);
    verifyNoMoreInteractionsHelper();
}