use of io.searchbox.core.Search in project graylog2-server by Graylog2.
the class MoreSearchAdapterES6 method eventSearch.
@Override
public MoreSearch.Result eventSearch(String queryString, TimeRange timerange, Set<String> affectedIndices, Sorting sorting, int page, int perPage, Set<String> eventStreams, String filterString, Set<String> forbiddenSourceStreams) {
final QueryBuilder query = (queryString.isEmpty() || queryString.equals("*")) ? matchAllQuery() : queryStringQuery(queryString).allowLeadingWildcard(allowLeadingWildcard);
final BoolQueryBuilder filter = boolQuery().filter(query).filter(termsQuery(EventDto.FIELD_STREAMS, eventStreams)).filter(requireNonNull(TimeRangeQueryFactory.create(timerange)));
if (!isNullOrEmpty(filterString)) {
filter.filter(queryStringQuery(filterString));
}
if (!forbiddenSourceStreams.isEmpty()) {
// If an event has any stream in "source_streams" that the calling search user is not allowed to access,
// the event must not be in the search result.
filter.filter(boolQuery().mustNot(termsQuery(EventDto.FIELD_SOURCE_STREAMS, forbiddenSourceStreams)));
}
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filter).from((page - 1) * perPage).size(perPage).sort(sorting.getField(), sortOrderMapper.fromSorting(sorting));
final Search.Builder searchBuilder = new Search.Builder(searchSourceBuilder.toString()).addType(IndexMapping.TYPE_MESSAGE).addIndex(affectedIndices.isEmpty() ? Collections.singleton("") : affectedIndices).allowNoIndices(false).ignoreUnavailable(false);
if (LOG.isDebugEnabled()) {
LOG.debug("Query:\n{}", searchSourceBuilder.toString(new ToXContent.MapParams(Collections.singletonMap("pretty", "true"))));
LOG.debug("Execute search: {}", searchBuilder.build().toString());
}
final io.searchbox.core.SearchResult searchResult = multiSearch.wrap(searchBuilder.build(), () -> "Unable to perform search query");
@SuppressWarnings("unchecked") final List<ResultMessage> hits = searchResult.getHits(Map.class, false).stream().map(hit -> ResultMessage.parseFromSource(hit.id, hit.index, (Map<String, Object>) hit.source, hit.highlight)).collect(Collectors.toList());
return MoreSearch.Result.builder().results(hits).resultsCount(searchResult.getTotal()).duration(multiSearch.tookMsFromSearchResult(searchResult)).usedIndexNames(affectedIndices).executedQuery(searchSourceBuilder.toString()).build();
}
use of io.searchbox.core.Search in project graylog2-server by Graylog2.
the class SearchesAdapterES6 method search.
@Override
public SearchResult search(Set<String> indices, Set<IndexRange> indexRanges, SearchesConfig config) {
final SearchSourceBuilder requestBuilder = searchRequest(config);
if (indexRanges.isEmpty()) {
return SearchResult.empty(config.query(), requestBuilder.toString());
}
final Search.Builder searchBuilder = new Search.Builder(requestBuilder.toString()).addType(IndexMapping.TYPE_MESSAGE).addIndex(indices);
final io.searchbox.core.SearchResult searchResult = multiSearch.wrap(searchBuilder.build(), () -> "Unable to perform search query");
final List<ResultMessage> hits = searchResult.getHits(Map.class, false).stream().map(hit -> ResultMessage.parseFromSource(hit.id, hit.index, (Map<String, Object>) hit.source, hit.highlight)).collect(Collectors.toList());
return new SearchResult(hits, searchResult.getTotal(), indexRanges, config.query(), requestBuilder.toString(), multiSearch.tookMsFromSearchResult(searchResult));
}
use of io.searchbox.core.Search in project graylog2-server by Graylog2.
the class SearchesAdapterES6 method fieldStats.
@Override
public FieldStatsResult fieldStats(String query, String filter, TimeRange range, Set<String> indices, String field, boolean includeCardinality, boolean includeStats, boolean includeCount) {
final SearchSourceBuilder searchSourceBuilder;
if (filter == null) {
searchSourceBuilder = standardSearchRequest(query, range);
} else {
searchSourceBuilder = filteredSearchRequest(query, filter, range);
}
final FilterAggregationBuilder filterBuilder = AggregationBuilders.filter(AGG_FILTER, standardAggregationFilters(range, filter));
if (includeCount) {
searchSourceBuilder.aggregation(AggregationBuilders.count(AGG_VALUE_COUNT).field(field));
}
if (includeStats) {
searchSourceBuilder.aggregation(AggregationBuilders.extendedStats(AGG_EXTENDED_STATS).field(field));
}
if (includeCardinality) {
searchSourceBuilder.aggregation(AggregationBuilders.cardinality(AGG_CARDINALITY).field(field));
}
searchSourceBuilder.aggregation(filterBuilder);
if (indices.isEmpty()) {
return FieldStatsResult.empty(query, searchSourceBuilder.toString());
}
final Search searchRequest = new Search.Builder(searchSourceBuilder.toString()).addType(IndexMapping.TYPE_MESSAGE).addIndex(indices).build();
final io.searchbox.core.SearchResult searchResponse = multiSearch.wrap(searchRequest, () -> "Unable to retrieve fields stats");
final List<ResultMessage> hits = searchResponse.getHits(Map.class, false).stream().map(hit -> ResultMessage.parseFromSource(hit.id, hit.index, (Map<String, Object>) hit.source)).collect(Collectors.toList());
final ExtendedStatsAggregation extendedStatsAggregation = searchResponse.getAggregations().getExtendedStatsAggregation(AGG_EXTENDED_STATS);
final ValueCountAggregation valueCountAggregation = searchResponse.getAggregations().getValueCountAggregation(AGG_VALUE_COUNT);
final CardinalityAggregation cardinalityAggregation = searchResponse.getAggregations().getCardinalityAggregation(AGG_CARDINALITY);
return createFieldStatsResult(valueCountAggregation, extendedStatsAggregation, cardinalityAggregation, hits, query, searchSourceBuilder.toString(), multiSearch.tookMsFromSearchResult(searchResponse));
}
use of io.searchbox.core.Search in project graylog2-server by Graylog2.
the class ElasticsearchBackend method doRun.
@Override
public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext queryContext) {
if (query.searchTypes().isEmpty()) {
return QueryResult.builder().query(query).searchTypes(Collections.emptyMap()).errors(new HashSet<>(queryContext.errors())).build();
}
LOG.debug("Running query {} for job {}", query.id(), job.getId());
final HashMap<String, SearchType.Result> resultsMap = Maps.newHashMap();
final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange());
final Map<String, SearchSourceBuilder> searchTypeQueries = queryContext.searchTypeQueries();
final List<String> searchTypeIds = new ArrayList<>(searchTypeQueries.keySet());
final List<Search> searches = searchTypeIds.stream().map(searchTypeId -> {
final Set<String> affectedIndicesForSearchType = query.searchTypes().stream().filter(s -> s.id().equalsIgnoreCase(searchTypeId)).findFirst().flatMap(searchType -> {
if (searchType.effectiveStreams().isEmpty() && !query.globalOverride().flatMap(GlobalOverride::timerange).isPresent() && !searchType.timerange().isPresent()) {
return Optional.empty();
}
final Set<String> usedStreamIds = searchType.effectiveStreams().isEmpty() ? query.usedStreamIds() : searchType.effectiveStreams();
return Optional.of(indexLookup.indexNamesForStreamsInTimeRange(usedStreamIds, query.effectiveTimeRange(searchType)));
}).orElse(affectedIndices);
return new Search.Builder(searchTypeQueries.get(searchTypeId).toString()).addType(IndexMapping.TYPE_MESSAGE).addIndex(affectedIndicesForSearchType.isEmpty() ? Collections.singleton("") : affectedIndicesForSearchType).allowNoIndices(false).ignoreUnavailable(false).build();
}).collect(Collectors.toList());
final MultiSearch.Builder multiSearchBuilder = new MultiSearch.Builder(searches);
final MultiSearchResult result = JestUtils.execute(jestClient, multiSearchBuilder.build(), () -> "Unable to perform search query: ");
for (SearchType searchType : query.searchTypes()) {
final String searchTypeId = searchType.id();
final Provider<ESSearchTypeHandler<? extends SearchType>> handlerProvider = elasticsearchSearchTypeHandlers.get(searchType.type());
if (handlerProvider == null) {
LOG.error("Unknown search type '{}', cannot convert query result.", searchType.type());
// no need to add another error here, as the query generation code will have added the error about the missing handler already
continue;
}
if (isSearchTypeWithError(queryContext, searchTypeId)) {
LOG.error("Failed search type '{}', cannot convert query result, skipping.", searchType.type());
// no need to add another error here, as the query generation code will have added the error about the missing handler already
continue;
}
// we create a new instance because some search type handlers might need to track information between generating the query and
// processing its result, such as aggregations, which depend on the name and type
final ESSearchTypeHandler<? extends SearchType> handler = handlerProvider.get();
final int searchTypeIndex = searchTypeIds.indexOf(searchTypeId);
final MultiSearchResult.MultiSearchResponse multiSearchResponse = result.getResponses().get(searchTypeIndex);
if (multiSearchResponse.isError) {
ElasticsearchException e = JestUtils.specificException(() -> "Search type returned error: ", multiSearchResponse.error);
queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
} else if (checkForFailedShards(multiSearchResponse.searchResult).isPresent()) {
ElasticsearchException e = checkForFailedShards(multiSearchResponse.searchResult).get();
queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
} else {
final SearchType.Result searchTypeResult = handler.extractResult(job, query, searchType, multiSearchResponse.searchResult, queryContext);
if (searchTypeResult != null) {
resultsMap.put(searchTypeId, searchTypeResult);
}
}
}
LOG.debug("Query {} ran for job {}", query.id(), job.getId());
return QueryResult.builder().query(query).searchTypes(resultsMap).errors(new HashSet<>(queryContext.errors())).build();
}
use of io.searchbox.core.Search in project graylog2-server by Graylog2.
the class CountsAdapterES6 method totalCount.
@Override
public long totalCount(List<String> indices) {
final String query = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(0).toString();
final Search request = new Search.Builder(query).addIndex(indices).build();
final MultiSearch multiSearch = new MultiSearch.Builder(request).build();
final MultiSearchResult searchResult = JestUtils.execute(jestClient, multiSearch, () -> "Fetching message count failed for indices " + indices);
final List<MultiSearchResult.MultiSearchResponse> responses = searchResult.getResponses();
long total = 0L;
for (MultiSearchResult.MultiSearchResponse response : responses) {
if (response.isError) {
throw JestUtils.specificException(() -> "Fetching message count failed for indices " + indices, response.error);
}
total += response.searchResult.getTotal();
}
return total;
}
Aggregations