Examples with Indices org.graylog2.indexer.indices.Indices used on opensource projects

Example 71 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class ClusterAdapterES7 method clusterHealth.

private Optional<ClusterHealthResponse> clusterHealth(Collection<String> indices) {
    final String[] indicesAry = indices.toArray(new String[0]);
    if (!indices.isEmpty() && !indicesExist(indicesAry)) {
        return Optional.empty();
    }
    final ClusterHealthRequest request = new ClusterHealthRequest(indicesAry).timeout(TimeValue.timeValueSeconds(Ints.saturatedCast(requestTimeout.toSeconds()))).indicesOptions(IndicesOptions.lenientExpand());
    try {
        return Optional.of(client.execute((c, requestOptions) -> c.cluster().health(request, requestOptions)));
    } catch (ElasticsearchException e) {
        if (LOG.isDebugEnabled()) {
            LOG.error("{} ({})", e.getMessage(), Optional.ofNullable(e.getCause()).map(Throwable::getMessage).orElse("n/a"), e);
        } else {
            LOG.error("{} ({})", e.getMessage(), Optional.ofNullable(e.getCause()).map(Throwable::getMessage).orElse("n/a"));
        }
        return Optional.empty();
    }
}

Example 72 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class IndicesAdapterES7 method indexRangeStatsOfIndex.

@Override
public IndexRangeStats indexRangeStatsOfIndex(String index) {
    final FilterAggregationBuilder builder = AggregationBuilders.filter("agg", QueryBuilders.existsQuery(Message.FIELD_TIMESTAMP)).subAggregation(AggregationBuilders.min("ts_min").field(Message.FIELD_TIMESTAMP)).subAggregation(AggregationBuilders.max("ts_max").field(Message.FIELD_TIMESTAMP)).subAggregation(AggregationBuilders.terms("streams").size(Integer.MAX_VALUE).field(Message.FIELD_STREAMS));
    final SearchSourceBuilder query = SearchSourceBuilder.searchSource().aggregation(builder).size(0);
    final SearchRequest request = new SearchRequest().source(query).indices(index).searchType(SearchType.DFS_QUERY_THEN_FETCH).indicesOptions(IndicesOptions.lenientExpandOpen());
    final SearchResponse result = client.execute((c, requestOptions) -> c.search(request, requestOptions), "Couldn't build index range of index " + index);
    if (result.getTotalShards() == 0 || result.getAggregations() == null) {
        throw new IndexNotFoundException("Couldn't build index range of index " + index + " because it doesn't exist.");
    }
    final Filter f = result.getAggregations().get("agg");
    if (f == null) {
        throw new IndexNotFoundException("Couldn't build index range of index " + index + " because it doesn't exist.");
    } else if (f.getDocCount() == 0L) {
        LOG.debug("No documents with attribute \"timestamp\" found in index <{}>", index);
        return IndexRangeStats.EMPTY;
    }
    final Min minAgg = f.getAggregations().get("ts_min");
    final long minUnixTime = new Double(minAgg.getValue()).longValue();
    final DateTime min = new DateTime(minUnixTime, DateTimeZone.UTC);
    final Max maxAgg = f.getAggregations().get("ts_max");
    final long maxUnixTime = new Double(maxAgg.getValue()).longValue();
    final DateTime max = new DateTime(maxUnixTime, DateTimeZone.UTC);
    // make sure we return an empty list, so we can differentiate between old indices that don't have this information
    // and newer ones that simply have no streams.
    final Terms streams = f.getAggregations().get("streams");
    final List<String> streamIds = streams.getBuckets().stream().map(MultiBucketsAggregation.Bucket::getKeyAsString).collect(toList());
    return IndexRangeStats.create(min, max, streamIds);
}

Example 73 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class MoreSearchAdapterES7 method eventSearch.

@Override
public MoreSearch.Result eventSearch(String queryString, TimeRange timerange, Set<String> affectedIndices, Sorting sorting, int page, int perPage, Set<String> eventStreams, String filterString, Set<String> forbiddenSourceStreams) {
    final QueryBuilder query = (queryString.isEmpty() || queryString.equals("*")) ? matchAllQuery() : queryStringQuery(queryString).allowLeadingWildcard(allowLeadingWildcard);
    final BoolQueryBuilder filter = boolQuery().filter(query).filter(termsQuery(EventDto.FIELD_STREAMS, eventStreams)).filter(requireNonNull(TimeRangeQueryFactory.create(timerange)));
    if (!isNullOrEmpty(filterString)) {
        filter.filter(queryStringQuery(filterString));
    }
    if (!forbiddenSourceStreams.isEmpty()) {
        // If an event has any stream in "source_streams" that the calling search user is not allowed to access,
        // the event must not be in the search result.
        filter.filter(boolQuery().mustNot(termsQuery(EventDto.FIELD_SOURCE_STREAMS, forbiddenSourceStreams)));
    }
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filter).from((page - 1) * perPage).size(perPage).sort(sorting.getField(), sortOrderMapper.fromSorting(sorting)).trackTotalHits(true);
    final Set<String> indices = affectedIndices.isEmpty() ? Collections.singleton("") : affectedIndices;
    final SearchRequest searchRequest = new SearchRequest(indices.toArray(new String[0])).source(searchSourceBuilder).indicesOptions(INDICES_OPTIONS);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Query:\n{}", searchSourceBuilder.toString(new ToXContent.MapParams(Collections.singletonMap("pretty", "true"))));
        LOG.debug("Execute search: {}", searchRequest.toString());
    }
    final SearchResponse searchResult = client.search(searchRequest, "Unable to perform search query");
    final List<ResultMessage> hits = Streams.stream(searchResult.getHits()).map(ResultMessageFactory::fromSearchHit).collect(Collectors.toList());
    final long total = searchResult.getHits().getTotalHits().value;
    return MoreSearch.Result.builder().results(hits).resultsCount(total).duration(searchResult.getTook().getMillis()).usedIndexNames(affectedIndices).executedQuery(searchSourceBuilder.toString()).build();
}

Example 74 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class ElasticsearchBackend method doRun.

@Override
public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext queryContext) {
    if (query.searchTypes().isEmpty()) {
        return QueryResult.builder().query(query).searchTypes(Collections.emptyMap()).errors(new HashSet<>(queryContext.errors())).build();
    }
    LOG.debug("Running query {} for job {}", query.id(), job.getId());
    final HashMap<String, SearchType.Result> resultsMap = Maps.newHashMap();
    final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange());
    final Map<String, SearchSourceBuilder> searchTypeQueries = queryContext.searchTypeQueries();
    final List<String> searchTypeIds = new ArrayList<>(searchTypeQueries.keySet());
    final List<SearchRequest> searches = searchTypeIds.stream().map(searchTypeId -> {
        final Set<String> affectedIndicesForSearchType = query.searchTypes().stream().filter(s -> s.id().equalsIgnoreCase(searchTypeId)).findFirst().flatMap(searchType -> {
            if (searchType.effectiveStreams().isEmpty() && !query.globalOverride().flatMap(GlobalOverride::timerange).isPresent() && !searchType.timerange().isPresent()) {
                return Optional.empty();
            }
            final Set<String> usedStreamIds = searchType.effectiveStreams().isEmpty() ? query.usedStreamIds() : searchType.effectiveStreams();
            return Optional.of(indexLookup.indexNamesForStreamsInTimeRange(usedStreamIds, query.effectiveTimeRange(searchType)));
        }).orElse(affectedIndices);
        Set<String> indices = affectedIndicesForSearchType.isEmpty() ? Collections.singleton("") : affectedIndicesForSearchType;
        return new SearchRequest().source(searchTypeQueries.get(searchTypeId)).indices(indices.toArray(new String[0])).indicesOptions(IndicesOptions.fromOptions(false, false, true, false));
    }).collect(Collectors.toList());
    final List<MultiSearchResponse.Item> results = client.msearch(searches, "Unable to perform search query: ");
    for (SearchType searchType : query.searchTypes()) {
        final String searchTypeId = searchType.id();
        final Provider<ESSearchTypeHandler<? extends SearchType>> handlerProvider = elasticsearchSearchTypeHandlers.get(searchType.type());
        if (handlerProvider == null) {
            LOG.error("Unknown search type '{}', cannot convert query result.", searchType.type());
            // no need to add another error here, as the query generation code will have added the error about the missing handler already
            continue;
        }
        if (isSearchTypeWithError(queryContext, searchTypeId)) {
            LOG.error("Failed search type '{}', cannot convert query result, skipping.", searchType.type());
            // no need to add another error here, as the query generation code will have added the error about the missing handler already
            continue;
        }
        // we create a new instance because some search type handlers might need to track information between generating the query and
        // processing its result, such as aggregations, which depend on the name and type
        final ESSearchTypeHandler<? extends SearchType> handler = handlerProvider.get();
        final int searchTypeIndex = searchTypeIds.indexOf(searchTypeId);
        final MultiSearchResponse.Item multiSearchResponse = results.get(searchTypeIndex);
        if (multiSearchResponse.isFailure()) {
            ElasticsearchException e = new ElasticsearchException("Search type returned error: ", multiSearchResponse.getFailure());
            queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
        } else if (checkForFailedShards(multiSearchResponse).isPresent()) {
            ElasticsearchException e = checkForFailedShards(multiSearchResponse).get();
            queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
        } else {
            final SearchType.Result searchTypeResult = handler.extractResult(job, query, searchType, multiSearchResponse.getResponse(), queryContext);
            if (searchTypeResult != null) {
                resultsMap.put(searchTypeId, searchTypeResult);
            }
        }
    }
    LOG.debug("Query {} ran for job {}", query.id(), job.getId());
    return QueryResult.builder().query(query).searchTypes(resultsMap).errors(new HashSet<>(queryContext.errors())).build();
}

Example 75 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class SearchesAdapterES7 method search.

@Override
public SearchResult search(Set<String> indices, Set<IndexRange> indexRanges, SearchesConfig config) {
    final SearchSourceBuilder searchSourceBuilder = searchRequestFactory.create(config);
    if (indexRanges.isEmpty()) {
        return SearchResult.empty(config.query(), searchSourceBuilder.toString());
    }
    final SearchRequest searchRequest = new SearchRequest(indices.toArray(new String[0])).source(searchSourceBuilder);
    final SearchResponse searchResult = client.search(searchRequest, "Unable to perform search query");
    final List<ResultMessage> resultMessages = extractResultMessages(searchResult);
    final long totalResults = searchResult.getHits().getTotalHits().value;
    final long tookMs = searchResult.getTook().getMillis();
    final String builtQuery = searchSourceBuilder.toString();
    return new SearchResult(resultMessages, totalResults, indexRanges, config.query(), builtQuery, tookMs);
}