Examples with Indices org.graylog2.indexer.indices.Indices used on opensource projects

Example 66 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class MoreSearchAdapterES6 method scrollEvents.

@Override
public void scrollEvents(String queryString, TimeRange timeRange, Set<String> affectedIndices, Set<String> streams, String scrollTime, int batchSize, ScrollEventsCallback resultCallback) throws EventProcessorException {
    final QueryBuilder query = (queryString.trim().isEmpty() || queryString.trim().equals("*")) ? matchAllQuery() : queryStringQuery(queryString).allowLeadingWildcard(allowLeadingWildcard);
    final BoolQueryBuilder filter = boolQuery().filter(query).filter(requireNonNull(TimeRangeQueryFactory.create(timeRange)));
    // Filtering with an empty streams list doesn't work and would return zero results
    if (!streams.isEmpty()) {
        filter.filter(termsQuery(Message.FIELD_STREAMS, streams));
    }
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filter).size(batchSize);
    final Search.Builder searchBuilder = new Search.Builder(searchSourceBuilder.toString()).addType(IndexMapping.TYPE_MESSAGE).addIndex(affectedIndices.isEmpty() ? Collections.singleton("") : affectedIndices).addSort(new Sort("timestamp", Sort.Sorting.ASC)).allowNoIndices(false).ignoreUnavailable(false).setParameter(Parameters.SCROLL, scrollTime);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Query:\n{}", searchSourceBuilder.toString(new ToXContent.MapParams(Collections.singletonMap("pretty", "true"))));
        LOG.debug("Execute search: {}", searchBuilder.build().toString());
    }
    final ScrollResult scrollResult = scroll.scroll(searchBuilder.build(), () -> "Unable to scroll indices.", searchSourceBuilder.toString(), scrollTime, Collections.emptyList());
    final AtomicBoolean continueScrolling = new AtomicBoolean(true);
    final Stopwatch stopwatch = Stopwatch.createStarted();
    try {
        ScrollResult.ScrollChunk scrollChunk = scrollResult.nextChunk();
        while (continueScrolling.get() && scrollChunk != null) {
            final List<ResultMessage> messages = scrollChunk.getMessages();
            LOG.debug("Passing <{}> messages to callback", messages.size());
            resultCallback.accept(Collections.unmodifiableList(messages), continueScrolling);
            // Stop if the resultCallback told us to stop
            if (!continueScrolling.get()) {
                break;
            }
            scrollChunk = scrollResult.nextChunk();
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        try {
            // Tell Elasticsearch that we are done with the scroll so it can release resources as soon as possible
            // instead of waiting for the scroll timeout to kick in.
            scrollResult.cancel();
        } catch (Exception ignored) {
        }
        LOG.debug("Scrolling done - took {} ms", stopwatch.stop().elapsed(TimeUnit.MILLISECONDS));
    }
}

Example 67 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class SearchesAdapterES6 method search.

@Override
public SearchResult search(Set<String> indices, Set<IndexRange> indexRanges, SearchesConfig config) {
    final SearchSourceBuilder requestBuilder = searchRequest(config);
    if (indexRanges.isEmpty()) {
        return SearchResult.empty(config.query(), requestBuilder.toString());
    }
    final Search.Builder searchBuilder = new Search.Builder(requestBuilder.toString()).addType(IndexMapping.TYPE_MESSAGE).addIndex(indices);
    final io.searchbox.core.SearchResult searchResult = multiSearch.wrap(searchBuilder.build(), () -> "Unable to perform search query");
    final List<ResultMessage> hits = searchResult.getHits(Map.class, false).stream().map(hit -> ResultMessage.parseFromSource(hit.id, hit.index, (Map<String, Object>) hit.source, hit.highlight)).collect(Collectors.toList());
    return new SearchResult(hits, searchResult.getTotal(), indexRanges, config.query(), requestBuilder.toString(), multiSearch.tookMsFromSearchResult(searchResult));
}

Example 68 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class SearchesAdapterES6 method fieldStats.

@Override
public FieldStatsResult fieldStats(String query, String filter, TimeRange range, Set<String> indices, String field, boolean includeCardinality, boolean includeStats, boolean includeCount) {
    final SearchSourceBuilder searchSourceBuilder;
    if (filter == null) {
        searchSourceBuilder = standardSearchRequest(query, range);
    } else {
        searchSourceBuilder = filteredSearchRequest(query, filter, range);
    }
    final FilterAggregationBuilder filterBuilder = AggregationBuilders.filter(AGG_FILTER, standardAggregationFilters(range, filter));
    if (includeCount) {
        searchSourceBuilder.aggregation(AggregationBuilders.count(AGG_VALUE_COUNT).field(field));
    }
    if (includeStats) {
        searchSourceBuilder.aggregation(AggregationBuilders.extendedStats(AGG_EXTENDED_STATS).field(field));
    }
    if (includeCardinality) {
        searchSourceBuilder.aggregation(AggregationBuilders.cardinality(AGG_CARDINALITY).field(field));
    }
    searchSourceBuilder.aggregation(filterBuilder);
    if (indices.isEmpty()) {
        return FieldStatsResult.empty(query, searchSourceBuilder.toString());
    }
    final Search searchRequest = new Search.Builder(searchSourceBuilder.toString()).addType(IndexMapping.TYPE_MESSAGE).addIndex(indices).build();
    final io.searchbox.core.SearchResult searchResponse = multiSearch.wrap(searchRequest, () -> "Unable to retrieve fields stats");
    final List<ResultMessage> hits = searchResponse.getHits(Map.class, false).stream().map(hit -> ResultMessage.parseFromSource(hit.id, hit.index, (Map<String, Object>) hit.source)).collect(Collectors.toList());
    final ExtendedStatsAggregation extendedStatsAggregation = searchResponse.getAggregations().getExtendedStatsAggregation(AGG_EXTENDED_STATS);
    final ValueCountAggregation valueCountAggregation = searchResponse.getAggregations().getValueCountAggregation(AGG_VALUE_COUNT);
    final CardinalityAggregation cardinalityAggregation = searchResponse.getAggregations().getCardinalityAggregation(AGG_CARDINALITY);
    return createFieldStatsResult(valueCountAggregation, extendedStatsAggregation, cardinalityAggregation, hits, query, searchSourceBuilder.toString(), multiSearch.tookMsFromSearchResult(searchResponse));
}

Example 69 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class ClusterAdapterES6 method shardStats.

@Override
public ShardStats shardStats(Collection<String> indices) {
    final Health clusterHealthRequest = new Health.Builder().addIndex(indices).build();
    final JestResult clusterHealthResponse = JestUtils.execute(jestClient, clusterHealthRequest, () -> "Couldn't read Elasticsearch cluster health");
    final JsonNode clusterHealthJson = clusterHealthResponse.getJsonObject();
    return ShardStats.create(clusterHealthJson.path("number_of_nodes").asInt(-1), clusterHealthJson.path("number_of_data_nodes").asInt(-1), clusterHealthJson.path("active_shards").asInt(-1), clusterHealthJson.path("relocating_shards").asInt(-1), clusterHealthJson.path("active_primary_shards").asInt(-1), clusterHealthJson.path("initializing_shards").asInt(-1), clusterHealthJson.path("unassigned_shards").asInt(-1), clusterHealthJson.path("timed_out").asBoolean());
}

Example 70 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class IndicesIT method setUp.

@BeforeEach
public void setUp() {
    // noinspection UnstableApiUsage
    eventBus = new EventBus("indices-test");
    final Node node = new Node(createNodeAdapter());
    final IndexMappingFactory indexMappingFactory = new IndexMappingFactory(node, ImmutableMap.of(MessageIndexTemplateProvider.MESSAGE_TEMPLATE_TYPE, new MessageIndexTemplateProvider()));
    indices = new Indices(indexMappingFactory, mock(NodeId.class), new NullAuditEventSender(), eventBus, indicesAdapter());
}