Search in sources :

Example 26 with IndexRange

use of org.graylog2.indexer.ranges.IndexRange in project graylog2-server by Graylog2.

the class Searches method search.

public SearchResult search(SearchesConfig config) {
    final Set<IndexRange> indexRanges = determineAffectedIndicesWithRanges(config.range(), config.filter());
    final Set<String> indices = extractIndexNamesFromIndexRanges(indexRanges);
    final SearchResult result = searchesAdapter.search(indices, indexRanges, config);
    recordEsMetrics(result.tookMs(), config.range());
    return result;
}
Also used : IndexRange(org.graylog2.indexer.ranges.IndexRange) SearchResult(org.graylog2.indexer.results.SearchResult)

Example 27 with IndexRange

use of org.graylog2.indexer.ranges.IndexRange in project graylog2-server by Graylog2.

the class RebuildIndexRangesJob method execute.

@Override
public void execute() {
    info("Recalculating index ranges.");
    // for each index set we know about
    final ListMultimap<IndexSet, String> indexSets = MultimapBuilder.hashKeys().arrayListValues().build();
    for (IndexSet indexSet : this.indexSets) {
        final String[] managedIndicesNames = indexSet.getManagedIndices();
        for (String name : managedIndicesNames) {
            indexSets.put(indexSet, name);
        }
    }
    if (indexSets.size() == 0) {
        info("No indices, nothing to calculate.");
        return;
    }
    indicesToCalculate = indexSets.values().size();
    Stopwatch sw = Stopwatch.createStarted();
    for (IndexSet indexSet : indexSets.keySet()) {
        LOG.info("Recalculating index ranges for index set {} ({}): {} indices affected.", indexSet.getConfig().title(), indexSet.getIndexWildcard(), indexSets.get(indexSet).size());
        for (String index : indexSets.get(indexSet)) {
            try {
                if (index.equals(indexSet.getActiveWriteIndex())) {
                    LOG.debug("{} is current write target, do not calculate index range for it", index);
                    final IndexRange emptyRange = indexRangeService.createUnknownRange(index);
                    try {
                        final IndexRange indexRange = indexRangeService.get(index);
                        if (indexRange.begin().getMillis() != 0 || indexRange.end().getMillis() != 0) {
                            LOG.info("Invalid date ranges for write index {}, resetting it.", index);
                            indexRangeService.save(emptyRange);
                        }
                    } catch (NotFoundException e) {
                        LOG.info("No index range found for write index {}, recreating it.", index);
                        indexRangeService.save(emptyRange);
                    }
                    indicesCalculated.incrementAndGet();
                    continue;
                }
            } catch (TooManyAliasesException e) {
                LOG.error("Multiple write alias targets found, this is a bug.");
                indicesCalculated.incrementAndGet();
                continue;
            }
            if (cancelRequested) {
                info("Stop requested. Not calculating next index range, not updating ranges.");
                sw.stop();
                return;
            }
            try {
                final IndexRange indexRange = indexRangeService.calculateRange(index);
                indexRangeService.save(indexRange);
                LOG.info("Created ranges for index {}: {}", index, indexRange);
            } catch (Exception e) {
                LOG.info("Could not calculate range of index [" + index + "]. Skipping.", e);
            } finally {
                indicesCalculated.incrementAndGet();
            }
        }
    }
    info("Done calculating index ranges for " + indicesToCalculate + " indices. Took " + sw.stop().elapsed(TimeUnit.MILLISECONDS) + "ms.");
}
Also used : Stopwatch(com.google.common.base.Stopwatch) NotFoundException(org.graylog2.database.NotFoundException) TooManyAliasesException(org.graylog2.indexer.indices.TooManyAliasesException) IndexSet(org.graylog2.indexer.IndexSet) TooManyAliasesException(org.graylog2.indexer.indices.TooManyAliasesException) NotFoundException(org.graylog2.database.NotFoundException)

Example 28 with IndexRange

use of org.graylog2.indexer.ranges.IndexRange in project graylog2-server by Graylog2.

the class IndexRangesCleanupPeriodical method doRun.

@Override
public void doRun() {
    if (!cluster.isConnected() || !cluster.isHealthy()) {
        LOG.info("Skipping index range cleanup because the Elasticsearch cluster is unreachable or unhealthy");
        return;
    }
    final Set<String> indexNames = ImmutableSet.copyOf(indexSetRegistry.getManagedIndices());
    final SortedSet<IndexRange> indexRanges = indexRangeService.findAll();
    final Set<String> removedIndices = new HashSet<>();
    for (IndexRange indexRange : indexRanges) {
        if (!indexNames.contains(indexRange.indexName())) {
            removedIndices.add(indexRange.indexName());
        }
    }
    if (!removedIndices.isEmpty()) {
        LOG.info("Removing index range information for unavailable indices: {}", removedIndices);
        eventBus.post(IndicesDeletedEvent.create(removedIndices));
    }
}
Also used : IndexRange(org.graylog2.indexer.ranges.IndexRange) HashSet(java.util.HashSet)

Example 29 with IndexRange

use of org.graylog2.indexer.ranges.IndexRange in project graylog2-server by Graylog2.

the class IndexRangesResource method list.

@GET
@Timed
@ApiOperation(value = "Get a list of all index ranges")
@Produces(MediaType.APPLICATION_JSON)
public IndexRangesResponse list() {
    final SortedSet<IndexRange> all = indexRangeService.findAll();
    final List<IndexRangeSummary> ranges = Lists.newArrayListWithCapacity(all.size());
    for (IndexRange range : all) {
        if (!isPermitted(RestPermissions.INDEXRANGES_READ, range.indexName())) {
            continue;
        }
        final IndexRangeSummary indexRange = IndexRangeSummary.create(range.indexName(), range.begin(), range.end(), range.calculatedAt(), range.calculationDuration());
        ranges.add(indexRange);
    }
    return IndexRangesResponse.create(ranges.size(), ranges);
}
Also used : IndexRange(org.graylog2.indexer.ranges.IndexRange) IndexRangeSummary(org.graylog2.rest.models.system.indexer.responses.IndexRangeSummary) Produces(javax.ws.rs.Produces) Timed(com.codahale.metrics.annotation.Timed) GET(javax.ws.rs.GET) ApiOperation(io.swagger.annotations.ApiOperation)

Example 30 with IndexRange

use of org.graylog2.indexer.ranges.IndexRange in project graylog2-server by Graylog2.

the class IndexLookupTest method mockSomeIndexRanges.

private List<IndexRange> mockSomeIndexRanges() {
    final IndexRange indexRange1 = mockIndexRange("index1");
    final IndexRange indexRange2 = mockIndexRange("index2");
    final SortedSet<IndexRange> indexRanges = sortedSetOf(indexRange1, indexRange2);
    when(indexRangeService.find(any(DateTime.class), any(DateTime.class))).thenReturn(indexRanges);
    return new ArrayList<>(indexRanges);
}
Also used : IndexRange(org.graylog2.indexer.ranges.IndexRange) ArrayList(java.util.ArrayList) DateTime(org.joda.time.DateTime)

Aggregations

IndexRange (org.graylog2.indexer.ranges.IndexRange)29 DateTime (org.joda.time.DateTime)18 Test (org.junit.Test)17 MongoIndexRange (org.graylog2.indexer.ranges.MongoIndexRange)15 ZonedDateTime (java.time.ZonedDateTime)14 TimeRange (org.graylog2.plugin.indexer.searches.timeranges.TimeRange)13 IndexSet (org.graylog2.indexer.IndexSet)7 SearchResult (org.graylog2.indexer.results.SearchResult)7 ElasticsearchBaseTest (org.graylog.testing.elasticsearch.ElasticsearchBaseTest)6 Stream (org.graylog2.plugin.streams.Stream)6 Set (java.util.Set)4 Collectors (java.util.stream.Collectors)4 ImmutableSortedSet (com.google.common.collect.ImmutableSortedSet)3 List (java.util.List)3 ResultMessage (org.graylog2.indexer.results.ResultMessage)3 Sorting (org.graylog2.indexer.searches.Sorting)3 Histogram (com.codahale.metrics.Histogram)2 MetricRegistry (com.codahale.metrics.MetricRegistry)2 Timer (com.codahale.metrics.Timer)2 Timed (com.codahale.metrics.annotation.Timed)2