Search in sources :

Example 1 with EVENT_TEMPLATE_TYPE

use of org.graylog2.indexer.EventIndexTemplateProvider.EVENT_TEMPLATE_TYPE in project graylog2-server by Graylog2.

the class V20190705071400_AddEventIndexSetsMigration method setupEventsIndexSet.

private IndexSet setupEventsIndexSet(String indexSetTitle, String indexSetDescription, String indexPrefix) {
    final Optional<IndexSetConfig> optionalIndexSetConfig = getEventsIndexSetConfig(indexPrefix);
    if (optionalIndexSetConfig.isPresent()) {
        return mongoIndexSetFactory.create(optionalIndexSetConfig.get());
    }
    final IndexSetConfig indexSetConfig = IndexSetConfig.builder().title(indexSetTitle).description(indexSetDescription).indexTemplateType(EVENT_TEMPLATE_TYPE).isWritable(true).isRegular(false).indexPrefix(indexPrefix).shards(elasticsearchConfiguration.getShards()).replicas(elasticsearchConfiguration.getReplicas()).rotationStrategyClass(TimeBasedRotationStrategy.class.getCanonicalName()).rotationStrategy(TimeBasedRotationStrategyConfig.create(Period.months(1), null)).retentionStrategyClass(DeletionRetentionStrategy.class.getCanonicalName()).retentionStrategy(DeletionRetentionStrategyConfig.create(12)).creationDate(ZonedDateTime.now(ZoneOffset.UTC)).indexAnalyzer(elasticsearchConfiguration.getAnalyzer()).indexTemplateName(indexPrefix + "-template").indexOptimizationMaxNumSegments(elasticsearchConfiguration.getIndexOptimizationMaxNumSegments()).indexOptimizationDisabled(elasticsearchConfiguration.isDisableIndexOptimization()).fieldTypeRefreshInterval(Duration.standardMinutes(1)).build();
    try {
        final Optional<IndexSetValidator.Violation> violation = indexSetValidator.validate(indexSetConfig);
        if (violation.isPresent()) {
            throw new RuntimeException(violation.get().message());
        }
        final IndexSetConfig savedIndexSet = indexSetService.save(indexSetConfig);
        LOG.info("Successfully created events index-set <{}/{}>", savedIndexSet.id(), savedIndexSet.title());
        return mongoIndexSetFactory.create(savedIndexSet);
    } catch (DuplicateKeyException e) {
        LOG.error("Couldn't create index-set <{}/{}>", indexSetTitle, indexPrefix);
        throw new RuntimeException(e.getMessage());
    }
}
Also used : DeletionRetentionStrategy(org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy) IndexSetConfig(org.graylog2.indexer.indexset.IndexSetConfig) TimeBasedRotationStrategy(org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategy) DuplicateKeyException(com.mongodb.DuplicateKeyException)

Example 2 with EVENT_TEMPLATE_TYPE

use of org.graylog2.indexer.EventIndexTemplateProvider.EVENT_TEMPLATE_TYPE in project graylog2-server by Graylog2.

the class SearchesIT method determineAffectedIndicesWithRangesExcludeEvents.

@Test
public void determineAffectedIndicesWithRangesExcludeEvents() throws Exception {
    final Set<IndexSet> eventIndexSets = Arrays.asList("gl-events", "gl-system-events").stream().map(prefix -> new TestIndexSet(indexSet.getConfig().toBuilder().indexPrefix(prefix).indexTemplateType(EVENT_TEMPLATE_TYPE).build())).collect(Collectors.toSet());
    when(indexSetRegistry.getForIndices(anyCollection())).thenReturn(eventIndexSets);
    final DateTime now = DateTime.now(DateTimeZone.UTC);
    final MongoIndexRange indexRange0 = MongoIndexRange.create("gl-events_0", now, now.plusDays(1), now, 0);
    final MongoIndexRange indexRange1 = MongoIndexRange.create("gl-system-events_2", now.plusDays(1), now.plusDays(2), now, 0);
    final MongoIndexRange indexRange2 = MongoIndexRange.create("graylog_0", now, now.plusDays(1), now, 0);
    final SortedSet<IndexRange> indices = ImmutableSortedSet.orderedBy(IndexRange.COMPARATOR).add(indexRange0).add(indexRange1).add(indexRange2).build();
    when(indexRangeService.find(any(DateTime.class), any(DateTime.class))).thenReturn(indices);
    final TimeRange absoluteRange = AbsoluteRange.create(now.minusDays(1), now.plusDays(1));
    assertThat(searches.determineAffectedIndicesWithRanges(absoluteRange, null)).containsExactly(indexRange2);
}
Also used : DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) IndexSetConfig(org.graylog2.indexer.indexset.IndexSetConfig) IndexRangeComparator(org.graylog2.indexer.ranges.IndexRangeComparator) SortedSet(java.util.SortedSet) ArgumentMatchers.eq(org.mockito.ArgumentMatchers.eq) ZonedDateTime(java.time.ZonedDateTime) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) EVENT_TEMPLATE_TYPE(org.graylog2.indexer.EventIndexTemplateProvider.EVENT_TEMPLATE_TYPE) RelativeRange(org.graylog2.plugin.indexer.searches.timeranges.RelativeRange) ResultMessage(org.graylog2.indexer.results.ResultMessage) DeletionRetentionStrategyConfig(org.graylog2.indexer.retention.strategies.DeletionRetentionStrategyConfig) Indices(org.graylog2.indexer.indices.Indices) AbsoluteRange(org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange) IndexSet(org.graylog2.indexer.IndexSet) ZoneOffset(java.time.ZoneOffset) ElasticsearchBaseTest(org.graylog.testing.elasticsearch.ElasticsearchBaseTest) SearchResult(org.graylog2.indexer.results.SearchResult) TimeRange(org.graylog2.plugin.indexer.searches.timeranges.TimeRange) FakeStream(org.graylog2.buffers.processors.fakestreams.FakeStream) ArgumentMatchers.startsWith(org.mockito.ArgumentMatchers.startsWith) ImmutableMap(com.google.common.collect.ImmutableMap) Set(java.util.Set) Collectors(java.util.stream.Collectors) CountResult(org.graylog2.indexer.results.CountResult) List(java.util.List) Stream(org.graylog2.plugin.streams.Stream) MongoIndexRange(org.graylog2.indexer.ranges.MongoIndexRange) StreamService(org.graylog2.streams.StreamService) Timer(com.codahale.metrics.Timer) Mockito.mock(org.mockito.Mockito.mock) ArgumentMatchers.any(org.mockito.ArgumentMatchers.any) Histogram(com.codahale.metrics.Histogram) FieldStatsResult(org.graylog2.indexer.results.FieldStatsResult) Mock(org.mockito.Mock) ScrollResult(org.graylog2.indexer.results.ScrollResult) MessageCountRotationStrategyConfig(org.graylog2.indexer.rotation.strategies.MessageCountRotationStrategyConfig) HashSet(java.util.HashSet) MockitoJUnit(org.mockito.junit.MockitoJUnit) MessageCountRotationStrategy(org.graylog2.indexer.rotation.strategies.MessageCountRotationStrategy) IndexSetRegistry(org.graylog2.indexer.IndexSetRegistry) Before(org.junit.Before) ImmutableSortedSet(com.google.common.collect.ImmutableSortedSet) IndexRangeService(org.graylog2.indexer.ranges.IndexRangeService) MetricRegistry(com.codahale.metrics.MetricRegistry) NO_BATCHSIZE(org.graylog2.indexer.searches.ScrollCommand.NO_BATCHSIZE) DateTime(org.joda.time.DateTime) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) TestIndexSet(org.graylog2.indexer.TestIndexSet) UTC(org.joda.time.DateTimeZone.UTC) IndexRange(org.graylog2.indexer.ranges.IndexRange) ArgumentMatchers.anyCollection(org.mockito.ArgumentMatchers.anyCollection) Rule(org.junit.Rule) DeletionRetentionStrategy(org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy) MockitoRule(org.mockito.junit.MockitoRule) KeywordRange(org.graylog2.plugin.indexer.searches.timeranges.KeywordRange) Collections(java.util.Collections) TestIndexSet(org.graylog2.indexer.TestIndexSet) MongoIndexRange(org.graylog2.indexer.ranges.MongoIndexRange) IndexRange(org.graylog2.indexer.ranges.IndexRange) TimeRange(org.graylog2.plugin.indexer.searches.timeranges.TimeRange) MongoIndexRange(org.graylog2.indexer.ranges.MongoIndexRange) IndexSet(org.graylog2.indexer.IndexSet) TestIndexSet(org.graylog2.indexer.TestIndexSet) ZonedDateTime(java.time.ZonedDateTime) DateTime(org.joda.time.DateTime) ElasticsearchBaseTest(org.graylog.testing.elasticsearch.ElasticsearchBaseTest) Test(org.junit.Test)

Aggregations

IndexSetConfig (org.graylog2.indexer.indexset.IndexSetConfig)2 DeletionRetentionStrategy (org.graylog2.indexer.retention.strategies.DeletionRetentionStrategy)2 Histogram (com.codahale.metrics.Histogram)1 MetricRegistry (com.codahale.metrics.MetricRegistry)1 Timer (com.codahale.metrics.Timer)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSortedSet (com.google.common.collect.ImmutableSortedSet)1 DuplicateKeyException (com.mongodb.DuplicateKeyException)1 ZoneOffset (java.time.ZoneOffset)1 ZonedDateTime (java.time.ZonedDateTime)1 Arrays (java.util.Arrays)1 Collections (java.util.Collections)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Set (java.util.Set)1 SortedSet (java.util.SortedSet)1 Collectors (java.util.stream.Collectors)1 Assertions.assertThat (org.assertj.core.api.Assertions.assertThat)1 ElasticsearchBaseTest (org.graylog.testing.elasticsearch.ElasticsearchBaseTest)1 FakeStream (org.graylog2.buffers.processors.fakestreams.FakeStream)1