Examples with IndexSet org.graylog2.indexer.IndexSet used on opensource projects

Example 1 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class IndexRetentionThread method doRun.

@Override
public void doRun() {
    if (!cluster.isConnected() || !cluster.isHealthy()) {
        LOG.info("Elasticsearch cluster not available, skipping index retention checks.");
        return;
    }
    for (final IndexSet indexSet : indexSetRegistry) {
        if (!indexSet.getConfig().isWritable()) {
            LOG.debug("Skipping non-writable index set <{}> ({})", indexSet.getConfig().id(), indexSet.getConfig().title());
            continue;
        }
        final IndexSetConfig config = indexSet.getConfig();
        final Provider<RetentionStrategy> retentionStrategyProvider = retentionStrategyMap.get(config.retentionStrategyClass());
        if (retentionStrategyProvider == null) {
            LOG.warn("Retention strategy \"{}\" not found, not running index retention!", config.retentionStrategyClass());
            retentionProblemNotification("Index Retention Problem!", "Index retention strategy " + config.retentionStrategyClass() + " not found! Please fix your index retention configuration!");
            continue;
        }
        retentionStrategyProvider.get().retain(indexSet);
    }
}

Example 2 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class TimeBasedRotationStrategyTest method shouldRotateNonIntegralPeriod.

@Test
public void shouldRotateNonIntegralPeriod() throws Exception {
    // start 5 minutes before full hour
    final DateTime initialTime = new DateTime(2014, 1, 1, 1, 55, 0, 0, DateTimeZone.UTC);
    final Period period = minutes(10);
    final InstantMillisProvider clock = new InstantMillisProvider(initialTime);
    DateTimeUtils.setCurrentMillisProvider(clock);
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    when(indices.indexCreationDate(anyString())).thenReturn(initialTime.minus(minutes(11)));
    // Should rotate the first index.
    // time is 01:55:00, index was created at 01:44:00, so we missed one period, and should rotate
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, times(1)).cycle();
    reset(indexSet);
    // advance time to 01:55:01
    clock.tick(seconds(1));
    // Did not cross rotation period.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, never()).cycle();
    reset(indexSet);
    // advance time to 02:00:00
    clock.tick(minutes(4).withSeconds(59));
    // Crossed rotation period.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, times(1)).cycle();
    reset(indexSet);
    // advance time multiple rotation periods into the future
    // to time 02:51:00
    clock.tick(minutes(51));
    // Crossed multiple rotation periods.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, times(1)).cycle();
    reset(indexSet);
    // move time to 2:52:00
    // this should not cycle again, because next valid rotation time is 3:00:00
    clock.tick(minutes(1));
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, never()).cycle();
    reset(indexSet);
}

Example 3 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class TimeBasedRotationStrategyTest method shouldRotateHourly.

@Test
public void shouldRotateHourly() throws Exception {
    final DateTime initialTime = new DateTime(2014, 1, 1, 1, 59, 59, 0, DateTimeZone.UTC);
    final Period period = Period.hours(1);
    final InstantMillisProvider clock = new InstantMillisProvider(initialTime);
    DateTimeUtils.setCurrentMillisProvider(clock);
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    when(indices.indexCreationDate(anyString())).thenReturn(initialTime.minus(minutes(5)));
    // Should not rotate the first index.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    rotationStrategy.rotate(indexSet);
    verify(indexSet, never()).cycle();
    reset(indexSet);
    clock.tick(seconds(2));
    // Crossed rotation period.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, times(1)).cycle();
    reset(indexSet);
    clock.tick(seconds(2));
    // Did not cross rotation period.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, never()).cycle();
    reset(indexSet);
}

Example 4 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class SearchesTest method determineAffectedIndicesFilterIndexPrefix.

@Test
public void determineAffectedIndicesFilterIndexPrefix() throws Exception {
    final DateTime now = DateTime.now(DateTimeZone.UTC);
    final MongoIndexRange indexRange0 = MongoIndexRange.create("graylog_0", now, now.plusDays(1), now, 0);
    final MongoIndexRange indexRange1 = MongoIndexRange.create("graylog_1", now.plusDays(1), now.plusDays(2), now, 0);
    final MongoIndexRange b0 = MongoIndexRange.create("b_0", now.plusDays(1), now.plusDays(2), now, 0);
    final MongoIndexRange b1 = MongoIndexRange.create("b_1", now.plusDays(1), now.plusDays(2), now, 0);
    final SortedSet<IndexRange> indices = ImmutableSortedSet.orderedBy(IndexRange.COMPARATOR).add(indexRange0).add(indexRange1).add(b0).add(b1).build();
    final Stream bStream = mock(Stream.class);
    when(indexRangeService.find(any(DateTime.class), any(DateTime.class))).thenReturn(indices);
    when(streamService.load(eq("123456789ABCDEF"))).thenReturn(bStream);
    final IndexSet indexSet = mock(IndexSet.class);
    when(indexSet.isManagedIndex(startsWith("b_"))).thenReturn(true);
    when(bStream.getIndexSet()).thenReturn(indexSet);
    final TimeRange absoluteRange = AbsoluteRange.create(now.minusDays(1), now.plusDays(1));
    assertThat(searches.determineAffectedIndices(absoluteRange, "streams:123456789ABCDEF")).containsOnly(b0.indexName(), b1.indexName());
}

Example 5 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class MessageResource method analyze.

@GET
@Path("/{index}/analyze")
@Timed
@ApiOperation(value = "Analyze a message string", notes = "Returns what tokens/terms a message string (message or full_message) is split to.")
@RequiresPermissions(RestPermissions.MESSAGES_ANALYZE)
@ApiResponses(value = { @ApiResponse(code = 404, message = "Specified index does not exist.") })
public MessageTokens analyze(@ApiParam(name = "index", value = "The index the message containing the string is stored in.", required = true) @PathParam("index") String index, @ApiParam(name = "analyzer", value = "The analyzer to use.") @QueryParam("analyzer") @Nullable String analyzer, @ApiParam(name = "string", value = "The string to analyze.", required = true) @QueryParam("string") @NotEmpty String string) {
    final String indexAnalyzer = indexSetRegistry.getForIndex(index).map(indexSet -> indexSet.getConfig().indexAnalyzer()).orElse("standard");
    final String messageAnalyzer = analyzer == null ? indexAnalyzer : analyzer;
    try {
        return MessageTokens.create(messages.analyze(string, index, messageAnalyzer));
    } catch (IndexNotFoundException e) {
        final String message = "Index " + index + " does not exist.";
        LOG.error(message, e);
        throw new NotFoundException(message);
    }
}