Examples with IndexSetConfig org.graylog2.indexer.indexset.IndexSetConfig used on opensource projects

Example 6 with IndexSetConfig

use of org.graylog2.indexer.indexset.IndexSetConfig in project graylog2-server by Graylog2.

the class IndexRetentionThread method doRun.

@Override
public void doRun() {
    if (!cluster.isConnected() || !cluster.isHealthy()) {
        LOG.info("Elasticsearch cluster not available, skipping index retention checks.");
        return;
    }
    for (final IndexSet indexSet : indexSetRegistry) {
        if (!indexSet.getConfig().isWritable()) {
            LOG.debug("Skipping non-writable index set <{}> ({})", indexSet.getConfig().id(), indexSet.getConfig().title());
            continue;
        }
        final IndexSetConfig config = indexSet.getConfig();
        final Provider<RetentionStrategy> retentionStrategyProvider = retentionStrategyMap.get(config.retentionStrategyClass());
        if (retentionStrategyProvider == null) {
            LOG.warn("Retention strategy \"{}\" not found, not running index retention!", config.retentionStrategyClass());
            retentionProblemNotification("Index Retention Problem!", "Index retention strategy " + config.retentionStrategyClass() + " not found! Please fix your index retention configuration!");
            continue;
        }
        retentionStrategyProvider.get().retain(indexSet);
    }
}

Example 7 with IndexSetConfig

use of org.graylog2.indexer.indexset.IndexSetConfig in project graylog2-server by Graylog2.

the class TimeBasedRotationStrategyTest method shouldRotateNonIntegralPeriod.

@Test
public void shouldRotateNonIntegralPeriod() throws Exception {
    // start 5 minutes before full hour
    final DateTime initialTime = new DateTime(2014, 1, 1, 1, 55, 0, 0, DateTimeZone.UTC);
    final Period period = minutes(10);
    final InstantMillisProvider clock = new InstantMillisProvider(initialTime);
    DateTimeUtils.setCurrentMillisProvider(clock);
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    when(indices.indexCreationDate(anyString())).thenReturn(initialTime.minus(minutes(11)));
    // Should rotate the first index.
    // time is 01:55:00, index was created at 01:44:00, so we missed one period, and should rotate
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, times(1)).cycle();
    reset(indexSet);
    // advance time to 01:55:01
    clock.tick(seconds(1));
    // Did not cross rotation period.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, never()).cycle();
    reset(indexSet);
    // advance time to 02:00:00
    clock.tick(minutes(4).withSeconds(59));
    // Crossed rotation period.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, times(1)).cycle();
    reset(indexSet);
    // advance time multiple rotation periods into the future
    // to time 02:51:00
    clock.tick(minutes(51));
    // Crossed multiple rotation periods.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, times(1)).cycle();
    reset(indexSet);
    // move time to 2:52:00
    // this should not cycle again, because next valid rotation time is 3:00:00
    clock.tick(minutes(1));
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, never()).cycle();
    reset(indexSet);
}

Example 8 with IndexSetConfig

use of org.graylog2.indexer.indexset.IndexSetConfig in project graylog2-server by Graylog2.

the class TimeBasedRotationStrategyTest method shouldRotateHourly.

@Test
public void shouldRotateHourly() throws Exception {
    final DateTime initialTime = new DateTime(2014, 1, 1, 1, 59, 59, 0, DateTimeZone.UTC);
    final Period period = Period.hours(1);
    final InstantMillisProvider clock = new InstantMillisProvider(initialTime);
    DateTimeUtils.setCurrentMillisProvider(clock);
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    when(indices.indexCreationDate(anyString())).thenReturn(initialTime.minus(minutes(5)));
    // Should not rotate the first index.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    rotationStrategy.rotate(indexSet);
    verify(indexSet, never()).cycle();
    reset(indexSet);
    clock.tick(seconds(2));
    // Crossed rotation period.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, times(1)).cycle();
    reset(indexSet);
    clock.tick(seconds(2));
    // Did not cross rotation period.
    when(indexSet.getNewestIndex()).thenReturn("ignored");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet);
    verify(indexSet, never()).cycle();
    reset(indexSet);
}

Example 9 with IndexSetConfig

use of org.graylog2.indexer.indexset.IndexSetConfig in project graylog2-server by Graylog2.

the class V20161124104700_AddRetentionRotationAndDefaultFlagToIndexSetMigrationTest method upgradeWithWrongRetentionPrefix.

@Test
public void upgradeWithWrongRetentionPrefix() throws Exception {
    final String rotationStrategyClass = MessageCountRotationStrategy.class.getCanonicalName();
    final String retentionStrategyClass = "bar";
    final RotationStrategyConfig rotationStrategy = MessageCountRotationStrategyConfig.createDefault();
    final RetentionStrategyConfig retentionStrategy = DeletionRetentionStrategyConfig.createDefault();
    final IndexSetConfig config1 = IndexSetConfig.builder().id("id-1").title("title-1").indexPrefix("prefix-1").shards(1).replicas(0).rotationStrategy(rotationStrategy).retentionStrategy(retentionStrategy).creationDate(ZonedDateTime.of(2016, 10, 12, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-1").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    final IndexSetConfig config2 = IndexSetConfig.builder().id("id-2").title("title-2").indexPrefix("prefix-2").shards(1).replicas(0).rotationStrategy(rotationStrategy).retentionStrategy(retentionStrategy).creationDate(ZonedDateTime.of(2016, 10, 13, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-2").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    when(clusterConfigService.get(IndexManagementConfig.class)).thenReturn(IndexManagementConfig.create(rotationStrategyClass, retentionStrategyClass));
    when(indexSetService.findAll()).thenReturn(Lists.newArrayList(config1, config2));
    expectedException.expect(IllegalStateException.class);
    expectedException.expectMessage("retention strategy config type <");
    migration.upgrade();
    verify(indexSetService, never()).save(any(IndexSetConfig.class));
    verify(clusterConfigService, never()).write(V20161124104700_AddRetentionRotationAndDefaultFlagToIndexSetMigration.MigrationCompleted.class);
}

Example 10 with IndexSetConfig

use of org.graylog2.indexer.indexset.IndexSetConfig in project graylog2-server by Graylog2.

the class V20161124104700_AddRetentionRotationAndDefaultFlagToIndexSetMigrationTest method upgradeWithWrongRotationPrefix.

@Test
public void upgradeWithWrongRotationPrefix() throws Exception {
    final String rotationStrategyClass = "foo";
    final String retentionStrategyClass = DeletionRetentionStrategy.class.getCanonicalName();
    final RotationStrategyConfig rotationStrategy = MessageCountRotationStrategyConfig.createDefault();
    final RetentionStrategyConfig retentionStrategy = DeletionRetentionStrategyConfig.createDefault();
    final IndexSetConfig config1 = IndexSetConfig.builder().id("id-1").title("title-1").indexPrefix("prefix-1").shards(1).replicas(0).rotationStrategy(rotationStrategy).retentionStrategy(retentionStrategy).creationDate(ZonedDateTime.of(2016, 10, 12, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-1").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    final IndexSetConfig config2 = IndexSetConfig.builder().id("id-2").title("title-2").indexPrefix("prefix-2").shards(1).replicas(0).rotationStrategy(rotationStrategy).retentionStrategy(retentionStrategy).creationDate(ZonedDateTime.of(2016, 10, 13, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-2").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    when(clusterConfigService.get(IndexManagementConfig.class)).thenReturn(IndexManagementConfig.create(rotationStrategyClass, retentionStrategyClass));
    when(indexSetService.findAll()).thenReturn(Lists.newArrayList(config1, config2));
    expectedException.expect(IllegalStateException.class);
    expectedException.expectMessage("rotation strategy config type <");
    migration.upgrade();
    verify(indexSetService, never()).save(any(IndexSetConfig.class));
    verify(clusterConfigService, never()).write(V20161124104700_AddRetentionRotationAndDefaultFlagToIndexSetMigration.MigrationCompleted.class);
}