Examples with RetentionStrategy org.graylog2.plugin.indexer.retention.RetentionStrategy used on opensource projects

Example 6 with RetentionStrategy

use of org.graylog2.plugin.indexer.retention.RetentionStrategy in project graylog2-server by Graylog2.

the class V20151210140600_ElasticsearchConfigMigration method upgrade.

// Migrate old Elasticsearch config settings to new ClusterConfig based ones.
@Override
public void upgrade() {
    // All default rotation strategy settings.
    final MessageCountRotationStrategyConfig messageCountRotationStrategyConfig = clusterConfigService.get(MessageCountRotationStrategyConfig.class);
    final SizeBasedRotationStrategyConfig sizeBasedRotationStrategyConfig = clusterConfigService.get(SizeBasedRotationStrategyConfig.class);
    final TimeBasedRotationStrategyConfig timeBasedRotationStrategyConfig = clusterConfigService.get(TimeBasedRotationStrategyConfig.class);
    if (messageCountRotationStrategyConfig == null) {
        final MessageCountRotationStrategyConfig countConfig = MessageCountRotationStrategyConfig.create(elasticsearchConfiguration.getMaxDocsPerIndex());
        clusterConfigService.write(countConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_docs_per_index", countConfig);
    }
    if (sizeBasedRotationStrategyConfig == null) {
        final SizeBasedRotationStrategyConfig sizeConfig = SizeBasedRotationStrategyConfig.create(elasticsearchConfiguration.getMaxSizePerIndex());
        clusterConfigService.write(sizeConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_size_per_index", sizeConfig);
    }
    if (timeBasedRotationStrategyConfig == null) {
        final TimeBasedRotationStrategyConfig timeConfig = TimeBasedRotationStrategyConfig.create(elasticsearchConfiguration.getMaxTimePerIndex(), elasticsearchConfiguration.getMaxWriteIndexAge());
        clusterConfigService.write(timeConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_time_per_index", timeConfig);
    }
    // All default retention strategy settings
    final ClosingRetentionStrategyConfig closingRetentionStrategyConfig = clusterConfigService.get(ClosingRetentionStrategyConfig.class);
    final DeletionRetentionStrategyConfig deletionRetentionStrategyConfig = clusterConfigService.get(DeletionRetentionStrategyConfig.class);
    if (closingRetentionStrategyConfig == null) {
        final ClosingRetentionStrategyConfig closingConfig = ClosingRetentionStrategyConfig.create(elasticsearchConfiguration.getMaxNumberOfIndices());
        clusterConfigService.write(closingConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_number_of_indices", closingConfig);
    }
    if (deletionRetentionStrategyConfig == null) {
        final DeletionRetentionStrategyConfig deletionConfig = DeletionRetentionStrategyConfig.create(elasticsearchConfiguration.getMaxNumberOfIndices());
        clusterConfigService.write(deletionConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_number_of_indices", deletionConfig);
    }
    // Selected rotation and retention strategies.
    final IndexManagementConfig indexManagementConfig = clusterConfigService.get(IndexManagementConfig.class);
    if (indexManagementConfig == null) {
        final Class<? extends RotationStrategy> rotationStrategyClass;
        switch(elasticsearchConfiguration.getRotationStrategy()) {
            case "size":
                rotationStrategyClass = SizeBasedRotationStrategy.class;
                break;
            case "time":
                rotationStrategyClass = TimeBasedRotationStrategy.class;
                break;
            case "count":
                rotationStrategyClass = MessageCountRotationStrategy.class;
                break;
            default:
                LOG.warn("Unknown retention strategy \"{}\"", elasticsearchConfiguration.getRotationStrategy());
                rotationStrategyClass = MessageCountRotationStrategy.class;
        }
        final Class<? extends RetentionStrategy> retentionStrategyClass;
        switch(elasticsearchConfiguration.getRetentionStrategy()) {
            case "close":
                retentionStrategyClass = ClosingRetentionStrategy.class;
                break;
            case "delete":
                retentionStrategyClass = DeletionRetentionStrategy.class;
                break;
            default:
                LOG.warn("Unknown retention strategy \"{}\"", elasticsearchConfiguration.getRetentionStrategy());
                retentionStrategyClass = DeletionRetentionStrategy.class;
        }
        final IndexManagementConfig config = IndexManagementConfig.create(rotationStrategyClass.getCanonicalName(), retentionStrategyClass.getCanonicalName());
        clusterConfigService.write(config);
        LOG.info("Migrated \"{}\" and \"{}\" setting: {}", "rotation_strategy", "retention_strategy", config);
    }
    final SearchesClusterConfig searchesClusterConfig = clusterConfigService.get(SearchesClusterConfig.class);
    if (searchesClusterConfig == null) {
        final SearchesClusterConfig config = SearchesClusterConfig.createDefault();
        LOG.info("Creating searches cluster config: {}", config);
        clusterConfigService.write(config);
    }
}

Example 7 with RetentionStrategy

use of org.graylog2.plugin.indexer.retention.RetentionStrategy in project graylog2-server by Graylog2.

the class V20161124104700_AddRetentionRotationAndDefaultFlagToIndexSetMigrationTest method upgradeWithWrongRetentionPrefix.

@Test
public void upgradeWithWrongRetentionPrefix() throws Exception {
    final String rotationStrategyClass = MessageCountRotationStrategy.class.getCanonicalName();
    final String retentionStrategyClass = "bar";
    final RotationStrategyConfig rotationStrategy = MessageCountRotationStrategyConfig.createDefault();
    final RetentionStrategyConfig retentionStrategy = DeletionRetentionStrategyConfig.createDefault();
    final IndexSetConfig config1 = IndexSetConfig.builder().id("id-1").title("title-1").indexPrefix("prefix-1").shards(1).replicas(0).rotationStrategy(rotationStrategy).retentionStrategy(retentionStrategy).creationDate(ZonedDateTime.of(2016, 10, 12, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-1").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    final IndexSetConfig config2 = IndexSetConfig.builder().id("id-2").title("title-2").indexPrefix("prefix-2").shards(1).replicas(0).rotationStrategy(rotationStrategy).retentionStrategy(retentionStrategy).creationDate(ZonedDateTime.of(2016, 10, 13, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-2").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    when(clusterConfigService.get(IndexManagementConfig.class)).thenReturn(IndexManagementConfig.create(rotationStrategyClass, retentionStrategyClass));
    when(indexSetService.findAll()).thenReturn(Lists.newArrayList(config1, config2));
    expectedException.expect(IllegalStateException.class);
    expectedException.expectMessage("retention strategy config type <");
    migration.upgrade();
    verify(indexSetService, never()).save(any(IndexSetConfig.class));
    verify(clusterConfigService, never()).write(V20161124104700_AddRetentionRotationAndDefaultFlagToIndexSetMigration.MigrationCompleted.class);
}

Example 8 with RetentionStrategy

use of org.graylog2.plugin.indexer.retention.RetentionStrategy in project graylog2-server by Graylog2.

the class V20161124104700_AddRetentionRotationAndDefaultFlagToIndexSetMigrationTest method upgradeWithWrongRotationPrefix.

@Test
public void upgradeWithWrongRotationPrefix() throws Exception {
    final String rotationStrategyClass = "foo";
    final String retentionStrategyClass = DeletionRetentionStrategy.class.getCanonicalName();
    final RotationStrategyConfig rotationStrategy = MessageCountRotationStrategyConfig.createDefault();
    final RetentionStrategyConfig retentionStrategy = DeletionRetentionStrategyConfig.createDefault();
    final IndexSetConfig config1 = IndexSetConfig.builder().id("id-1").title("title-1").indexPrefix("prefix-1").shards(1).replicas(0).rotationStrategy(rotationStrategy).retentionStrategy(retentionStrategy).creationDate(ZonedDateTime.of(2016, 10, 12, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-1").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    final IndexSetConfig config2 = IndexSetConfig.builder().id("id-2").title("title-2").indexPrefix("prefix-2").shards(1).replicas(0).rotationStrategy(rotationStrategy).retentionStrategy(retentionStrategy).creationDate(ZonedDateTime.of(2016, 10, 13, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-2").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    when(clusterConfigService.get(IndexManagementConfig.class)).thenReturn(IndexManagementConfig.create(rotationStrategyClass, retentionStrategyClass));
    when(indexSetService.findAll()).thenReturn(Lists.newArrayList(config1, config2));
    expectedException.expect(IllegalStateException.class);
    expectedException.expectMessage("rotation strategy config type <");
    migration.upgrade();
    verify(indexSetService, never()).save(any(IndexSetConfig.class));
    verify(clusterConfigService, never()).write(V20161124104700_AddRetentionRotationAndDefaultFlagToIndexSetMigration.MigrationCompleted.class);
}

Example 9 with RetentionStrategy

use of org.graylog2.plugin.indexer.retention.RetentionStrategy in project graylog2-server by Graylog2.

the class SearchesIT method testCountWithFilter.

@Test
public void testCountWithFilter() throws Exception {
    importFixture("org/graylog2/indexer/searches/SearchesIT.json");
    final IndexSetConfig indexSetConfig = IndexSetConfig.builder().id("id").title("title").indexPrefix("prefix").shards(1).replicas(0).rotationStrategy(MessageCountRotationStrategyConfig.createDefault()).retentionStrategyClass(DeletionRetentionStrategy.class.getCanonicalName()).retentionStrategy(DeletionRetentionStrategyConfig.createDefault()).creationDate(ZonedDateTime.of(2017, 5, 24, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    final IndexSet indexSet = new TestIndexSet(indexSetConfig);
    final Stream stream = new FakeStream("test") {

        @Override
        public IndexSet getIndexSet() {
            return indexSet;
        }
    };
    when(streamService.load(STREAM_ID)).thenReturn(stream);
    CountResult result = searches.count("*", AbsoluteRange.create(new DateTime(2015, 1, 1, 0, 0, DateTimeZone.UTC), new DateTime(2015, 1, 2, 0, 0, DateTimeZone.UTC)), "streams:" + STREAM_ID);
    assertThat(result.count()).isEqualTo(5L);
}

Example 10 with RetentionStrategy

use of org.graylog2.plugin.indexer.retention.RetentionStrategy in project graylog2-server by Graylog2.

the class RetentionStrategyResource method config.

@GET
@Path("config")
@Timed
@ApiOperation(value = "Configuration of the current retention strategy", notes = "This resource returns the configuration of the currently used retention strategy.")
public RetentionStrategySummary config() {
    final IndexManagementConfig indexManagementConfig = clusterConfigService.get(IndexManagementConfig.class);
    if (indexManagementConfig == null) {
        throw new InternalServerErrorException("Couldn't retrieve index management configuration");
    }
    final String strategyName = indexManagementConfig.retentionStrategy();
    final Provider<RetentionStrategy> provider = retentionStrategies.get(strategyName);
    if (provider == null) {
        LOG.error("Couldn't retrieve retention strategy provider for {}. Returning no-op strategy config.", strategyName);
        return RetentionStrategySummary.create(NoopRetentionStrategy.class.getCanonicalName(), NoopRetentionStrategyConfig.createDefault());
    }
    final RetentionStrategy retentionStrategy = provider.get();
    @SuppressWarnings("unchecked") final Class<RetentionStrategyConfig> configClass = (Class<RetentionStrategyConfig>) retentionStrategy.configurationClass();
    final RetentionStrategyConfig config = clusterConfigService.get(configClass);
    return RetentionStrategySummary.create(strategyName, config);
}