Examples with RetentionStrategy org.graylog2.plugin.indexer.retention.RetentionStrategy used on opensource projects

Example 1 with RetentionStrategy

use of org.graylog2.plugin.indexer.retention.RetentionStrategy in project graylog2-server by Graylog2.

the class CountsTest method setUp.

@Before
public void setUp() throws Exception {
    final Map<String, Object> settings = ImmutableMap.of("number_of_shards", 1, "index.number_of_replicas", 0);
    final CreateIndexResponse createIndexResponse1 = client.admin().indices().prepareCreate(INDEX_NAME_1).setSettings(settings).setTimeout(TimeValue.timeValueSeconds(10L)).execute().get();
    assumeTrue(createIndexResponse1.isAcknowledged());
    final CreateIndexResponse createIndexResponse2 = client.admin().indices().prepareCreate(INDEX_NAME_2).setSettings(settings).setTimeout(TimeValue.timeValueSeconds(10L)).execute().get();
    assumeTrue(createIndexResponse2.isAcknowledged());
    final ClusterHealthResponse clusterHealthResponse1 = client.admin().cluster().prepareHealth(INDEX_NAME_1).setWaitForGreenStatus().execute().get();
    assumeTrue(clusterHealthResponse1.getStatus() == ClusterHealthStatus.GREEN);
    final ClusterHealthResponse clusterHealthResponse2 = client.admin().cluster().prepareHealth(INDEX_NAME_2).setWaitForGreenStatus().execute().get();
    assumeTrue(clusterHealthResponse2.getStatus() == ClusterHealthStatus.GREEN);
    counts = new Counts(client, indexSetRegistry);
    indexSetConfig1 = IndexSetConfig.builder().id("id-1").title("title-1").indexPrefix("index_set_1_counts_test").shards(1).replicas(0).rotationStrategyClass(MessageCountRotationStrategy.class.getCanonicalName()).rotationStrategy(MessageCountRotationStrategyConfig.createDefault()).retentionStrategyClass(DeletionRetentionStrategy.class.getCanonicalName()).retentionStrategy(DeletionRetentionStrategyConfig.createDefault()).creationDate(ZonedDateTime.of(2016, 10, 12, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-1").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    indexSetConfig2 = IndexSetConfig.builder().id("id-2").title("title-2").indexPrefix("index_set_2_counts_test").shards(1).replicas(0).rotationStrategyClass(MessageCountRotationStrategy.class.getCanonicalName()).rotationStrategy(MessageCountRotationStrategyConfig.createDefault()).retentionStrategyClass(DeletionRetentionStrategy.class.getCanonicalName()).retentionStrategy(DeletionRetentionStrategyConfig.createDefault()).creationDate(ZonedDateTime.of(2016, 10, 13, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-2").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    when(indexSetRegistry.getManagedIndices()).thenReturn(new String[] { INDEX_NAME_1, INDEX_NAME_2 });
    when(indexSetRegistry.get(indexSetConfig1.id())).thenReturn(Optional.of(indexSet1));
    when(indexSetRegistry.get(indexSetConfig2.id())).thenReturn(Optional.of(indexSet2));
    when(indexSet1.getManagedIndices()).thenReturn(new String[] { INDEX_NAME_1 });
    when(indexSet2.getManagedIndices()).thenReturn(new String[] { INDEX_NAME_2 });
}

Example 2 with RetentionStrategy

use of org.graylog2.plugin.indexer.retention.RetentionStrategy in project graylog2-server by Graylog2.

the class IndexRetentionThread method doRun.

@Override
public void doRun() {
    if (!cluster.isConnected() || !cluster.isHealthy()) {
        LOG.info("Elasticsearch cluster not available, skipping index retention checks.");
        return;
    }
    for (final IndexSet indexSet : indexSetRegistry) {
        if (!indexSet.getConfig().isWritable()) {
            LOG.debug("Skipping non-writable index set <{}> ({})", indexSet.getConfig().id(), indexSet.getConfig().title());
            continue;
        }
        final IndexSetConfig config = indexSet.getConfig();
        final Provider<RetentionStrategy> retentionStrategyProvider = retentionStrategyMap.get(config.retentionStrategyClass());
        if (retentionStrategyProvider == null) {
            LOG.warn("Retention strategy \"{}\" not found, not running index retention!", config.retentionStrategyClass());
            retentionProblemNotification("Index Retention Problem!", "Index retention strategy " + config.retentionStrategyClass() + " not found! Please fix your index retention configuration!");
            continue;
        }
        retentionStrategyProvider.get().retain(indexSet);
    }
}

Example 3 with RetentionStrategy

use of org.graylog2.plugin.indexer.retention.RetentionStrategy in project graylog2-server by Graylog2.

the class V20190705071400_AddEventIndexSetsMigration method setupEventsIndexSet.

private IndexSet setupEventsIndexSet(String indexSetTitle, String indexSetDescription, String indexPrefix) {
    final Optional<IndexSetConfig> optionalIndexSetConfig = getEventsIndexSetConfig(indexPrefix);
    if (optionalIndexSetConfig.isPresent()) {
        return mongoIndexSetFactory.create(optionalIndexSetConfig.get());
    }
    final IndexSetConfig indexSetConfig = IndexSetConfig.builder().title(indexSetTitle).description(indexSetDescription).indexTemplateType(EVENT_TEMPLATE_TYPE).isWritable(true).isRegular(false).indexPrefix(indexPrefix).shards(elasticsearchConfiguration.getShards()).replicas(elasticsearchConfiguration.getReplicas()).rotationStrategyClass(TimeBasedRotationStrategy.class.getCanonicalName()).rotationStrategy(TimeBasedRotationStrategyConfig.create(Period.months(1), null)).retentionStrategyClass(DeletionRetentionStrategy.class.getCanonicalName()).retentionStrategy(DeletionRetentionStrategyConfig.create(12)).creationDate(ZonedDateTime.now(ZoneOffset.UTC)).indexAnalyzer(elasticsearchConfiguration.getAnalyzer()).indexTemplateName(indexPrefix + "-template").indexOptimizationMaxNumSegments(elasticsearchConfiguration.getIndexOptimizationMaxNumSegments()).indexOptimizationDisabled(elasticsearchConfiguration.isDisableIndexOptimization()).fieldTypeRefreshInterval(Duration.standardMinutes(1)).build();
    try {
        final Optional<IndexSetValidator.Violation> violation = indexSetValidator.validate(indexSetConfig);
        if (violation.isPresent()) {
            throw new RuntimeException(violation.get().message());
        }
        final IndexSetConfig savedIndexSet = indexSetService.save(indexSetConfig);
        LOG.info("Successfully created events index-set <{}/{}>", savedIndexSet.id(), savedIndexSet.title());
        return mongoIndexSetFactory.create(savedIndexSet);
    } catch (DuplicateKeyException e) {
        LOG.error("Couldn't create index-set <{}/{}>", indexSetTitle, indexPrefix);
        throw new RuntimeException(e.getMessage());
    }
}

Example 4 with RetentionStrategy

use of org.graylog2.plugin.indexer.retention.RetentionStrategy in project graylog2-server by Graylog2.

the class V20161116172100_DefaultIndexSetMigration method upgrade.

@Override
public void upgrade() {
    // Do not run again if the migration marker can be found in the database.
    if (clusterConfigService.get(DefaultIndexSetCreated.class) != null) {
        return;
    }
    final IndexManagementConfig indexManagementConfig = clusterConfigService.get(IndexManagementConfig.class);
    checkState(indexManagementConfig != null, "Couldn't find index management configuration");
    final IndexSetConfig config = IndexSetConfig.builder().title("Default index set").description("The Graylog default index set").isRegular(true).indexPrefix(elasticsearchConfiguration.getIndexPrefix()).shards(elasticsearchConfiguration.getShards()).replicas(elasticsearchConfiguration.getReplicas()).rotationStrategy(getRotationStrategyConfig(indexManagementConfig)).retentionStrategy(getRetentionStrategyConfig(indexManagementConfig)).creationDate(ZonedDateTime.now(ZoneOffset.UTC)).indexAnalyzer(elasticsearchConfiguration.getAnalyzer()).indexTemplateName(elasticsearchConfiguration.getTemplateName()).indexOptimizationMaxNumSegments(elasticsearchConfiguration.getIndexOptimizationMaxNumSegments()).indexOptimizationDisabled(elasticsearchConfiguration.isDisableIndexOptimization()).build();
    final IndexSetConfig savedConfig = indexSetService.save(config);
    clusterConfigService.write(DefaultIndexSetConfig.create(savedConfig.id()));
    clusterConfigService.write(DefaultIndexSetCreated.create());
    LOG.debug("Successfully created default index set: {}", savedConfig);
}

Example 5 with RetentionStrategy

use of org.graylog2.plugin.indexer.retention.RetentionStrategy in project graylog2-server by Graylog2.

the class RetentionStrategyResource method getRetentionStrategyDescription.

private RetentionStrategyDescription getRetentionStrategyDescription(@ApiParam(name = "strategy", value = "The name of the retention strategy", required = true) @PathParam("strategy") @NotEmpty String strategyName) {
    final Provider<RetentionStrategy> provider = retentionStrategies.get(strategyName);
    if (provider == null) {
        throw new NotFoundException("Couldn't find retention strategy for given type " + strategyName);
    }
    final RetentionStrategy retentionStrategy = provider.get();
    final RetentionStrategyConfig defaultConfig = retentionStrategy.defaultConfiguration();
    final SchemaFactoryWrapper visitor = new SchemaFactoryWrapper();
    try {
        objectMapper.acceptJsonFormatVisitor(objectMapper.constructType(retentionStrategy.configurationClass()), visitor);
    } catch (JsonMappingException e) {
        throw new InternalServerErrorException("Couldn't generate JSON schema for retention strategy " + strategyName, e);
    }
    return RetentionStrategyDescription.create(strategyName, defaultConfig, visitor.finalSchema());
}