Examples with TimeBasedRotationStrategyConfig org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig used on opensource projects

Example 1 with TimeBasedRotationStrategyConfig

use of org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig in project graylog2-server by Graylog2.

the class TimeBasedRotationStrategy method shouldRotate.

@Nullable
@Override
protected Result shouldRotate(String index, IndexSet indexSet) {
    final IndexSetConfig indexSetConfig = requireNonNull(indexSet.getConfig(), "Index set configuration must not be null");
    final String indexSetId = indexSetConfig.id();
    checkState(!isNullOrEmpty(index), "Index name must not be null or empty");
    checkState(!isNullOrEmpty(indexSetId), "Index set ID must not be null or empty");
    checkState(indexSetConfig.rotationStrategy() instanceof TimeBasedRotationStrategyConfig, "Invalid rotation strategy config <" + indexSetConfig.rotationStrategy().getClass().getCanonicalName() + "> for index set <" + indexSetId + ">");
    final TimeBasedRotationStrategyConfig config = (TimeBasedRotationStrategyConfig) indexSetConfig.rotationStrategy();
    Period rotationPeriod = config.rotationPeriod();
    Period maxPeriod = elasticsearchConfiguration.getMaxWriteIndexAge();
    boolean overriding = false;
    if (maxPeriod != null && isLonger(rotationPeriod, maxPeriod)) {
        log.debug("Max rotation limit {} overrides configured period {}", maxPeriod, rotationPeriod);
        rotationPeriod = maxPeriod;
        overriding = true;
    }
    final Period normalizedPeriod = rotationPeriod.normalizedStandard();
    // when first started, we might not know the last rotation time, look up the creation time of the index instead.
    if (!lastRotation.containsKey(indexSetId)) {
        indices.indexCreationDate(index).ifPresent(creationDate -> {
            final DateTime currentAnchor = determineRotationPeriodAnchor(creationDate, normalizedPeriod);
            anchor.put(indexSetId, currentAnchor);
            lastRotation.put(indexSetId, creationDate);
        });
        // still not able to figure out the last rotation time, we'll rotate forcibly
        if (!lastRotation.containsKey(indexSetId)) {
            return new SimpleResult(true, "No known previous rotation time, forcing index rotation now.");
        }
    }
    final DateTime now = Tools.nowUTC();
    final DateTime currentAnchor = anchor.get(indexSetId);
    final DateTime nextRotation = currentAnchor.plus(normalizedPeriod);
    if (nextRotation.isAfter(now)) {
        final String message = new MessageFormat("Next rotation at {0} {1}", Locale.ENGLISH).format(new Object[] { nextRotation, overriding ? "(elasticsearch_max_write_index_age overrides configured period)" : "" });
        return new SimpleResult(false, message);
    }
    // determine new anchor (push it to within less then one period before now) in case we missed one or more periods
    DateTime tmpAnchor;
    int multiplicator = 0;
    do {
        tmpAnchor = currentAnchor.withPeriodAdded(normalizedPeriod, ++multiplicator);
    } while (tmpAnchor.isBefore(now));
    final DateTime nextAnchor = currentAnchor.withPeriodAdded(normalizedPeriod, multiplicator - 1);
    anchor.put(indexSetId, nextAnchor);
    lastRotation.put(indexSetId, now);
    final String message = new MessageFormat("Rotation period {0} elapsed, next rotation at {1} {2}", Locale.ENGLISH).format(new Object[] { now, nextAnchor, overriding ? "(elasticsearch_max_write_index_age overrides configured period)" : "" });
    return new SimpleResult(true, message);
}

Example 2 with TimeBasedRotationStrategyConfig

use of org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig in project graylog2-server by Graylog2.

the class V20151210140600_ElasticsearchConfigMigration method upgrade.

// Migrate old Elasticsearch config settings to new ClusterConfig based ones.
@Override
public void upgrade() {
    // All default rotation strategy settings.
    final MessageCountRotationStrategyConfig messageCountRotationStrategyConfig = clusterConfigService.get(MessageCountRotationStrategyConfig.class);
    final SizeBasedRotationStrategyConfig sizeBasedRotationStrategyConfig = clusterConfigService.get(SizeBasedRotationStrategyConfig.class);
    final TimeBasedRotationStrategyConfig timeBasedRotationStrategyConfig = clusterConfigService.get(TimeBasedRotationStrategyConfig.class);
    if (messageCountRotationStrategyConfig == null) {
        final MessageCountRotationStrategyConfig countConfig = MessageCountRotationStrategyConfig.create(elasticsearchConfiguration.getMaxDocsPerIndex());
        clusterConfigService.write(countConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_docs_per_index", countConfig);
    }
    if (sizeBasedRotationStrategyConfig == null) {
        final SizeBasedRotationStrategyConfig sizeConfig = SizeBasedRotationStrategyConfig.create(elasticsearchConfiguration.getMaxSizePerIndex());
        clusterConfigService.write(sizeConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_size_per_index", sizeConfig);
    }
    if (timeBasedRotationStrategyConfig == null) {
        final TimeBasedRotationStrategyConfig timeConfig = TimeBasedRotationStrategyConfig.create(elasticsearchConfiguration.getMaxTimePerIndex(), elasticsearchConfiguration.getMaxWriteIndexAge());
        clusterConfigService.write(timeConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_time_per_index", timeConfig);
    }
    // All default retention strategy settings
    final ClosingRetentionStrategyConfig closingRetentionStrategyConfig = clusterConfigService.get(ClosingRetentionStrategyConfig.class);
    final DeletionRetentionStrategyConfig deletionRetentionStrategyConfig = clusterConfigService.get(DeletionRetentionStrategyConfig.class);
    if (closingRetentionStrategyConfig == null) {
        final ClosingRetentionStrategyConfig closingConfig = ClosingRetentionStrategyConfig.create(elasticsearchConfiguration.getMaxNumberOfIndices());
        clusterConfigService.write(closingConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_number_of_indices", closingConfig);
    }
    if (deletionRetentionStrategyConfig == null) {
        final DeletionRetentionStrategyConfig deletionConfig = DeletionRetentionStrategyConfig.create(elasticsearchConfiguration.getMaxNumberOfIndices());
        clusterConfigService.write(deletionConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_number_of_indices", deletionConfig);
    }
    // Selected rotation and retention strategies.
    final IndexManagementConfig indexManagementConfig = clusterConfigService.get(IndexManagementConfig.class);
    if (indexManagementConfig == null) {
        final Class<? extends RotationStrategy> rotationStrategyClass;
        switch(elasticsearchConfiguration.getRotationStrategy()) {
            case "size":
                rotationStrategyClass = SizeBasedRotationStrategy.class;
                break;
            case "time":
                rotationStrategyClass = TimeBasedRotationStrategy.class;
                break;
            case "count":
                rotationStrategyClass = MessageCountRotationStrategy.class;
                break;
            default:
                LOG.warn("Unknown retention strategy \"{}\"", elasticsearchConfiguration.getRotationStrategy());
                rotationStrategyClass = MessageCountRotationStrategy.class;
        }
        final Class<? extends RetentionStrategy> retentionStrategyClass;
        switch(elasticsearchConfiguration.getRetentionStrategy()) {
            case "close":
                retentionStrategyClass = ClosingRetentionStrategy.class;
                break;
            case "delete":
                retentionStrategyClass = DeletionRetentionStrategy.class;
                break;
            default:
                LOG.warn("Unknown retention strategy \"{}\"", elasticsearchConfiguration.getRetentionStrategy());
                retentionStrategyClass = DeletionRetentionStrategy.class;
        }
        final IndexManagementConfig config = IndexManagementConfig.create(rotationStrategyClass.getCanonicalName(), retentionStrategyClass.getCanonicalName());
        clusterConfigService.write(config);
        LOG.info("Migrated \"{}\" and \"{}\" setting: {}", "rotation_strategy", "retention_strategy", config);
    }
    final SearchesClusterConfig searchesClusterConfig = clusterConfigService.get(SearchesClusterConfig.class);
    if (searchesClusterConfig == null) {
        final SearchesClusterConfig config = SearchesClusterConfig.createDefault();
        LOG.info("Creating searches cluster config: {}", config);
        clusterConfigService.write(config);
    }
}

Example 3 with TimeBasedRotationStrategyConfig

use of org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig in project graylog2-server by Graylog2.

the class TimeBasedRotationStrategyConfigTest method testDeserialization.

@Test
public void testDeserialization() throws IOException {
    final ObjectMapper objectMapper = new ObjectMapperProvider().get();
    final String json = "{ \"type\": \"org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig\", \"rotation_period\": \"P1D\", \"max_rotation_period\": \"P99D\" }";
    final RotationStrategyConfig config = objectMapper.readValue(json, RotationStrategyConfig.class);
    assertThat(config).isInstanceOf(TimeBasedRotationStrategyConfig.class);
    assertThat(((TimeBasedRotationStrategyConfig) config).rotationPeriod()).isEqualTo(Period.days(1));
    assertThat(((TimeBasedRotationStrategyConfig) config).maxRotationPeriod()).isEqualTo(Period.days(99));
}

Example 4 with TimeBasedRotationStrategyConfig

use of org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig in project graylog2-server by Graylog2.

the class TimeBasedRotationStrategyConfigTest method testDeserializationWithMissingProperty.

@Test
public void testDeserializationWithMissingProperty() throws IOException {
    final ObjectMapper objectMapper = new ObjectMapperProvider().get();
    final String json = "{ \"type\": \"org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategyConfig\", \"rotation_period\": \"P1D\"}";
    final RotationStrategyConfig config = objectMapper.readValue(json, RotationStrategyConfig.class);
    assertThat(config).isInstanceOf(TimeBasedRotationStrategyConfig.class);
    assertThat(((TimeBasedRotationStrategyConfig) config).rotationPeriod()).isEqualTo(Period.days(1));
    assertNull(((TimeBasedRotationStrategyConfig) config).maxRotationPeriod());
}