Examples with Cluster org.graylog2.indexer.cluster.Cluster used on opensource projects

Example 1 with Cluster

use of org.graylog2.indexer.cluster.Cluster in project graylog2-server by Graylog2.

the class V20151210140600_ElasticsearchConfigMigration method upgrade.

// Migrate old Elasticsearch config settings to new ClusterConfig based ones.
@Override
public void upgrade() {
    // All default rotation strategy settings.
    final MessageCountRotationStrategyConfig messageCountRotationStrategyConfig = clusterConfigService.get(MessageCountRotationStrategyConfig.class);
    final SizeBasedRotationStrategyConfig sizeBasedRotationStrategyConfig = clusterConfigService.get(SizeBasedRotationStrategyConfig.class);
    final TimeBasedRotationStrategyConfig timeBasedRotationStrategyConfig = clusterConfigService.get(TimeBasedRotationStrategyConfig.class);
    if (messageCountRotationStrategyConfig == null) {
        final MessageCountRotationStrategyConfig countConfig = MessageCountRotationStrategyConfig.create(elasticsearchConfiguration.getMaxDocsPerIndex());
        clusterConfigService.write(countConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_docs_per_index", countConfig);
    }
    if (sizeBasedRotationStrategyConfig == null) {
        final SizeBasedRotationStrategyConfig sizeConfig = SizeBasedRotationStrategyConfig.create(elasticsearchConfiguration.getMaxSizePerIndex());
        clusterConfigService.write(sizeConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_size_per_index", sizeConfig);
    }
    if (timeBasedRotationStrategyConfig == null) {
        final TimeBasedRotationStrategyConfig timeConfig = TimeBasedRotationStrategyConfig.create(elasticsearchConfiguration.getMaxTimePerIndex());
        clusterConfigService.write(timeConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_time_per_index", timeConfig);
    }
    // All default retention strategy settings
    final ClosingRetentionStrategyConfig closingRetentionStrategyConfig = clusterConfigService.get(ClosingRetentionStrategyConfig.class);
    final DeletionRetentionStrategyConfig deletionRetentionStrategyConfig = clusterConfigService.get(DeletionRetentionStrategyConfig.class);
    if (closingRetentionStrategyConfig == null) {
        final ClosingRetentionStrategyConfig closingConfig = ClosingRetentionStrategyConfig.create(elasticsearchConfiguration.getMaxNumberOfIndices());
        clusterConfigService.write(closingConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_number_of_indices", closingConfig);
    }
    if (deletionRetentionStrategyConfig == null) {
        final DeletionRetentionStrategyConfig deletionConfig = DeletionRetentionStrategyConfig.create(elasticsearchConfiguration.getMaxNumberOfIndices());
        clusterConfigService.write(deletionConfig);
        LOG.info("Migrated \"{}\" setting: {}", "elasticsearch_max_number_of_indices", deletionConfig);
    }
    // Selected rotation and retention strategies.
    final IndexManagementConfig indexManagementConfig = clusterConfigService.get(IndexManagementConfig.class);
    if (indexManagementConfig == null) {
        final Class<? extends RotationStrategy> rotationStrategyClass;
        switch(elasticsearchConfiguration.getRotationStrategy()) {
            case "size":
                rotationStrategyClass = SizeBasedRotationStrategy.class;
                break;
            case "time":
                rotationStrategyClass = TimeBasedRotationStrategy.class;
                break;
            case "count":
                rotationStrategyClass = MessageCountRotationStrategy.class;
                break;
            default:
                LOG.warn("Unknown retention strategy \"{}\"", elasticsearchConfiguration.getRotationStrategy());
                rotationStrategyClass = MessageCountRotationStrategy.class;
        }
        final Class<? extends RetentionStrategy> retentionStrategyClass;
        switch(elasticsearchConfiguration.getRetentionStrategy()) {
            case "close":
                retentionStrategyClass = ClosingRetentionStrategy.class;
                break;
            case "delete":
                retentionStrategyClass = DeletionRetentionStrategy.class;
                break;
            default:
                LOG.warn("Unknown retention strategy \"{}\"", elasticsearchConfiguration.getRetentionStrategy());
                retentionStrategyClass = DeletionRetentionStrategy.class;
        }
        final IndexManagementConfig config = IndexManagementConfig.create(rotationStrategyClass.getCanonicalName(), retentionStrategyClass.getCanonicalName());
        clusterConfigService.write(config);
        LOG.info("Migrated \"{}\" and \"{}\" setting: {}", "rotation_strategy", "retention_strategy", config);
    }
    final SearchesClusterConfig searchesClusterConfig = clusterConfigService.get(SearchesClusterConfig.class);
    if (searchesClusterConfig == null) {
        final SearchesClusterConfig config = SearchesClusterConfig.createDefault();
        LOG.info("Creating searches cluster config: {}", config);
        clusterConfigService.write(config);
    }
}

Example 2 with Cluster

use of org.graylog2.indexer.cluster.Cluster in project graylog2-server by Graylog2.

the class V20161116172100_DefaultIndexSetMigration method upgrade.

@Override
public void upgrade() {
    // Do not run again if the migration marker can be found in the database.
    if (clusterConfigService.get(DefaultIndexSetCreated.class) != null) {
        return;
    }
    final IndexManagementConfig indexManagementConfig = clusterConfigService.get(IndexManagementConfig.class);
    checkState(indexManagementConfig != null, "Couldn't find index management configuration");
    final IndexSetConfig config = IndexSetConfig.builder().title("Default index set").description("The Graylog default index set").indexPrefix(elasticsearchConfiguration.getIndexPrefix()).shards(elasticsearchConfiguration.getShards()).replicas(elasticsearchConfiguration.getReplicas()).rotationStrategy(getRotationStrategyConfig(indexManagementConfig)).retentionStrategy(getRetentionStrategyConfig(indexManagementConfig)).creationDate(ZonedDateTime.now(ZoneOffset.UTC)).indexAnalyzer(elasticsearchConfiguration.getAnalyzer()).indexTemplateName(elasticsearchConfiguration.getTemplateName()).indexOptimizationMaxNumSegments(elasticsearchConfiguration.getIndexOptimizationMaxNumSegments()).indexOptimizationDisabled(elasticsearchConfiguration.isDisableIndexOptimization()).build();
    final IndexSetConfig savedConfig = indexSetService.save(config);
    clusterConfigService.write(DefaultIndexSetConfig.create(savedConfig.id()));
    clusterConfigService.write(DefaultIndexSetCreated.create());
    // Publish event to cluster event bus so the stream router will reload.
    clusterEventBus.post(IndexSetCreatedEvent.create(savedConfig));
    LOG.debug("Successfully created default index set: {}", savedConfig);
}

Example 3 with Cluster

use of org.graylog2.indexer.cluster.Cluster in project graylog2-server by Graylog2.

the class V20161216123500_DefaultIndexSetMigration method migrateIndexSet.

private void migrateIndexSet(IndexSetConfig indexSetConfig, String templateName) {
    final String analyzer = elasticsearchConfiguration.getAnalyzer();
    final IndexSetConfig updatedConfig = indexSetConfig.toBuilder().indexAnalyzer(analyzer).indexTemplateName(templateName).indexOptimizationMaxNumSegments(elasticsearchConfiguration.getIndexOptimizationMaxNumSegments()).indexOptimizationDisabled(elasticsearchConfiguration.isDisableIndexOptimization()).build();
    final IndexSetConfig savedConfig = indexSetService.save(updatedConfig);
    // Publish event to cluster event bus so the stream router will reload.
    clusterEventBus.post(IndexSetCreatedEvent.create(savedConfig));
    LOG.debug("Successfully updated index set: {}", savedConfig);
}

Example 4 with Cluster

use of org.graylog2.indexer.cluster.Cluster in project graylog2-server by Graylog2.

the class IndexRetentionThread method doRun.

@Override
public void doRun() {
    if (!cluster.isConnected() || !cluster.isHealthy()) {
        LOG.info("Elasticsearch cluster not available, skipping index retention checks.");
        return;
    }
    for (final IndexSet indexSet : indexSetRegistry) {
        if (!indexSet.getConfig().isWritable()) {
            LOG.debug("Skipping non-writable index set <{}> ({})", indexSet.getConfig().id(), indexSet.getConfig().title());
            continue;
        }
        final IndexSetConfig config = indexSet.getConfig();
        final Provider<RetentionStrategy> retentionStrategyProvider = retentionStrategyMap.get(config.retentionStrategyClass());
        if (retentionStrategyProvider == null) {
            LOG.warn("Retention strategy \"{}\" not found, not running index retention!", config.retentionStrategyClass());
            retentionProblemNotification("Index Retention Problem!", "Index retention strategy " + config.retentionStrategyClass() + " not found! Please fix your index retention configuration!");
            continue;
        }
        retentionStrategyProvider.get().retain(indexSet);
    }
}

Example 5 with Cluster

use of org.graylog2.indexer.cluster.Cluster in project graylog2-server by Graylog2.

the class V20161215163900_MoveIndexSetDefaultConfigTest method upgrade.

@Test
public void upgrade() throws Exception {
    final long count = collection.count();
    migration.upgrade();
    final MigrationCompleted migrationCompleted = clusterConfigService.get(MigrationCompleted.class);
    assertThat(collection.count()).withFailMessage("No document should be deleted by the migration!").isEqualTo(count);
    assertThat(collection.count(Filters.exists("default"))).withFailMessage("The migration should have deleted the \"default\" field from the documents!").isEqualTo(0L);
    assertThat(clusterConfigService.get(DefaultIndexSetConfig.class)).withFailMessage("The DefaultIndexSetConfig should have been written to cluster config!").isNotNull();
    assertThat(clusterConfigService.get(DefaultIndexSetConfig.class).defaultIndexSetId()).isEqualTo("57f3d721a43c2d59cb750001");
    assertThat(migrationCompleted).isNotNull();
    assertThat(migrationCompleted.indexSetIds()).containsExactly("57f3d721a43c2d59cb750001", "57f3d721a43c2d59cb750003");
}