Examples with IndexSetConfig org.graylog2.indexer.indexset.IndexSetConfig used on opensource projects

Example 36 with IndexSetConfig

use of org.graylog2.indexer.indexset.IndexSetConfig in project graylog2-server by Graylog2.

the class TimeBasedRotationStrategyTest method shouldRotateConcurrently.

@Test
public void shouldRotateConcurrently() throws Exception {
    final DateTime initialTime = new DateTime(2014, 1, 1, 1, 59, 59, 0, DateTimeZone.UTC);
    final Period period = Period.hours(1);
    final InstantMillisProvider clock = new InstantMillisProvider(initialTime);
    DateTimeUtils.setCurrentMillisProvider(clock);
    final IndexSet indexSet1 = mock(IndexSet.class);
    final IndexSet indexSet2 = mock(IndexSet.class);
    final IndexSetConfig indexSetConfig1 = mock(IndexSetConfig.class);
    final IndexSetConfig indexSetConfig2 = mock(IndexSetConfig.class);
    when(indexSetConfig1.id()).thenReturn("id1");
    when(indexSetConfig2.id()).thenReturn("id2");
    when(indexSet1.getConfig()).thenReturn(indexSetConfig1);
    when(indexSet2.getConfig()).thenReturn(indexSetConfig2);
    when(indexSetConfig1.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    when(indexSetConfig2.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    when(indices.indexCreationDate(anyString())).thenReturn(initialTime.minus(minutes(5)));
    // Should not rotate the initial index.
    when(indexSet1.getNewestIndex()).thenReturn("index1");
    rotationStrategy.rotate(indexSet1);
    verify(indexSet1, never()).cycle();
    reset(indexSet1);
    when(indexSet2.getNewestIndex()).thenReturn("index2");
    rotationStrategy.rotate(indexSet2);
    verify(indexSet2, never()).cycle();
    reset(indexSet2);
    clock.tick(seconds(2));
    // Crossed rotation period.
    when(indexSet1.getNewestIndex()).thenReturn("index1");
    when(indexSet1.getConfig()).thenReturn(indexSetConfig1);
    when(indexSetConfig1.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet1);
    verify(indexSet1, times(1)).cycle();
    reset(indexSet1);
    when(indexSet2.getNewestIndex()).thenReturn("index2");
    when(indexSet2.getConfig()).thenReturn(indexSetConfig2);
    when(indexSetConfig2.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet2);
    verify(indexSet2, times(1)).cycle();
    reset(indexSet2);
    clock.tick(seconds(2));
    // Did not cross rotation period.
    when(indexSet1.getNewestIndex()).thenReturn("index1");
    when(indexSet1.getConfig()).thenReturn(indexSetConfig1);
    when(indexSetConfig1.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet1);
    verify(indexSet1, never()).cycle();
    reset(indexSet1);
    when(indexSet2.getNewestIndex()).thenReturn("index2");
    when(indexSet2.getConfig()).thenReturn(indexSetConfig2);
    when(indexSetConfig2.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet2);
    verify(indexSet2, never()).cycle();
    reset(indexSet2);
}

Example 37 with IndexSetConfig

use of org.graylog2.indexer.indexset.IndexSetConfig in project graylog2-server by Graylog2.

the class V20161130141500_DefaultStreamRecalcIndexRanges method upgrade.

@Override
public void upgrade() {
    IndexSet indexSet;
    try {
        indexSet = indexSetRegistry.getDefault();
    } catch (IllegalStateException e) {
        // Try to find the default index set manually if the index set registry cannot find it.
        // This is needed if you run this migration with a 2.2.0-beta.2 state before commit 460cac6af.
        final IndexSetConfig indexSetConfig = indexSetService.findOne(DBQuery.is("default", true)).orElseThrow(() -> new IllegalStateException("No default index set configured! This is a bug!"));
        indexSet = indexSetFactory.create(indexSetConfig);
    }
    final IndexSet defaultIndexSet = indexSet;
    if (!cluster.isConnected()) {
        LOG.info("Cluster not connected yet, delaying migration until it is reachable.");
        while (true) {
            try {
                cluster.waitForConnectedAndDeflectorHealthy();
                break;
            } catch (InterruptedException | TimeoutException e) {
                LOG.warn("Interrupted or timed out waiting for Elasticsearch cluster, checking again.");
            }
        }
    }
    final Set<String> indexRangesWithoutStreams = indexRangeService.findAll().stream().filter(indexRange -> defaultIndexSet.isManagedIndex(indexRange.indexName())).filter(indexRange -> indexRange.streamIds() == null).map(IndexRange::indexName).collect(Collectors.toSet());
    if (indexRangesWithoutStreams.size() == 0) {
        // all ranges have a stream list, even if it is empty, nothing more to do
        return;
    }
    final String currentWriteTarget;
    try {
        currentWriteTarget = defaultIndexSet.getActiveWriteIndex();
    } catch (TooManyAliasesException e) {
        LOG.error("Multiple write targets found for write alias. Cannot continue to assign streams to older indices", e);
        return;
    }
    for (String indexName : defaultIndexSet.getManagedIndices()) {
        if (indexName.equals(currentWriteTarget)) {
            // do not recalculate for current write target
            continue;
        }
        if (!indexRangesWithoutStreams.contains(indexName)) {
            // already computed streams for this index
            continue;
        }
        LOG.info("Recalculating streams in index {}", indexName);
        final CreateNewSingleIndexRangeJob createNewSingleIndexRangeJob = rebuildIndexRangeJobFactory.create(indexSetRegistry.getAll(), indexName);
        createNewSingleIndexRangeJob.execute();
    }
}

Example 38 with IndexSetConfig

use of org.graylog2.indexer.indexset.IndexSetConfig in project graylog2-server by Graylog2.

the class IndexRangesMigrationPeriodical method doRun.

@Override
public void doRun() {
    final MongoIndexRangesMigrationComplete migrationComplete = clusterConfigService.get(MongoIndexRangesMigrationComplete.class);
    if (migrationComplete != null && migrationComplete.complete) {
        LOG.debug("Migration of index ranges (pre Graylog 1.2.2) already complete. Skipping migration process.");
        return;
    }
    while (!cluster.isConnected() || !cluster.isHealthy()) {
        Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
    }
    final Set<String> indexNames = ImmutableSet.copyOf(indexSetRegistry.getManagedIndices());
    // Migrate old MongoDB index ranges
    final SortedSet<IndexRange> mongoIndexRanges = legacyMongoIndexRangeService.findAll();
    for (IndexRange indexRange : mongoIndexRanges) {
        if (indexNames.contains(indexRange.indexName())) {
            LOG.info("Migrating index range from MongoDB: {}", indexRange);
            indexRangeService.save(indexRange);
        } else {
            LOG.info("Removing stale index range from MongoDB: {}", indexRange);
        }
        legacyMongoIndexRangeService.delete(indexRange.indexName());
    }
    // Check whether all index ranges have been migrated
    final int numberOfIndices = indexNames.size();
    final SortedSet<IndexRange> allIndexRanges = indexRangeService.findAll();
    final int numberOfIndexRanges = allIndexRanges.size();
    if (numberOfIndices > numberOfIndexRanges) {
        LOG.info("There are more indices ({}) than there are index ranges ({}). Notifying administrator.", numberOfIndices, numberOfIndexRanges);
        // remove all present index names so we can display the index sets that need manual fixing
        final Set<String> extraIndices = Sets.newHashSet(indexNames);
        allIndexRanges.forEach(indexRange -> extraIndices.remove(indexRange.indexName()));
        final Set<String> affectedIndexSetNames = extraIndices.stream().map(indexSetRegistry::getForIndex).filter(Optional::isPresent).map(Optional::get).map(IndexSet::getConfig).map(IndexSetConfig::title).collect(Collectors.toSet());
        final Notification notification = notificationService.buildNow().addSeverity(Notification.Severity.URGENT).addType(Notification.Type.INDEX_RANGES_RECALCULATION).addDetail("indices", numberOfIndices).addDetail("index_ranges", numberOfIndexRanges).addDetail("index_sets", affectedIndexSetNames.isEmpty() ? null : Joiner.on(", ").join(affectedIndexSetNames));
        notificationService.publishIfFirst(notification);
    }
    clusterConfigService.write(new MongoIndexRangesMigrationComplete(true));
}

Example 39 with IndexSetConfig

use of org.graylog2.indexer.indexset.IndexSetConfig in project graylog2-server by Graylog2.

the class IndexRotationThread method checkForRotation.

protected void checkForRotation(IndexSet indexSet) {
    final IndexSetConfig config = indexSet.getConfig();
    final Provider<RotationStrategy> rotationStrategyProvider = rotationStrategyMap.get(config.rotationStrategyClass());
    if (rotationStrategyProvider == null) {
        LOG.warn("Rotation strategy \"{}\" not found, not running index rotation!", config.rotationStrategyClass());
        rotationProblemNotification("Index Rotation Problem!", "Index rotation strategy " + config.rotationStrategyClass() + " not found! Please fix your index rotation configuration!");
        return;
    }
    final RotationStrategy rotationStrategy = rotationStrategyProvider.get();
    if (rotationStrategy == null) {
        LOG.warn("No rotation strategy found, not running index rotation!");
        return;
    }
    rotationStrategy.rotate(indexSet);
}

Example 40 with IndexSetConfig

use of org.graylog2.indexer.indexset.IndexSetConfig in project graylog2-server by Graylog2.

the class V20161216123500_DefaultIndexSetMigrationTest method upgradeCreatesDefaultIndexSet.

@Test
public void upgradeCreatesDefaultIndexSet() throws Exception {
    final RotationStrategyConfig rotationStrategyConfig = mock(RotationStrategyConfig.class);
    final RetentionStrategyConfig retentionStrategyConfig = mock(RetentionStrategyConfig.class);
    final IndexSetConfig defaultConfig = IndexSetConfig.builder().id("id").title("title").description("description").indexPrefix("prefix").shards(1).replicas(0).rotationStrategy(rotationStrategyConfig).retentionStrategy(retentionStrategyConfig).creationDate(ZonedDateTime.of(2016, 10, 12, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("prefix-template").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    final IndexSetConfig additionalConfig = defaultConfig.toBuilder().id("foo").indexPrefix("foo").build();
    final IndexSetConfig savedDefaultConfig = defaultConfig.toBuilder().indexAnalyzer(elasticsearchConfiguration.getAnalyzer()).indexTemplateName(elasticsearchConfiguration.getTemplateName()).indexOptimizationMaxNumSegments(elasticsearchConfiguration.getIndexOptimizationMaxNumSegments()).indexOptimizationDisabled(elasticsearchConfiguration.isDisableIndexOptimization()).build();
    final IndexSetConfig savedAdditionalConfig = additionalConfig.toBuilder().indexAnalyzer(elasticsearchConfiguration.getAnalyzer()).indexTemplateName("foo-template").indexOptimizationMaxNumSegments(elasticsearchConfiguration.getIndexOptimizationMaxNumSegments()).indexOptimizationDisabled(elasticsearchConfiguration.isDisableIndexOptimization()).build();
    when(indexSetService.save(any(IndexSetConfig.class))).thenReturn(savedAdditionalConfig, savedDefaultConfig);
    when(indexSetService.getDefault()).thenReturn(defaultConfig);
    when(indexSetService.findAll()).thenReturn(ImmutableList.of(defaultConfig, additionalConfig));
    when(clusterConfigService.get(DefaultIndexSetCreated.class)).thenReturn(DefaultIndexSetCreated.create());
    final ArgumentCaptor<IndexSetConfig> indexSetConfigCaptor = ArgumentCaptor.forClass(IndexSetConfig.class);
    migration.upgrade();
    verify(indexSetService, times(2)).save(indexSetConfigCaptor.capture());
    verify(clusterEventBus, times(2)).post(any(IndexSetCreatedEvent.class));
    verify(clusterConfigService).write(V20161216123500_Succeeded.create());
    final List<IndexSetConfig> allValues = indexSetConfigCaptor.getAllValues();
    assertThat(allValues).hasSize(2);
    final IndexSetConfig capturedDefaultIndexSetConfig = allValues.get(0);
    assertThat(capturedDefaultIndexSetConfig.id()).isEqualTo("id");
    assertThat(capturedDefaultIndexSetConfig.title()).isEqualTo("title");
    assertThat(capturedDefaultIndexSetConfig.description()).isEqualTo("description");
    assertThat(capturedDefaultIndexSetConfig.indexPrefix()).isEqualTo("prefix");
    assertThat(capturedDefaultIndexSetConfig.shards()).isEqualTo(1);
    assertThat(capturedDefaultIndexSetConfig.replicas()).isEqualTo(0);
    assertThat(capturedDefaultIndexSetConfig.rotationStrategy()).isEqualTo(rotationStrategyConfig);
    assertThat(capturedDefaultIndexSetConfig.retentionStrategy()).isEqualTo(retentionStrategyConfig);
    assertThat(capturedDefaultIndexSetConfig.indexAnalyzer()).isEqualTo(elasticsearchConfiguration.getAnalyzer());
    assertThat(capturedDefaultIndexSetConfig.indexTemplateName()).isEqualTo(elasticsearchConfiguration.getTemplateName());
    assertThat(capturedDefaultIndexSetConfig.indexOptimizationMaxNumSegments()).isEqualTo(elasticsearchConfiguration.getIndexOptimizationMaxNumSegments());
    assertThat(capturedDefaultIndexSetConfig.indexOptimizationDisabled()).isEqualTo(elasticsearchConfiguration.isDisableIndexOptimization());
    final IndexSetConfig capturedAdditionalIndexSetConfig = allValues.get(1);
    assertThat(capturedAdditionalIndexSetConfig.id()).isEqualTo("foo");
    assertThat(capturedAdditionalIndexSetConfig.title()).isEqualTo("title");
    assertThat(capturedAdditionalIndexSetConfig.description()).isEqualTo("description");
    assertThat(capturedAdditionalIndexSetConfig.indexPrefix()).isEqualTo("foo");
    assertThat(capturedAdditionalIndexSetConfig.shards()).isEqualTo(1);
    assertThat(capturedAdditionalIndexSetConfig.replicas()).isEqualTo(0);
    assertThat(capturedAdditionalIndexSetConfig.rotationStrategy()).isEqualTo(rotationStrategyConfig);
    assertThat(capturedAdditionalIndexSetConfig.retentionStrategy()).isEqualTo(retentionStrategyConfig);
    assertThat(capturedAdditionalIndexSetConfig.indexAnalyzer()).isEqualTo(elasticsearchConfiguration.getAnalyzer());
    assertThat(capturedAdditionalIndexSetConfig.indexTemplateName()).isEqualTo("foo-template");
    assertThat(capturedAdditionalIndexSetConfig.indexOptimizationMaxNumSegments()).isEqualTo(elasticsearchConfiguration.getIndexOptimizationMaxNumSegments());
    assertThat(capturedAdditionalIndexSetConfig.indexOptimizationDisabled()).isEqualTo(elasticsearchConfiguration.isDisableIndexOptimization());
}