Examples with Indices org.graylog2.indexer.indices.Indices used on opensource projects

Example 11 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class AbstractIndexCountBasedRetentionStrategy method retain.

@Override
public void retain(IndexSet indexSet) {
    final Map<String, Set<String>> deflectorIndices = indexSet.getAllIndexAliases();
    final int indexCount = (int) deflectorIndices.keySet().stream().filter(indexName -> !indices.isReopened(indexName)).count();
    final Optional<Integer> maxIndices = getMaxNumberOfIndices(indexSet);
    if (!maxIndices.isPresent()) {
        LOG.warn("No retention strategy configuration found, not running index retention!");
        return;
    }
    // Do we have more indices than the configured maximum?
    if (indexCount <= maxIndices.get()) {
        LOG.debug("Number of indices ({}) lower than limit ({}). Not performing any retention actions.", indexCount, maxIndices.get());
        return;
    }
    // We have more indices than the configured maximum! Remove as many as needed.
    final int removeCount = indexCount - maxIndices.get();
    final String msg = "Number of indices (" + indexCount + ") higher than limit (" + maxIndices.get() + "). " + "Running retention for " + removeCount + " indices.";
    LOG.info(msg);
    activityWriter.write(new Activity(msg, IndexRetentionThread.class));
    runRetention(indexSet, deflectorIndices, removeCount);
}

Example 12 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class MongoIndexSetTest method cycleSetsOldIndexToReadOnly.

@Test
public void cycleSetsOldIndexToReadOnly() throws SystemJobConcurrencyException {
    final String newIndexName = "graylog_1";
    final String oldIndexName = "graylog_0";
    final Map<String, Set<String>> indexNameAliases = ImmutableMap.of(oldIndexName, Collections.singleton("graylog_deflector"));
    when(indices.getIndexNamesAndAliases(anyString())).thenReturn(indexNameAliases);
    when(indices.create(newIndexName, mongoIndexSet)).thenReturn(true);
    when(indices.waitForRecovery(newIndexName)).thenReturn(ClusterHealthStatus.GREEN);
    final SetIndexReadOnlyAndCalculateRangeJob rangeJob = mock(SetIndexReadOnlyAndCalculateRangeJob.class);
    when(jobFactory.create(oldIndexName)).thenReturn(rangeJob);
    final MongoIndexSet mongoIndexSet = new MongoIndexSet(config, indices, nodeId, indexRangeService, auditEventSender, systemJobManager, jobFactory, activityWriter);
    mongoIndexSet.cycle();
    verify(jobFactory, times(1)).create(oldIndexName);
    verify(systemJobManager, times(1)).submitWithDelay(rangeJob, 30L, TimeUnit.SECONDS);
}

Example 13 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class CountsTest method setUp.

@Before
public void setUp() throws Exception {
    final Map<String, Object> settings = ImmutableMap.of("number_of_shards", 1, "index.number_of_replicas", 0);
    final CreateIndexResponse createIndexResponse1 = client.admin().indices().prepareCreate(INDEX_NAME_1).setSettings(settings).setTimeout(TimeValue.timeValueSeconds(10L)).execute().get();
    assumeTrue(createIndexResponse1.isAcknowledged());
    final CreateIndexResponse createIndexResponse2 = client.admin().indices().prepareCreate(INDEX_NAME_2).setSettings(settings).setTimeout(TimeValue.timeValueSeconds(10L)).execute().get();
    assumeTrue(createIndexResponse2.isAcknowledged());
    final ClusterHealthResponse clusterHealthResponse1 = client.admin().cluster().prepareHealth(INDEX_NAME_1).setWaitForGreenStatus().execute().get();
    assumeTrue(clusterHealthResponse1.getStatus() == ClusterHealthStatus.GREEN);
    final ClusterHealthResponse clusterHealthResponse2 = client.admin().cluster().prepareHealth(INDEX_NAME_2).setWaitForGreenStatus().execute().get();
    assumeTrue(clusterHealthResponse2.getStatus() == ClusterHealthStatus.GREEN);
    counts = new Counts(client, indexSetRegistry);
    indexSetConfig1 = IndexSetConfig.builder().id("id-1").title("title-1").indexPrefix("index_set_1_counts_test").shards(1).replicas(0).rotationStrategyClass(MessageCountRotationStrategy.class.getCanonicalName()).rotationStrategy(MessageCountRotationStrategyConfig.createDefault()).retentionStrategyClass(DeletionRetentionStrategy.class.getCanonicalName()).retentionStrategy(DeletionRetentionStrategyConfig.createDefault()).creationDate(ZonedDateTime.of(2016, 10, 12, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-1").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    indexSetConfig2 = IndexSetConfig.builder().id("id-2").title("title-2").indexPrefix("index_set_2_counts_test").shards(1).replicas(0).rotationStrategyClass(MessageCountRotationStrategy.class.getCanonicalName()).rotationStrategy(MessageCountRotationStrategyConfig.createDefault()).retentionStrategyClass(DeletionRetentionStrategy.class.getCanonicalName()).retentionStrategy(DeletionRetentionStrategyConfig.createDefault()).creationDate(ZonedDateTime.of(2016, 10, 13, 0, 0, 0, 0, ZoneOffset.UTC)).indexAnalyzer("standard").indexTemplateName("template-2").indexOptimizationMaxNumSegments(1).indexOptimizationDisabled(false).build();
    when(indexSetRegistry.getManagedIndices()).thenReturn(new String[] { INDEX_NAME_1, INDEX_NAME_2 });
    when(indexSetRegistry.get(indexSetConfig1.id())).thenReturn(Optional.of(indexSet1));
    when(indexSetRegistry.get(indexSetConfig2.id())).thenReturn(Optional.of(indexSet2));
    when(indexSet1.getManagedIndices()).thenReturn(new String[] { INDEX_NAME_1 });
    when(indexSet2.getManagedIndices()).thenReturn(new String[] { INDEX_NAME_2 });
}

Example 14 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class IndicesGetAllMessageFieldsTest method setUp.

@Before
public void setUp() throws Exception {
    elasticsearchRule.getDatabaseOperation().deleteAll();
    indices = new Indices(client, new IndexMapping(), new Messages(client, new MetricRegistry()), mock(NodeId.class), new NullAuditEventSender());
}

Example 15 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class SizeBasedRotationStrategyTest method testDontRotate.

@Test
public void testDontRotate() throws Exception {
    final CommonStats commonStats = new CommonStats();
    commonStats.store = new StoreStats(1000, 0);
    final IndexStatistics stats = IndexStatistics.create("name", commonStats, commonStats, Collections.<ShardRouting>emptyList());
    when(indices.getIndexStats("name")).thenReturn(stats);
    when(indexSet.getNewestIndex()).thenReturn("name");
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetConfig.rotationStrategy()).thenReturn(SizeBasedRotationStrategyConfig.create(100000L));
    final SizeBasedRotationStrategy strategy = new SizeBasedRotationStrategy(indices, nodeId, auditEventSender);
    strategy.rotate(indexSet);
    verify(indexSet, never()).cycle();
    reset(indexSet);
}