Examples with IndexSet org.graylog2.indexer.IndexSet used on opensource projects

Example 11 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class AbstractIndexCountBasedRetentionStrategy method runRetention.

private void runRetention(IndexSet indexSet, Map<String, Set<String>> deflectorIndices, int removeCount) {
    final Set<String> orderedIndices = Arrays.stream(indexSet.getManagedIndices()).filter(indexName -> !indices.isReopened(indexName)).filter(indexName -> !(deflectorIndices.getOrDefault(indexName, Collections.emptySet()).contains(indexSet.getWriteIndexAlias()))).sorted((indexName1, indexName2) -> indexSet.extractIndexNumber(indexName2).orElse(0).compareTo(indexSet.extractIndexNumber(indexName1).orElse(0))).collect(Collectors.toCollection(LinkedHashSet::new));
    orderedIndices.stream().skip(orderedIndices.size() - removeCount).forEach(indexName -> {
        final String strategyName = this.getClass().getCanonicalName();
        final String msg = "Running retention strategy [" + strategyName + "] for index <" + indexName + ">";
        LOG.info(msg);
        activityWriter.write(new Activity(msg, IndexRetentionThread.class));
        retain(indexName, indexSet);
    });
}

Example 12 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class AbstractIndexCountBasedRetentionStrategy method retain.

@Override
public void retain(IndexSet indexSet) {
    final Map<String, Set<String>> deflectorIndices = indexSet.getAllIndexAliases();
    final int indexCount = (int) deflectorIndices.keySet().stream().filter(indexName -> !indices.isReopened(indexName)).count();
    final Optional<Integer> maxIndices = getMaxNumberOfIndices(indexSet);
    if (!maxIndices.isPresent()) {
        LOG.warn("No retention strategy configuration found, not running index retention!");
        return;
    }
    // Do we have more indices than the configured maximum?
    if (indexCount <= maxIndices.get()) {
        LOG.debug("Number of indices ({}) lower than limit ({}). Not performing any retention actions.", indexCount, maxIndices.get());
        return;
    }
    // We have more indices than the configured maximum! Remove as many as needed.
    final int removeCount = indexCount - maxIndices.get();
    final String msg = "Number of indices (" + indexCount + ") higher than limit (" + maxIndices.get() + "). " + "Running retention for " + removeCount + " indices.";
    LOG.info(msg);
    activityWriter.write(new Activity(msg, IndexRetentionThread.class));
    runRetention(indexSet, deflectorIndices, removeCount);
}

Example 13 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class ClosingRetentionStrategy method getMaxNumberOfIndices.

@Override
protected Optional<Integer> getMaxNumberOfIndices(IndexSet indexSet) {
    final IndexSetConfig indexSetConfig = indexSet.getConfig();
    final RetentionStrategyConfig strategyConfig = indexSetConfig.retentionStrategy();
    if (!(strategyConfig instanceof ClosingRetentionStrategyConfig)) {
        throw new IllegalStateException("Invalid retention strategy config <" + strategyConfig.getClass().getCanonicalName() + "> for index set <" + indexSetConfig.id() + ">");
    }
    final ClosingRetentionStrategyConfig config = (ClosingRetentionStrategyConfig) strategyConfig;
    return Optional.of(config.maxNumberOfIndices());
}

Example 14 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class DeletionRetentionStrategy method getMaxNumberOfIndices.

@Override
protected Optional<Integer> getMaxNumberOfIndices(IndexSet indexSet) {
    final IndexSetConfig indexSetConfig = indexSet.getConfig();
    final RetentionStrategyConfig strategyConfig = indexSetConfig.retentionStrategy();
    if (!(strategyConfig instanceof DeletionRetentionStrategyConfig)) {
        throw new IllegalStateException("Invalid retention strategy config <" + strategyConfig.getClass().getCanonicalName() + "> for index set <" + indexSetConfig.id() + ">");
    }
    final DeletionRetentionStrategyConfig config = (DeletionRetentionStrategyConfig) strategyConfig;
    return Optional.of(config.maxNumberOfIndices());
}

Example 15 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class Messages method propagateFailure.

private void propagateFailure(BulkItemResponse[] items, List<Map.Entry<IndexSet, Message>> messageList, String errorMessage) {
    final List<IndexFailure> indexFailures = new LinkedList<>();
    for (BulkItemResponse item : items) {
        if (item.isFailed()) {
            LOG.trace("Failed to index message: {}", item.getFailureMessage());
            // Write failure to index_failures.
            final BulkItemResponse.Failure f = item.getFailure();
            final Map.Entry<IndexSet, Message> messageEntry = messageList.get(item.getItemId());
            final Map<String, Object> doc = ImmutableMap.<String, Object>builder().put("letter_id", item.getId()).put("index", f.getIndex()).put("type", f.getType()).put("message", f.getMessage()).put("timestamp", messageEntry.getValue().getTimestamp()).build();
            indexFailures.add(new IndexFailureImpl(doc));
        }
    }
    LOG.error("Failed to index [{}] messages. Please check the index error log in your web interface for the reason. Error: {}", indexFailures.size(), errorMessage);
    try {
        // TODO: Magic number
        indexFailureQueue.offer(indexFailures, 25, TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        LOG.warn("Couldn't save index failures.", e);
    }
}