Examples with Indices org.graylog2.indexer.indices.Indices used on opensource projects

Example 51 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class IndexFieldTypePollerPeriodical method refreshFieldTypes.

private void refreshFieldTypes(Collection<IndexSetConfig> indexSetConfigs) {
    LOG.debug("Refreshing index field types for {} index sets.", indexSetConfigs.size());
    // this is the first time we run, or the index sets have changed, so we re-initialize the field types
    indexSetConfigs.forEach(indexSetConfig -> {
        final String indexSetId = indexSetConfig.id();
        final String indexSetTitle = indexSetConfig.title();
        try {
            final Set<IndexFieldTypesDTO> existingIndexTypes = ImmutableSet.copyOf(dbService.findForIndexSet(indexSetId));
            final IndexSet indexSet = mongoIndexSetFactory.create(indexSetConfig);
            // We check that we have the field types for all existing indices
            LOG.debug("Refreshing index field types for index set <{}/{}>", indexSetTitle, indexSetId);
            poller.poll(indexSet, existingIndexTypes).forEach(dbService::upsert);
            // Cleanup orphaned field type entries that haven't been removed by the event handler
            dbService.findForIndexSet(indexSetId).stream().filter(types -> !indices.exists(types.indexName())).forEach(types -> dbService.delete(types.id()));
        } finally {
            lastPoll.put(indexSetId, Instant.now());
        }
    });
}

Example 52 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class Cluster method elasticsearchStats.

public ElasticsearchStats elasticsearchStats() {
    final List<String> indices = Arrays.asList(indexSetRegistry.getIndexWildcards());
    final org.graylog2.system.stats.elasticsearch.ClusterStats clusterStats = clusterAdapter.clusterStats();
    final PendingTasksStats pendingTasksStats = clusterAdapter.pendingTasks();
    final ShardStats shardStats = clusterAdapter.shardStats(indices);
    final org.graylog2.system.stats.elasticsearch.ClusterHealth clusterHealth = org.graylog2.system.stats.elasticsearch.ClusterHealth.from(shardStats, pendingTasksStats);
    final HealthStatus healthStatus = clusterAdapter.health(indices).orElseThrow(() -> new IllegalStateException("Unable to retrieve cluster health."));
    return ElasticsearchStats.create(clusterStats.clusterName(), clusterStats.clusterVersion(), healthStatus, clusterHealth, clusterStats.nodesStats(), clusterStats.indicesStats());
}

Example 53 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class Searches method search.

public SearchResult search(SearchesConfig config) {
    final Set<IndexRange> indexRanges = determineAffectedIndicesWithRanges(config.range(), config.filter());
    final Set<String> indices = extractIndexNamesFromIndexRanges(indexRanges);
    final SearchResult result = searchesAdapter.search(indices, indexRanges, config);
    recordEsMetrics(result.tookMs(), config.range());
    return result;
}

Example 54 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class RebuildIndexRangesJob method execute.

@Override
public void execute() {
    info("Recalculating index ranges.");
    // for each index set we know about
    final ListMultimap<IndexSet, String> indexSets = MultimapBuilder.hashKeys().arrayListValues().build();
    for (IndexSet indexSet : this.indexSets) {
        final String[] managedIndicesNames = indexSet.getManagedIndices();
        for (String name : managedIndicesNames) {
            indexSets.put(indexSet, name);
        }
    }
    if (indexSets.size() == 0) {
        info("No indices, nothing to calculate.");
        return;
    }
    indicesToCalculate = indexSets.values().size();
    Stopwatch sw = Stopwatch.createStarted();
    for (IndexSet indexSet : indexSets.keySet()) {
        LOG.info("Recalculating index ranges for index set {} ({}): {} indices affected.", indexSet.getConfig().title(), indexSet.getIndexWildcard(), indexSets.get(indexSet).size());
        for (String index : indexSets.get(indexSet)) {
            try {
                if (index.equals(indexSet.getActiveWriteIndex())) {
                    LOG.debug("{} is current write target, do not calculate index range for it", index);
                    final IndexRange emptyRange = indexRangeService.createUnknownRange(index);
                    try {
                        final IndexRange indexRange = indexRangeService.get(index);
                        if (indexRange.begin().getMillis() != 0 || indexRange.end().getMillis() != 0) {
                            LOG.info("Invalid date ranges for write index {}, resetting it.", index);
                            indexRangeService.save(emptyRange);
                        }
                    } catch (NotFoundException e) {
                        LOG.info("No index range found for write index {}, recreating it.", index);
                        indexRangeService.save(emptyRange);
                    }
                    indicesCalculated.incrementAndGet();
                    continue;
                }
            } catch (TooManyAliasesException e) {
                LOG.error("Multiple write alias targets found, this is a bug.");
                indicesCalculated.incrementAndGet();
                continue;
            }
            if (cancelRequested) {
                info("Stop requested. Not calculating next index range, not updating ranges.");
                sw.stop();
                return;
            }
            try {
                final IndexRange indexRange = indexRangeService.calculateRange(index);
                indexRangeService.save(indexRange);
                LOG.info("Created ranges for index {}: {}", index, indexRange);
            } catch (Exception e) {
                LOG.info("Could not calculate range of index [" + index + "]. Skipping.", e);
            } finally {
                indicesCalculated.incrementAndGet();
            }
        }
    }
    info("Done calculating index ranges for " + indicesToCalculate + " indices. Took " + sw.stop().elapsed(TimeUnit.MILLISECONDS) + "ms.");
}

Example 55 with Indices

use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.

the class IndexSetCleanupJob method execute.

@Override
public void execute() {
    final IndexSetConfig config = indexSet.getConfig();
    final String[] managedIndices = indexSet.getManagedIndices();
    this.total = managedIndices.length;
    try {
        LOG.info("Deleting index template <{}> from Elasticsearch", config.indexTemplateName());
        indices.deleteIndexTemplate(indexSet);
    } catch (Exception e) {
        LOG.error("Unable to delete index template <{}>", config.indexTemplateName(), e);
    }
    for (String indexName : managedIndices) {
        if (cancel) {
            LOG.info("Cancel requested. Deleted <{}> of <{}> indices.", deleted, total);
            break;
        }
        try {
            LOG.info("Removing index range information for index: {}", indexName);
            indexRangeService.remove(indexName);
            LOG.info("Deleting index <{}> in index set <{}> ({})", indexName, config.id(), config.title());
            indices.delete(indexName);
            deleted.incrementAndGet();
        } catch (Exception e) {
            LOG.error("Unable to delete index <{}>", indexName, e);
        }
    }
}