use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.
the class SearchesTest method determineAffectedIndicesIncludesDeflectorTarget.
@Test
public void determineAffectedIndicesIncludesDeflectorTarget() throws Exception {
final DateTime now = DateTime.now(DateTimeZone.UTC);
final MongoIndexRange indexRange0 = MongoIndexRange.create("graylog_0", now, now.plusDays(1), now, 0);
final MongoIndexRange indexRange1 = MongoIndexRange.create("graylog_1", now.plusDays(1), now.plusDays(2), now, 0);
final MongoIndexRange indexRangeLatest = MongoIndexRange.create("graylog_2", new DateTime(0L, DateTimeZone.UTC), new DateTime(0L, DateTimeZone.UTC), now, 0);
final SortedSet<IndexRange> indices = ImmutableSortedSet.orderedBy(IndexRange.COMPARATOR).add(indexRange0).add(indexRange1).add(indexRangeLatest).build();
when(indexRangeService.find(any(DateTime.class), any(DateTime.class))).thenReturn(indices);
final TimeRange absoluteRange = AbsoluteRange.create(now.minusDays(1), now.plusDays(1));
final TimeRange keywordRange = KeywordRange.create("1 day ago");
final TimeRange relativeRange = RelativeRange.create(3600);
assertThat(searches.determineAffectedIndices(absoluteRange, null)).containsExactlyInAnyOrder(indexRangeLatest.indexName(), indexRange0.indexName(), indexRange1.indexName());
assertThat(searches.determineAffectedIndices(keywordRange, null)).containsExactlyInAnyOrder(indexRangeLatest.indexName(), indexRange0.indexName(), indexRange1.indexName());
assertThat(searches.determineAffectedIndices(relativeRange, null)).containsExactlyInAnyOrder(indexRangeLatest.indexName(), indexRange0.indexName(), indexRange1.indexName());
}
use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.
the class V20161130141500_DefaultStreamRecalcIndexRanges method upgrade.
@Override
public void upgrade() {
IndexSet indexSet;
try {
indexSet = indexSetRegistry.getDefault();
} catch (IllegalStateException e) {
// Try to find the default index set manually if the index set registry cannot find it.
// This is needed if you run this migration with a 2.2.0-beta.2 state before commit 460cac6af.
final IndexSetConfig indexSetConfig = indexSetService.findOne(DBQuery.is("default", true)).orElseThrow(() -> new IllegalStateException("No default index set configured! This is a bug!"));
indexSet = indexSetFactory.create(indexSetConfig);
}
final IndexSet defaultIndexSet = indexSet;
if (!cluster.isConnected()) {
LOG.info("Cluster not connected yet, delaying migration until it is reachable.");
while (true) {
try {
cluster.waitForConnectedAndDeflectorHealthy();
break;
} catch (InterruptedException | TimeoutException e) {
LOG.warn("Interrupted or timed out waiting for Elasticsearch cluster, checking again.");
}
}
}
final Set<String> indexRangesWithoutStreams = indexRangeService.findAll().stream().filter(indexRange -> defaultIndexSet.isManagedIndex(indexRange.indexName())).filter(indexRange -> indexRange.streamIds() == null).map(IndexRange::indexName).collect(Collectors.toSet());
if (indexRangesWithoutStreams.size() == 0) {
// all ranges have a stream list, even if it is empty, nothing more to do
return;
}
final String currentWriteTarget;
try {
currentWriteTarget = defaultIndexSet.getActiveWriteIndex();
} catch (TooManyAliasesException e) {
LOG.error("Multiple write targets found for write alias. Cannot continue to assign streams to older indices", e);
return;
}
for (String indexName : defaultIndexSet.getManagedIndices()) {
if (indexName.equals(currentWriteTarget)) {
// do not recalculate for current write target
continue;
}
if (!indexRangesWithoutStreams.contains(indexName)) {
// already computed streams for this index
continue;
}
LOG.info("Recalculating streams in index {}", indexName);
final CreateNewSingleIndexRangeJob createNewSingleIndexRangeJob = rebuildIndexRangeJobFactory.create(indexSetRegistry.getAll(), indexName);
createNewSingleIndexRangeJob.execute();
}
}
use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.
the class IndexRangesCleanupPeriodical method doRun.
@Override
public void doRun() {
if (!cluster.isConnected() || !cluster.isHealthy()) {
LOG.info("Skipping index range cleanup because the Elasticsearch cluster is unreachable or unhealthy");
return;
}
final Set<String> indexNames = ImmutableSet.copyOf(indexSetRegistry.getManagedIndices());
final SortedSet<IndexRange> indexRanges = indexRangeService.findAll();
final Set<String> removedIndices = new HashSet<>();
for (IndexRange indexRange : indexRanges) {
if (!indexNames.contains(indexRange.indexName())) {
removedIndices.add(indexRange.indexName());
}
}
if (!removedIndices.isEmpty()) {
LOG.info("Removing index range information for unavailable indices: {}", removedIndices);
eventBus.post(IndicesDeletedEvent.create(removedIndices));
}
}
use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.
the class IndexRangesMigrationPeriodical method doRun.
@Override
public void doRun() {
final MongoIndexRangesMigrationComplete migrationComplete = clusterConfigService.get(MongoIndexRangesMigrationComplete.class);
if (migrationComplete != null && migrationComplete.complete) {
LOG.debug("Migration of index ranges (pre Graylog 1.2.2) already complete. Skipping migration process.");
return;
}
while (!cluster.isConnected() || !cluster.isHealthy()) {
Uninterruptibles.sleepUninterruptibly(5, TimeUnit.SECONDS);
}
final Set<String> indexNames = ImmutableSet.copyOf(indexSetRegistry.getManagedIndices());
// Migrate old MongoDB index ranges
final SortedSet<IndexRange> mongoIndexRanges = legacyMongoIndexRangeService.findAll();
for (IndexRange indexRange : mongoIndexRanges) {
if (indexNames.contains(indexRange.indexName())) {
LOG.info("Migrating index range from MongoDB: {}", indexRange);
indexRangeService.save(indexRange);
} else {
LOG.info("Removing stale index range from MongoDB: {}", indexRange);
}
legacyMongoIndexRangeService.delete(indexRange.indexName());
}
// Check whether all index ranges have been migrated
final int numberOfIndices = indexNames.size();
final SortedSet<IndexRange> allIndexRanges = indexRangeService.findAll();
final int numberOfIndexRanges = allIndexRanges.size();
if (numberOfIndices > numberOfIndexRanges) {
LOG.info("There are more indices ({}) than there are index ranges ({}). Notifying administrator.", numberOfIndices, numberOfIndexRanges);
// remove all present index names so we can display the index sets that need manual fixing
final Set<String> extraIndices = Sets.newHashSet(indexNames);
allIndexRanges.forEach(indexRange -> extraIndices.remove(indexRange.indexName()));
final Set<String> affectedIndexSetNames = extraIndices.stream().map(indexSetRegistry::getForIndex).filter(Optional::isPresent).map(Optional::get).map(IndexSet::getConfig).map(IndexSetConfig::title).collect(Collectors.toSet());
final Notification notification = notificationService.buildNow().addSeverity(Notification.Severity.URGENT).addType(Notification.Type.INDEX_RANGES_RECALCULATION).addDetail("indices", numberOfIndices).addDetail("index_ranges", numberOfIndexRanges).addDetail("index_sets", affectedIndexSetNames.isEmpty() ? null : Joiner.on(", ").join(affectedIndexSetNames));
notificationService.publishIfFirst(notification);
}
clusterConfigService.write(new MongoIndexRangesMigrationComplete(true));
}
use of org.graylog2.indexer.indices.Indices in project graylog2-server by Graylog2.
the class IndexRotationThread method checkAndRepair.
protected void checkAndRepair(IndexSet indexSet) {
if (!indexSet.isUp()) {
if (indices.exists(indexSet.getWriteIndexAlias())) {
// Publish a notification if there is an *index* called graylog2_deflector
Notification notification = notificationService.buildNow().addType(Notification.Type.DEFLECTOR_EXISTS_AS_INDEX).addSeverity(Notification.Severity.URGENT);
final boolean published = notificationService.publishIfFirst(notification);
if (published) {
LOG.warn("There is an index called [" + indexSet.getWriteIndexAlias() + "]. Cannot fix this automatically and published a notification.");
}
} else {
indexSet.setUp();
}
} else {
try {
String currentTarget;
try {
currentTarget = indexSet.getActiveWriteIndex();
} catch (TooManyAliasesException e) {
// If we get this exception, there are multiple indices which have the deflector alias set.
// We try to cleanup the alias and try again. This should not happen, but might under certain
// circumstances.
indexSet.cleanupAliases(e.getIndices());
try {
currentTarget = indexSet.getActiveWriteIndex();
} catch (TooManyAliasesException e1) {
throw new IllegalStateException(e1);
}
}
String shouldBeTarget = indexSet.getNewestIndex();
if (!shouldBeTarget.equals(currentTarget)) {
String msg = "Deflector is pointing to [" + currentTarget + "], not the newest one: [" + shouldBeTarget + "]. Re-pointing.";
LOG.warn(msg);
activityWriter.write(new Activity(msg, IndexRotationThread.class));
if (ClusterHealthStatus.RED == indices.waitForRecovery(shouldBeTarget)) {
LOG.error("New target index for deflector didn't get healthy within timeout. Skipping deflector update.");
} else {
indexSet.pointTo(shouldBeTarget, currentTarget);
}
}
} catch (NoTargetIndexException e) {
LOG.warn("Deflector is not up. Not trying to point to another index.");
}
}
}
Aggregations