Examples with IndexSet org.graylog2.indexer.IndexSet used on opensource projects

Example 31 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class IndexSetsResourceTest method delete.

@Test
public void delete() throws Exception {
    final IndexSet indexSet = mock(IndexSet.class);
    final IndexSetConfig indexSetConfig = mock(IndexSetConfig.class);
    when(indexSet.getConfig()).thenReturn(indexSetConfig);
    when(indexSetRegistry.get("id")).thenReturn(Optional.of(indexSet));
    when(indexSetCleanupJobFactory.create(indexSet)).thenReturn(mock(IndexSetCleanupJob.class));
    when(indexSetRegistry.getDefault()).thenReturn(null);
    when(indexSetService.delete("id")).thenReturn(1);
    indexSetsResource.delete("id", false);
    indexSetsResource.delete("id", true);
    verify(indexSetRegistry, times(2)).getDefault();
    verify(indexSetService, times(2)).delete("id");
    verify(systemJobManager, times(1)).submit(any(IndexSetCleanupJob.class));
    verifyNoMoreInteractions(indexSetService);
}

Example 32 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class RebuildIndexRangesJob method execute.

@Override
public void execute() {
    info("Recalculating index ranges.");
    // for each index set we know about
    final ListMultimap<IndexSet, String> indexSets = MultimapBuilder.hashKeys().arrayListValues().build();
    for (IndexSet indexSet : this.indexSets) {
        final String[] managedIndicesNames = indexSet.getManagedIndices();
        for (String name : managedIndicesNames) {
            indexSets.put(indexSet, name);
        }
    }
    if (indexSets.size() == 0) {
        info("No indices, nothing to calculate.");
        return;
    }
    indicesToCalculate = indexSets.values().size();
    Stopwatch sw = Stopwatch.createStarted();
    for (IndexSet indexSet : indexSets.keySet()) {
        LOG.info("Recalculating index ranges for index set {} ({}): {} indices affected.", indexSet.getConfig().title(), indexSet.getIndexWildcard(), indexSets.get(indexSet).size());
        for (String index : indexSets.get(indexSet)) {
            try {
                if (index.equals(indexSet.getActiveWriteIndex())) {
                    LOG.debug("{} is current write target, do not calculate index range for it", index);
                    final IndexRange emptyRange = indexRangeService.createUnknownRange(index);
                    try {
                        final IndexRange indexRange = indexRangeService.get(index);
                        if (indexRange.begin().getMillis() != 0 || indexRange.end().getMillis() != 0) {
                            LOG.info("Invalid date ranges for write index {}, resetting it.", index);
                            indexRangeService.save(emptyRange);
                        }
                    } catch (NotFoundException e) {
                        LOG.info("No index range found for write index {}, recreating it.", index);
                        indexRangeService.save(emptyRange);
                    }
                    indicesCalculated.incrementAndGet();
                    continue;
                }
            } catch (TooManyAliasesException e) {
                LOG.error("Multiple write alias targets found, this is a bug.");
                indicesCalculated.incrementAndGet();
                continue;
            }
            if (cancelRequested) {
                info("Stop requested. Not calculating next index range, not updating ranges.");
                sw.stop();
                return;
            }
            try {
                final IndexRange indexRange = indexRangeService.calculateRange(index);
                indexRangeService.save(indexRange);
                LOG.info("Created ranges for index {}: {}", index, indexRange);
            } catch (Exception e) {
                LOG.info("Could not calculate range of index [" + index + "]. Skipping.", e);
            } finally {
                indicesCalculated.incrementAndGet();
            }
        }
    }
    info("Done calculating index ranges for " + indicesToCalculate + " indices. Took " + sw.stop().elapsed(TimeUnit.MILLISECONDS) + "ms.");
}

Example 33 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class SetIndexReadOnlyJob method execute.

@Override
public void execute() {
    final Optional<IndexSet> indexSet = indexSetRegistry.getForIndex(index);
    if (!indexSet.isPresent()) {
        log.error("Couldn't find index set for index <{}>", index);
        return;
    }
    log.info("Flushing old index <{}>.", index);
    indices.flush(index);
    log.info("Setting old index <{}> to read-only.", index);
    indices.setReadOnly(index);
    activityWriter.write(new Activity("Flushed and set <" + index + "> to read-only.", SetIndexReadOnlyJob.class));
    if (!indexSet.get().getConfig().indexOptimizationDisabled()) {
        try {
            systemJobManager.submit(optimizeIndexJobFactory.create(index, indexSet.get().getConfig().indexOptimizationMaxNumSegments()));
        } catch (SystemJobConcurrencyException e) {
            // The concurrency limit is very high. This should never happen.
            log.error("Cannot optimize index <" + index + ">.", e);
        }
    }
}

Example 34 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class FixDeflectorByDeleteJob method doExecute.

public void doExecute(IndexSet indexSet) {
    if (!indexSet.getConfig().isWritable()) {
        LOG.debug("No need to fix deflector for non-writable index set <{}> ({})", indexSet.getConfig().id(), indexSet.getConfig().title());
        return;
    }
    if (indexSet.isUp() || !indices.exists(indexSet.getWriteIndexAlias())) {
        LOG.error("There is no index <{}>. No need to run this job. Aborting.", indexSet.getWriteIndexAlias());
        return;
    }
    LOG.info("Attempting to fix deflector with delete strategy.");
    // Pause message processing and lock the pause.
    boolean wasProcessing = serverStatus.isProcessing();
    serverStatus.pauseMessageProcessing();
    progress = 10;
    bufferSynchronizer.waitForEmptyBuffers();
    progress = 25;
    // Delete deflector index.
    LOG.info("Deleting <{}> index.", indexSet.getWriteIndexAlias());
    indices.delete(indexSet.getWriteIndexAlias());
    progress = 70;
    // Set up deflector.
    indexSet.setUp();
    progress = 80;
    // Start message processing again.
    try {
        serverStatus.unlockProcessingPause();
        if (wasProcessing) {
            serverStatus.resumeMessageProcessing();
        }
    } catch (Exception e) {
        // lol checked exceptions
        throw new RuntimeException("Could not unlock processing pause.", e);
    }
    progress = 90;
    activityWriter.write(new Activity("Notification condition [" + Notification.Type.DEFLECTOR_EXISTS_AS_INDEX + "] " + "has been fixed.", this.getClass()));
    notificationService.fixed(Notification.Type.DEFLECTOR_EXISTS_AS_INDEX);
    progress = 100;
    LOG.info("Finished.");
}

Example 35 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class FixDeflectorByMoveJob method doExecute.

public void doExecute(IndexSet indexSet) {
    if (!indexSet.getConfig().isWritable()) {
        LOG.debug("No need to fix deflector for non-writable index set <{}> ({})", indexSet.getConfig().id(), indexSet.getConfig().title());
        return;
    }
    if (indexSet.isUp() || !indices.exists(indexSet.getWriteIndexAlias())) {
        LOG.error("There is no index <{}>. No need to run this job. Aborting.", indexSet.getWriteIndexAlias());
        return;
    }
    LOG.info("Attempting to fix deflector with move strategy.");
    boolean wasProcessing = true;
    try {
        // Pause message processing and lock the pause.
        wasProcessing = serverStatus.isProcessing();
        serverStatus.pauseMessageProcessing();
        progress = 5;
        bufferSynchronizer.waitForEmptyBuffers();
        progress = 10;
        // Copy messages to new index.
        String newTarget = null;
        try {
            newTarget = indexSet.getNewestIndex();
            LOG.info("Starting to move <{}> to <{}>.", indexSet.getWriteIndexAlias(), newTarget);
            indices.move(indexSet.getWriteIndexAlias(), newTarget);
        } catch (Exception e) {
            LOG.error("Moving index failed. Rolling back.", e);
            if (newTarget != null) {
                indices.delete(newTarget);
            }
            throw new RuntimeException(e);
        }
        LOG.info("Done moving deflector index.");
        progress = 85;
        // Delete deflector index.
        LOG.info("Deleting <{}> index.", indexSet.getWriteIndexAlias());
        indices.delete(indexSet.getWriteIndexAlias());
        progress = 90;
        // Set up deflector.
        indexSet.setUp();
        progress = 95;
    } finally {
        // Start message processing again.
        serverStatus.unlockProcessingPause();
        if (wasProcessing) {
            serverStatus.resumeMessageProcessing();
        }
    }
    progress = 90;
    activityWriter.write(new Activity("Notification condition [" + Notification.Type.DEFLECTOR_EXISTS_AS_INDEX + "] " + "has been fixed.", this.getClass()));
    notificationService.fixed(Notification.Type.DEFLECTOR_EXISTS_AS_INDEX);
    progress = 100;
    LOG.info("Finished.");
}