Examples with IndexSet org.graylog2.indexer.IndexSet used on opensource projects

Example 36 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class SizeBasedRotationStrategy method shouldRotate.

@Nullable
@Override
protected Result shouldRotate(final String index, IndexSet indexSet) {
    if (!(indexSet.getConfig().rotationStrategy() instanceof SizeBasedRotationStrategyConfig)) {
        throw new IllegalStateException("Invalid rotation strategy config <" + indexSet.getConfig().rotationStrategy().getClass().getCanonicalName() + "> for index set <" + indexSet.getConfig().id() + ">");
    }
    final SizeBasedRotationStrategyConfig config = (SizeBasedRotationStrategyConfig) indexSet.getConfig().rotationStrategy();
    final IndexStatistics indexStats = indices.getIndexStats(index);
    if (indexStats == null) {
        return null;
    }
    final long sizeInBytes = indexStats.primaries().getStore().getSizeInBytes();
    final boolean shouldRotate = sizeInBytes > config.maxSize();
    return new Result() {

        public final MessageFormat ROTATE = new MessageFormat("Storage size for index <{0}> is {1} bytes, exceeding the maximum of {2} bytes. Rotating index.", Locale.ENGLISH);

        public final MessageFormat NOT_ROTATE = new MessageFormat("Storage size for index <{0}> is {1} bytes, below the maximum of {2} bytes. Not doing anything.", Locale.ENGLISH);

        @Override
        public String getDescription() {
            MessageFormat format = shouldRotate() ? ROTATE : NOT_ROTATE;
            return format.format(new Object[] { index, sizeInBytes, config.maxSize() });
        }

        @Override
        public boolean shouldRotate() {
            return shouldRotate;
        }
    };
}

Example 37 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class Indices method ensureIndexTemplate.

private void ensureIndexTemplate(IndexSet indexSet) {
    final Map<String, Object> template = indexMapping.messageTemplate(indexSet.getIndexWildcard(), indexSet.getConfig().indexAnalyzer());
    final PutIndexTemplateRequest itr = c.admin().indices().preparePutTemplate(indexSet.getConfig().indexTemplateName()).setOrder(// Make sure templates with "order: 0" and higher are applied after our template!
    -1).setSource(template).request();
    try {
        final boolean acknowledged = c.admin().indices().putTemplate(itr).actionGet().isAcknowledged();
        if (acknowledged) {
            LOG.info("Created Graylog index template \"{}\" in Elasticsearch.", indexSet.getConfig().indexTemplateName());
        }
    } catch (Exception e) {
        LOG.error("Unable to create the Graylog index template: " + indexSet.getConfig().indexTemplateName(), e);
    }
}

Example 38 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class IndexSetCleanupJob method execute.

@Override
public void execute() {
    final IndexSetConfig config = indexSet.getConfig();
    final String[] managedIndices = indexSet.getManagedIndices();
    this.total = managedIndices.length;
    try {
        LOG.info("Deleting index template <{}> from Elasticsearch", config.indexTemplateName());
        indices.deleteIndexTemplate(indexSet);
    } catch (IndexTemplateMissingException ignored) {
        LOG.debug("Unable to delete index template <{}> because it does not exist.", config.indexTemplateName());
    } catch (Exception e) {
        LOG.error("Unable to delete index template <{}>", config.indexTemplateName(), e);
    }
    for (String indexName : managedIndices) {
        if (cancel) {
            LOG.info("Cancel requested. Deleted <{}> of <{}> indices.", deleted, total);
            break;
        }
        try {
            LOG.info("Removing index range information for index: {}", indexName);
            indexRangeService.remove(indexName);
            LOG.info("Deleting index <{}> in index set <{}> ({})", indexName, config.id(), config.title());
            indices.delete(indexName);
            deleted.incrementAndGet();
        } catch (Exception e) {
            LOG.error("Unable to delete index <{}>", indexName, e);
        }
    }
}

Example 39 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class StreamResource method update.

@PUT
@Timed
@Path("/{streamId}")
@ApiOperation(value = "Update a stream")
@Consumes(MediaType.APPLICATION_JSON)
@Produces(MediaType.APPLICATION_JSON)
@ApiResponses(value = { @ApiResponse(code = 404, message = "Stream not found."), @ApiResponse(code = 400, message = "Invalid ObjectId.") })
@AuditEvent(type = AuditEventTypes.STREAM_UPDATE)
public StreamResponse update(@ApiParam(name = "streamId", required = true) @PathParam("streamId") String streamId, @ApiParam(name = "JSON body", required = true) @Valid @NotNull UpdateStreamRequest cr) throws NotFoundException, ValidationException {
    checkPermission(RestPermissions.STREAMS_EDIT, streamId);
    checkNotDefaultStream(streamId, "The default stream cannot be edited.");
    final Stream stream = streamService.load(streamId);
    if (!Strings.isNullOrEmpty(cr.title())) {
        stream.setTitle(cr.title());
    }
    if (!Strings.isNullOrEmpty(cr.description())) {
        stream.setDescription(cr.description());
    }
    if (cr.matchingType() != null) {
        try {
            stream.setMatchingType(Stream.MatchingType.valueOf(cr.matchingType()));
        } catch (IllegalArgumentException e) {
            throw new BadRequestException("Invalid matching type '" + cr.matchingType() + "' specified. Should be one of: " + Arrays.toString(Stream.MatchingType.values()));
        }
    }
    final Boolean removeMatchesFromDefaultStream = cr.removeMatchesFromDefaultStream();
    if (removeMatchesFromDefaultStream != null) {
        stream.setRemoveMatchesFromDefaultStream(removeMatchesFromDefaultStream);
    }
    // id if it's null/empty in the update request.
    if (!Strings.isNullOrEmpty(cr.indexSetId())) {
        stream.setIndexSetId(cr.indexSetId());
    }
    final Optional<IndexSet> indexSet = indexSetRegistry.get(stream.getIndexSetId());
    if (!indexSet.isPresent()) {
        throw new BadRequestException("Index set with ID <" + stream.getIndexSetId() + "> does not exist!");
    } else if (!indexSet.get().getConfig().isWritable()) {
        throw new BadRequestException("Assigned index set must be writable!");
    }
    streamService.save(stream);
    clusterEventBus.post(StreamsChangedEvent.create(stream.getId()));
    return streamToResponse(stream);
}

Example 40 with IndexSet

use of org.graylog2.indexer.IndexSet in project graylog2-server by Graylog2.

the class TimeBasedRotationStrategyTest method shouldRotateConcurrently.

@Test
public void shouldRotateConcurrently() throws Exception {
    final DateTime initialTime = new DateTime(2014, 1, 1, 1, 59, 59, 0, DateTimeZone.UTC);
    final Period period = Period.hours(1);
    final InstantMillisProvider clock = new InstantMillisProvider(initialTime);
    DateTimeUtils.setCurrentMillisProvider(clock);
    final IndexSet indexSet1 = mock(IndexSet.class);
    final IndexSet indexSet2 = mock(IndexSet.class);
    final IndexSetConfig indexSetConfig1 = mock(IndexSetConfig.class);
    final IndexSetConfig indexSetConfig2 = mock(IndexSetConfig.class);
    when(indexSetConfig1.id()).thenReturn("id1");
    when(indexSetConfig2.id()).thenReturn("id2");
    when(indexSet1.getConfig()).thenReturn(indexSetConfig1);
    when(indexSet2.getConfig()).thenReturn(indexSetConfig2);
    when(indexSetConfig1.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    when(indexSetConfig2.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    when(indices.indexCreationDate(anyString())).thenReturn(initialTime.minus(minutes(5)));
    // Should not rotate the initial index.
    when(indexSet1.getNewestIndex()).thenReturn("index1");
    rotationStrategy.rotate(indexSet1);
    verify(indexSet1, never()).cycle();
    reset(indexSet1);
    when(indexSet2.getNewestIndex()).thenReturn("index2");
    rotationStrategy.rotate(indexSet2);
    verify(indexSet2, never()).cycle();
    reset(indexSet2);
    clock.tick(seconds(2));
    // Crossed rotation period.
    when(indexSet1.getNewestIndex()).thenReturn("index1");
    when(indexSet1.getConfig()).thenReturn(indexSetConfig1);
    when(indexSetConfig1.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet1);
    verify(indexSet1, times(1)).cycle();
    reset(indexSet1);
    when(indexSet2.getNewestIndex()).thenReturn("index2");
    when(indexSet2.getConfig()).thenReturn(indexSetConfig2);
    when(indexSetConfig2.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet2);
    verify(indexSet2, times(1)).cycle();
    reset(indexSet2);
    clock.tick(seconds(2));
    // Did not cross rotation period.
    when(indexSet1.getNewestIndex()).thenReturn("index1");
    when(indexSet1.getConfig()).thenReturn(indexSetConfig1);
    when(indexSetConfig1.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet1);
    verify(indexSet1, never()).cycle();
    reset(indexSet1);
    when(indexSet2.getNewestIndex()).thenReturn("index2");
    when(indexSet2.getConfig()).thenReturn(indexSetConfig2);
    when(indexSetConfig2.rotationStrategy()).thenReturn(TimeBasedRotationStrategyConfig.create(period));
    rotationStrategy.rotate(indexSet2);
    verify(indexSet2, never()).cycle();
    reset(indexSet2);
}