Search in sources :

Example 26 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class IndicesAdapterES6 method move.

@Override
public void move(String source, String target, Consumer<IndexMoveResult> resultCallback) {
    // TODO: This method should use the Re-index API: https://www.elastic.co/guide/en/elasticsearch/reference/5.3/docs-reindex.html
    final String query = SearchSourceBuilder.searchSource().query(QueryBuilders.matchAllQuery()).size(350).sort(SortBuilders.fieldSort(FieldSortBuilder.DOC_FIELD_NAME)).toString();
    final Search request = new Search.Builder(query).setParameter(Parameters.SCROLL, "10s").addIndex(source).build();
    final SearchResult searchResult = JestUtils.execute(jestClient, request, () -> "Couldn't process search query response");
    final String scrollId = searchResult.getJsonObject().path("_scroll_id").asText(null);
    if (scrollId == null) {
        throw new ElasticsearchException("Couldn't find scroll ID in search query response");
    }
    while (true) {
        final SearchScroll scrollRequest = new SearchScroll.Builder(scrollId, "1m").build();
        final JestResult scrollResult = JestUtils.execute(jestClient, scrollRequest, () -> "Couldn't process result of scroll query");
        final JsonNode scrollHits = scrollResult.getJsonObject().path("hits").path("hits");
        // No more hits.
        if (scrollHits.size() == 0) {
            break;
        }
        final Bulk.Builder bulkRequestBuilder = new Bulk.Builder();
        for (JsonNode jsonElement : scrollHits) {
            Optional.ofNullable(jsonElement.path("_source")).map(sourceJson -> objectMapper.<Map<String, Object>>convertValue(sourceJson, TypeReferences.MAP_STRING_OBJECT)).ifPresent(doc -> {
                final String id = (String) doc.remove("_id");
                if (!Strings.isNullOrEmpty(id)) {
                    bulkRequestBuilder.addAction(indexingHelper.prepareIndexRequest(target, doc, id));
                }
            });
        }
        final BulkResult bulkResult = JestUtils.execute(jestClient, bulkRequestBuilder.build(), () -> "Couldn't bulk index messages into index " + target);
        final boolean hasFailedItems = !bulkResult.getFailedItems().isEmpty();
        final IndexMoveResult result = IndexMoveResult.create(bulkResult.getItems().size(), bulkResult.getJsonObject().path("took").asLong(), hasFailedItems);
        resultCallback.accept(result);
    }
}
Also used : TermsAggregation(io.searchbox.core.search.aggregation.TermsAggregation) DateTimeZone(org.joda.time.DateTimeZone) Arrays(java.util.Arrays) PutTemplate(io.searchbox.indices.template.PutTemplate) LoggerFactory(org.slf4j.LoggerFactory) ModifyAliases(io.searchbox.indices.aliases.ModifyAliases) RequestConfig(org.apache.http.client.config.RequestConfig) TypeReferences(org.graylog2.jackson.TypeReferences) UpdateSettings(io.searchbox.indices.settings.UpdateSettings) FieldSortBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.sort.FieldSortBuilder) MaxAggregation(io.searchbox.core.search.aggregation.MaxAggregation) IndicesAdapter(org.graylog2.indexer.indices.IndicesAdapter) HealthStatus(org.graylog2.indexer.indices.HealthStatus) JestUtils(org.graylog.storage.elasticsearch6.jest.JestUtils) Indices(org.graylog2.indexer.indices.Indices) Locale(java.util.Locale) Map(java.util.Map) JsonNode(com.fasterxml.jackson.databind.JsonNode) IndexRangeStats(org.graylog2.indexer.searches.IndexRangeStats) Bulk(io.searchbox.core.Bulk) Cat(io.searchbox.core.Cat) IndexMapping(org.graylog2.indexer.IndexMapping) FilterAggregation(io.searchbox.core.search.aggregation.FilterAggregation) QueryBuilders(org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.QueryBuilders) SearchSourceBuilder.searchSource(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.builder.SearchSourceBuilder.searchSource) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) Set(java.util.Set) Health(io.searchbox.cluster.Health) AddAliasMapping(io.searchbox.indices.aliases.AddAliasMapping) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) GetAliases(io.searchbox.indices.aliases.GetAliases) RemoveAliasMapping(io.searchbox.indices.aliases.RemoveAliasMapping) PutMapping(io.searchbox.indices.mapping.PutMapping) DeleteIndex(io.searchbox.indices.DeleteIndex) Stats(io.searchbox.indices.Stats) List(java.util.List) Parameters(io.searchbox.params.Parameters) OpenIndex(io.searchbox.indices.OpenIndex) AggregationBuilders(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.aggregations.AggregationBuilders) IndexMoveResult(org.graylog2.indexer.indices.IndexMoveResult) Optional(java.util.Optional) UnsupportedEncodingException(java.io.UnsupportedEncodingException) SearchSourceBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.builder.SearchSourceBuilder) SearchResult(io.searchbox.core.SearchResult) FilterAggregationBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder) HashMap(java.util.HashMap) ElasticsearchException(org.graylog2.indexer.ElasticsearchException) SearchType(io.searchbox.params.SearchType) Iterators(com.google.common.collect.Iterators) JestClient(io.searchbox.client.JestClient) Inject(javax.inject.Inject) Strings(com.google.common.base.Strings) SortBuilders(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.sort.SortBuilders) ImmutableList(com.google.common.collect.ImmutableList) Flush(io.searchbox.indices.Flush) DeleteTemplate(io.searchbox.indices.template.DeleteTemplate) IndexStatistics(org.graylog2.indexer.indices.stats.IndexStatistics) Duration(com.github.joschi.jadconfig.util.Duration) StreamSupport(java.util.stream.StreamSupport) Nonnull(javax.annotation.Nonnull) GetSettings(io.searchbox.indices.settings.GetSettings) Logger(org.slf4j.Logger) MinAggregation(io.searchbox.core.search.aggregation.MinAggregation) Iterator(java.util.Iterator) IndexSettings(org.graylog2.indexer.indices.IndexSettings) IndexNotFoundException(org.graylog2.indexer.IndexNotFoundException) SearchScroll(io.searchbox.core.SearchScroll) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Search(io.searchbox.core.Search) AliasMapping(io.searchbox.indices.aliases.AliasMapping) DateTime(org.joda.time.DateTime) ForceMerge(io.searchbox.indices.ForceMerge) GetSingleAlias(org.graylog.storage.elasticsearch6.indices.GetSingleAlias) IOException(java.io.IOException) JestResult(io.searchbox.client.JestResult) Ints(com.google.common.primitives.Ints) CreateIndex(io.searchbox.indices.CreateIndex) BulkResult(io.searchbox.core.BulkResult) Consumer(java.util.function.Consumer) URLEncoder(java.net.URLEncoder) Collectors.toList(java.util.stream.Collectors.toList) CatResult(io.searchbox.core.CatResult) SerializationFeature(com.fasterxml.jackson.databind.SerializationFeature) CloseIndex(io.searchbox.indices.CloseIndex) GetTemplate(io.searchbox.indices.template.GetTemplate) Message(org.graylog2.plugin.Message) Collections(java.util.Collections) State(io.searchbox.cluster.State) FieldSortBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.sort.FieldSortBuilder) SearchSourceBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.builder.SearchSourceBuilder) FilterAggregationBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.aggregations.bucket.filter.FilterAggregationBuilder) SearchResult(io.searchbox.core.SearchResult) JsonNode(com.fasterxml.jackson.databind.JsonNode) ElasticsearchException(org.graylog2.indexer.ElasticsearchException) IndexMoveResult(org.graylog2.indexer.indices.IndexMoveResult) Bulk(io.searchbox.core.Bulk) BulkResult(io.searchbox.core.BulkResult) SearchScroll(io.searchbox.core.SearchScroll) Search(io.searchbox.core.Search) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) JestResult(io.searchbox.client.JestResult)

Example 27 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class AggregationEventProcessor method sourceMessagesForEvent.

@Override
public void sourceMessagesForEvent(Event event, Consumer<List<MessageSummary>> messageConsumer, long limit) throws EventProcessorException {
    if (config.series().isEmpty()) {
        if (limit <= 0) {
            return;
        }
        final EventOriginContext.ESEventOriginContext esContext = EventOriginContext.parseESContext(event.getOriginContext()).orElseThrow(() -> new EventProcessorException("Failed to parse origin context", false, eventDefinition));
        try {
            final ResultMessage message;
            message = messages.get(esContext.messageId(), esContext.indexName());
            messageConsumer.accept(Lists.newArrayList(new MessageSummary(message.getIndex(), message.getMessage())));
        } catch (IOException e) {
            throw new EventProcessorException("Failed to query origin context message", false, eventDefinition, e);
        }
    } else {
        final AtomicLong msgCount = new AtomicLong(0L);
        final MoreSearch.ScrollCallback callback = (messages, continueScrolling) -> {
            final List<MessageSummary> summaries = Lists.newArrayList();
            for (final ResultMessage resultMessage : messages) {
                if (msgCount.incrementAndGet() > limit) {
                    continueScrolling.set(false);
                    break;
                }
                final Message msg = resultMessage.getMessage();
                summaries.add(new MessageSummary(resultMessage.getIndex(), msg));
            }
            messageConsumer.accept(summaries);
        };
        ElasticsearchQueryString scrollQueryString = ElasticsearchQueryString.of(config.query());
        scrollQueryString = scrollQueryString.concatenate(groupByQueryString(event));
        LOG.debug("scrollQueryString: {}", scrollQueryString);
        final TimeRange timeRange = AbsoluteRange.create(event.getTimerangeStart(), event.getTimerangeEnd());
        moreSearch.scrollQuery(scrollQueryString.queryString(), config.streams(), config.queryParameters(), timeRange, Math.min(500, Ints.saturatedCast(limit)), callback);
    }
}
Also used : EventProcessorException(org.graylog.events.processor.EventProcessorException) MoreSearch(org.graylog.events.search.MoreSearch) LoggerFactory(org.slf4j.LoggerFactory) EventOriginContext(org.graylog.events.event.EventOriginContext) MessageSummary(org.graylog2.plugin.MessageSummary) EventConsumer(org.graylog.events.processor.EventConsumer) Assisted(com.google.inject.assistedinject.Assisted) EventProcessor(org.graylog.events.processor.EventProcessor) ResultMessage(org.graylog2.indexer.results.ResultMessage) Locale(java.util.Locale) Map(java.util.Map) Event(org.graylog.events.event.Event) AbsoluteRange(org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange) EventDefinition(org.graylog.events.processor.EventDefinition) EventProcessorException(org.graylog.events.processor.EventProcessorException) TimeRange(org.graylog2.plugin.indexer.searches.timeranges.TimeRange) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Persisted(org.graylog2.plugin.database.Persisted) Set(java.util.Set) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) MoreSearch(org.graylog.events.search.MoreSearch) EventFactory(org.graylog.events.event.EventFactory) ParameterExpansionError(org.graylog.plugins.views.search.errors.ParameterExpansionError) List(java.util.List) Stream(org.graylog2.plugin.streams.Stream) StreamService(org.graylog2.streams.StreamService) Strings(org.apache.logging.log4j.util.Strings) Optional(java.util.Optional) MoreSearch.luceneEscape(org.graylog.events.search.MoreSearch.luceneEscape) HashMap(java.util.HashMap) SearchException(org.graylog.plugins.views.search.errors.SearchException) ElasticsearchQueryString(org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString) Inject(javax.inject.Inject) DBEventProcessorStateService(org.graylog.events.processor.DBEventProcessorStateService) BooleanNumberConditionsVisitor(org.graylog.events.conditions.BooleanNumberConditionsVisitor) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) Messages(org.graylog2.indexer.messages.Messages) EventProcessorParameters(org.graylog.events.processor.EventProcessorParameters) Logger(org.slf4j.Logger) EventWithContext(org.graylog.events.event.EventWithContext) DateTime(org.joda.time.DateTime) IOException(java.io.IOException) Maps(com.google.common.collect.Maps) Ints(com.google.common.primitives.Ints) Consumer(java.util.function.Consumer) AtomicLong(java.util.concurrent.atomic.AtomicLong) EventProcessorDependencyCheck(org.graylog.events.processor.EventProcessorDependencyCheck) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Message(org.graylog2.plugin.Message) EventProcessorPreconditionException(org.graylog.events.processor.EventProcessorPreconditionException) ResultMessage(org.graylog2.indexer.results.ResultMessage) Message(org.graylog2.plugin.Message) ElasticsearchQueryString(org.graylog.plugins.views.search.elasticsearch.ElasticsearchQueryString) IOException(java.io.IOException) ResultMessage(org.graylog2.indexer.results.ResultMessage) TimeRange(org.graylog2.plugin.indexer.searches.timeranges.TimeRange) AtomicLong(java.util.concurrent.atomic.AtomicLong) EventOriginContext(org.graylog.events.event.EventOriginContext) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) MessageSummary(org.graylog2.plugin.MessageSummary)

Example 28 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class GelfChunkAggregator method checkForCompletion.

/**
 * Checks whether the presented gelf message chunk completes the incoming raw message and returns it if it does.
 * If the message isn't complete, it adds the chunk to the internal buffer and waits for more incoming messages.
 * Outdated chunks are being purged regularly.
 *
 * @param gelfMessage the gelf message chunk
 * @return null or a {@link org.graylog2.plugin.journal.RawMessage raw message} object
 */
@Nullable
private ByteBuf checkForCompletion(GELFMessage gelfMessage) {
    if (!chunks.isEmpty() && log.isDebugEnabled()) {
        log.debug("Dumping GELF chunk map [chunks for {} messages]:\n{}", chunks.size(), humanReadableChunkMap());
    }
    // TODO second parameter
    final GELFMessageChunk chunk = new GELFMessageChunk(gelfMessage, null);
    final int sequenceCount = chunk.getSequenceCount();
    final String messageId = chunk.getId();
    ChunkEntry entry = new ChunkEntry(sequenceCount, chunk.getArrival(), messageId);
    final ChunkEntry existing = chunks.putIfAbsent(messageId, entry);
    if (existing == null) {
        // add this chunk entry to the eviction set
        waitingMessages.inc();
        sortedEvictionSet.add(entry);
    } else {
        // the entry is already in the eviction set and chunk map
        entry = existing;
    }
    final int sequenceNumber = chunk.getSequenceNumber();
    if (!entry.payloadArray.compareAndSet(sequenceNumber, null, chunk)) {
        log.error("Received duplicate chunk {} for message {} from {}", sequenceNumber, messageId, gelfMessage.getSourceAddress());
        duplicateChunks.inc();
        return null;
    }
    final int chunkWatermark = entry.chunkSlotsWritten.incrementAndGet();
    if (chunkWatermark > MAX_CHUNKS) {
        getAndCleanupEntry(messageId);
        throw new IllegalStateException("Maximum number of chunks reached, discarding message");
    }
    if (chunkWatermark == sequenceCount) {
        // message is complete by chunk count, assemble and return it.
        // it might still be corrupt etc, but we've seen enough chunks
        // remove before operating on it, to avoid racing too much with the clean up job, some race is inevitable, though.
        entry = getAndCleanupEntry(messageId);
        final byte[][] allChunks = new byte[sequenceCount][];
        for (int i = 0; i < entry.payloadArray.length(); i++) {
            final GELFMessageChunk messageChunk = entry.payloadArray.get(i);
            if (messageChunk == null) {
                log.debug("Couldn't read chunk {} of message {}, skipping this chunk.", i, messageId);
            } else {
                allChunks[i] = messageChunk.getData();
            }
        }
        completeMessages.inc();
        return Unpooled.wrappedBuffer(allChunks);
    }
    // message isn't complete yet, check if we should remove the other parts as well
    if (isOutdated(entry)) {
        // chunks are outdated, the oldest came in over 5 seconds ago, clean them all up
        log.debug("Not all chunks of <{}> arrived within {}ms. Dropping chunks.", messageId, VALIDITY_PERIOD);
        expireEntry(messageId);
    }
    return null;
}
Also used : GELFMessageChunk(org.graylog2.inputs.codecs.gelf.GELFMessageChunk) Nullable(javax.annotation.Nullable)

Example 29 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class AbsoluteSearchResource method searchAbsoluteChunked.

@GET
@Timed
@ApiOperation(value = "Message search with absolute timerange.", notes = "Search for messages using an absolute timerange, specified as from/to " + "with format yyyy-MM-ddTHH:mm:ss.SSSZ (e.g. 2014-01-23T15:34:49.000Z) or yyyy-MM-dd HH:mm:ss.")
@Produces(MoreMediaTypes.TEXT_CSV)
@ApiResponses(value = { @ApiResponse(code = 400, message = "Invalid timerange parameters provided.") })
public ChunkedOutput<ScrollResult.ScrollChunk> searchAbsoluteChunked(@ApiParam(name = "query", value = "Query (Lucene syntax)", required = true) @QueryParam("query") @NotEmpty String query, @ApiParam(name = "from", value = "Timerange start. See description for date format", required = true) @QueryParam("from") @NotEmpty String from, @ApiParam(name = "to", value = "Timerange end. See description for date format", required = true) @QueryParam("to") @NotEmpty String to, @ApiParam(name = "limit", value = "Maximum number of messages to return.", required = false) @QueryParam("limit") int limit, @ApiParam(name = "offset", value = "Offset", required = false) @QueryParam("offset") int offset, @ApiParam(name = "batch_size", value = "Batch size for the backend storage export request.", required = false) @QueryParam("batch_size") @DefaultValue(DEFAULT_SCROLL_BATCH_SIZE) int batchSize, @ApiParam(name = "filter", value = "Filter", required = false) @QueryParam("filter") String filter, @ApiParam(name = "fields", value = "Comma separated list of fields to return", required = true) @QueryParam("fields") @NotEmpty String fields) {
    checkSearchPermission(filter, RestPermissions.SEARCHES_ABSOLUTE);
    final List<String> fieldList = parseFields(fields);
    final TimeRange timeRange = buildAbsoluteTimeRange(from, to);
    final ScrollResult scroll = searches.scroll(query, timeRange, limit, offset, fieldList, filter, batchSize);
    return buildChunkedOutput(scroll);
}
Also used : TimeRange(org.graylog2.plugin.indexer.searches.timeranges.TimeRange) ScrollResult(org.graylog2.indexer.results.ScrollResult) Produces(javax.ws.rs.Produces) Timed(com.codahale.metrics.annotation.Timed) GET(javax.ws.rs.GET) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Example 30 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class KeywordSearchResource method searchKeywordChunked.

@GET
@Timed
@ApiOperation(value = "Message search with keyword as timerange.", notes = "Search for messages in a timerange defined by a keyword like \"yesterday\" or \"2 weeks ago to wednesday\".")
@Produces(MoreMediaTypes.TEXT_CSV)
@ApiResponses(value = { @ApiResponse(code = 400, message = "Invalid keyword provided.") })
public ChunkedOutput<ScrollResult.ScrollChunk> searchKeywordChunked(@ApiParam(name = "query", value = "Query (Lucene syntax)", required = true) @QueryParam("query") @NotEmpty String query, @ApiParam(name = "keyword", value = "Range keyword", required = true) @QueryParam("keyword") @NotEmpty String keyword, @QueryParam("timezone") @NotEmpty String timezone, @ApiParam(name = "limit", value = "Maximum number of messages to return.", required = false) @QueryParam("limit") int limit, @ApiParam(name = "offset", value = "Offset", required = false) @QueryParam("offset") int offset, @ApiParam(name = "batch_size", value = "Batch size for the backend storage export request.", required = false) @QueryParam("batch_size") @DefaultValue(DEFAULT_SCROLL_BATCH_SIZE) int batchSize, @ApiParam(name = "filter", value = "Filter", required = false) @QueryParam("filter") String filter, @ApiParam(name = "fields", value = "Comma separated list of fields to return", required = true) @QueryParam("fields") @NotEmpty String fields) {
    checkSearchPermission(filter, RestPermissions.SEARCHES_KEYWORD);
    final List<String> fieldList = parseFields(fields);
    final TimeRange timeRange = buildKeywordTimeRange(keyword, timezone);
    final ScrollResult scroll = searches.scroll(query, timeRange, limit, offset, fieldList, filter, batchSize);
    return buildChunkedOutput(scroll);
}
Also used : TimeRange(org.graylog2.plugin.indexer.searches.timeranges.TimeRange) ScrollResult(org.graylog2.indexer.results.ScrollResult) Produces(javax.ws.rs.Produces) Timed(com.codahale.metrics.annotation.Timed) GET(javax.ws.rs.GET) ApiOperation(io.swagger.annotations.ApiOperation) ApiResponses(io.swagger.annotations.ApiResponses)

Aggregations

Message (org.graylog2.plugin.Message)41 Test (org.junit.Test)31 DateTime (org.joda.time.DateTime)17 Map (java.util.Map)15 ApiOperation (io.swagger.annotations.ApiOperation)14 Produces (javax.ws.rs.Produces)14 Timed (com.codahale.metrics.annotation.Timed)13 ApiResponses (io.swagger.annotations.ApiResponses)12 Messages (org.graylog2.plugin.Messages)12 List (java.util.List)11 GET (javax.ws.rs.GET)11 AbsoluteRange (org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange)11 ResultMessage (org.graylog2.indexer.results.ResultMessage)10 TimeRange (org.graylog2.plugin.indexer.searches.timeranges.TimeRange)10 ArrayList (java.util.ArrayList)9 Collectors (java.util.stream.Collectors)9 ResultMessageSummary (org.graylog2.rest.models.messages.responses.ResultMessageSummary)9 ImmutableMap (com.google.common.collect.ImmutableMap)8 IOException (java.io.IOException)8 Inject (javax.inject.Inject)8