Examples with Messages org.graylog2.indexer.messages.Messages used on opensource projects

Example 1 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class BenchmarkOutput method write.

@Override
public void write(List<Message> messages) throws Exception {
    long maxOffset = Long.MIN_VALUE;
    for (final Message message : messages) {
        maxOffset = Math.max(message.getJournalOffset(), maxOffset);
    }
    journal.markJournalOffsetCommitted(maxOffset);
    messagesWritten.mark(messages.size());
}

Example 2 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class KeywordSearchResource method searchKeyword.

@GET
@Timed
@ApiOperation(value = "Message search with keyword as timerange.", notes = "Search for messages in a timerange defined by a keyword like \"yesterday\" or \"2 weeks ago to wednesday\".")
@Produces(MediaType.APPLICATION_JSON)
@ApiResponses(value = { @ApiResponse(code = 400, message = "Invalid keyword provided.") })
public SearchResponse searchKeyword(@ApiParam(name = "query", value = "Query (Lucene syntax)", required = true) @QueryParam("query") @NotEmpty String query, @ApiParam(name = "keyword", value = "Range keyword", required = true) @QueryParam("keyword") @NotEmpty String keyword, @ApiParam(name = "limit", value = "Maximum number of messages to return.", required = false) @QueryParam("limit") int limit, @ApiParam(name = "offset", value = "Offset", required = false) @QueryParam("offset") int offset, @ApiParam(name = "filter", value = "Filter", required = false) @QueryParam("filter") String filter, @ApiParam(name = "fields", value = "Comma separated list of fields to return", required = false) @QueryParam("fields") String fields, @ApiParam(name = "sort", value = "Sorting (field:asc / field:desc)", required = false) @QueryParam("sort") String sort, @ApiParam(name = "decorate", value = "Run decorators on search result", required = false) @QueryParam("decorate") @DefaultValue("true") boolean decorate) {
    checkSearchPermission(filter, RestPermissions.SEARCHES_KEYWORD);
    final List<String> fieldList = parseOptionalFields(fields);
    final Sorting sorting = buildSorting(sort);
    final TimeRange timeRange = buildKeywordTimeRange(keyword);
    final SearchesConfig searchesConfig = SearchesConfig.builder().query(query).filter(filter).fields(fieldList).range(timeRange).limit(limit).offset(offset).sorting(sorting).build();
    final Optional<String> streamId = Searches.extractStreamId(filter);
    try {
        return buildSearchResponse(searches.search(searchesConfig), timeRange, decorate, streamId);
    } catch (SearchPhaseExecutionException e) {
        throw createRequestExceptionForParseFailure(query, e);
    }
}

Example 3 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class Messages method propagateFailure.

private void propagateFailure(BulkItemResponse[] items, List<Map.Entry<IndexSet, Message>> messageList, String errorMessage) {
    final List<IndexFailure> indexFailures = new LinkedList<>();
    for (BulkItemResponse item : items) {
        if (item.isFailed()) {
            LOG.trace("Failed to index message: {}", item.getFailureMessage());
            // Write failure to index_failures.
            final BulkItemResponse.Failure f = item.getFailure();
            final Map.Entry<IndexSet, Message> messageEntry = messageList.get(item.getItemId());
            final Map<String, Object> doc = ImmutableMap.<String, Object>builder().put("letter_id", item.getId()).put("index", f.getIndex()).put("type", f.getType()).put("message", f.getMessage()).put("timestamp", messageEntry.getValue().getTimestamp()).build();
            indexFailures.add(new IndexFailureImpl(doc));
        }
    }
    LOG.error("Failed to index [{}] messages. Please check the index error log in your web interface for the reason. Error: {}", indexFailures.size(), errorMessage);
    try {
        // TODO: Magic number
        indexFailureQueue.offer(indexFailures, 25, TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        LOG.warn("Couldn't save index failures.", e);
    }
}

Example 4 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class Messages method bulkIndex.

public boolean bulkIndex(final List<Map.Entry<IndexSet, Message>> messageList) {
    if (messageList.isEmpty()) {
        return true;
    }
    final BulkRequestBuilder requestBuilder = c.prepareBulk().setConsistencyLevel(WriteConsistencyLevel.ONE);
    for (Map.Entry<IndexSet, Message> entry : messageList) {
        requestBuilder.add(buildIndexRequest(entry.getKey().getWriteIndexAlias(), entry.getValue().toElasticSearchObject(invalidTimestampMeter), entry.getValue().getId()));
    }
    final BulkResponse response = runBulkRequest(requestBuilder.request());
    LOG.debug("Index: Bulk indexed {} messages, took {} ms, failures: {}", response.getItems().length, response.getTookInMillis(), response.hasFailures());
    if (response.hasFailures()) {
        propagateFailure(response.getItems(), messageList, response.buildFailureMessage());
    }
    return !response.hasFailures();
}

Example 5 with Messages

use of org.graylog2.indexer.messages.Messages in project graylog2-server by Graylog2.

the class IndicesGetAllMessageFieldsTest method setUp.

@Before
public void setUp() throws Exception {
    elasticsearchRule.getDatabaseOperation().deleteAll();
    indices = new Indices(client, new IndexMapping(), new Messages(client, new MetricRegistry()), mock(NodeId.class), new NullAuditEventSender());
}