Examples with IndexingRequest org.graylog2.indexer.messages.IndexingRequest used on opensource projects

Example 1 with IndexingRequest

use of org.graylog2.indexer.messages.IndexingRequest in project graylog2-server by Graylog2.

the class MessagesAdapterES6Test method bulkIndexingParsesIndexMappingErrors.

@Test
public void bulkIndexingParsesIndexMappingErrors() throws Exception {
    final String messageId = "BOOMID";
    final BulkResult jestResult = mock(BulkResult.class);
    final BulkResult.BulkResultItem bulkResultItem = new MockedBulkResult().createResultItem("index", "someindex", "message", messageId, 400, "{\"type\":\"mapper_parsing_exception\",\"reason\":\"failed to parse [http_response_code]\",\"caused_by\":{\"type\":\"number_format_exception\",\"reason\":\"For input string: \\\"FOOBAR\\\"\"}}", null, "mapper_parsing_exception", "failed to parse [http_response_code]");
    when(jestResult.isSucceeded()).thenReturn(false);
    when(jestResult.getFailedItems()).thenReturn(ImmutableList.of(bulkResultItem));
    when(jestClient.execute(any())).thenReturn(jestResult).thenThrow(new IllegalStateException("JestResult#execute should not be called twice."));
    final List<IndexingRequest> messageList = messageListWith(messageWithId(messageId));
    final List<Messages.IndexingError> result = messagesAdapter.bulkIndex(messageList);
    assertThat(result).hasSize(1).extracting(indexingError -> indexingError.message().getId(), Messages.IndexingError::errorType, Messages.IndexingError::errorMessage).containsExactly(tuple(messageId, Messages.IndexingError.ErrorType.MappingError, "failed to parse [http_response_code]"));
}

Example 2 with IndexingRequest

use of org.graylog2.indexer.messages.IndexingRequest in project graylog2-server by Graylog2.

the class MessagesAdapterES6Test method bulkIndexPropagatesIOExceptions.

@Test
public void bulkIndexPropagatesIOExceptions() throws Exception {
    when(jestClient.execute(any())).thenThrow(new IOException("Boom!"));
    final List<IndexingRequest> messageList = messageListWith(mock(Message.class));
    assertThatThrownBy(() -> messagesAdapter.bulkIndex(messageList)).isInstanceOf(IOException.class);
}

Example 3 with IndexingRequest

use of org.graylog2.indexer.messages.IndexingRequest in project graylog2-server by Graylog2.

the class MessagesAdapterES7 method bulkIndexChunked.

private List<Messages.IndexingError> bulkIndexChunked(ChunkedBulkIndexer.Chunk command) throws ChunkedBulkIndexer.EntityTooLargeException {
    final List<IndexingRequest> messageList = command.requests;
    final int offset = command.offset;
    final int chunkSize = command.size;
    if (messageList.isEmpty()) {
        return Collections.emptyList();
    }
    final Iterable<List<IndexingRequest>> chunks = Iterables.partition(messageList.subList(offset, messageList.size()), chunkSize);
    int chunkCount = 1;
    int indexedSuccessfully = 0;
    final List<Messages.IndexingError> indexFailures = new ArrayList<>();
    for (List<IndexingRequest> chunk : chunks) {
        final BulkResponse result = runBulkRequest(indexedSuccessfully, chunk);
        indexedSuccessfully += chunk.size();
        final List<BulkItemResponse> failures = extractFailures(result);
        indexFailures.addAll(indexingErrorsFrom(failures, messageList));
        logDebugInfo(messageList, offset, chunkSize, chunkCount, result, failures);
        logFailures(result, failures.size());
        chunkCount++;
    }
    return indexFailures;
}

Example 4 with IndexingRequest

use of org.graylog2.indexer.messages.IndexingRequest in project graylog2-server by Graylog2.

the class MessagesAdapterES7 method runBulkRequest.

private BulkResponse runBulkRequest(int indexedSuccessfully, List<IndexingRequest> chunk) throws ChunkedBulkIndexer.EntityTooLargeException {
    final BulkRequest bulkRequest = createBulkRequest(chunk);
    final BulkResponse result;
    try {
        result = this.client.execute((c, requestOptions) -> c.bulk(bulkRequest, requestOptions));
    } catch (ElasticsearchException e) {
        for (ElasticsearchException cause : e.guessRootCauses()) {
            if (cause.status().equals(RestStatus.REQUEST_ENTITY_TOO_LARGE)) {
                throw new ChunkedBulkIndexer.EntityTooLargeException(indexedSuccessfully, indexingErrorsFrom(chunk));
            }
        }
        throw new org.graylog2.indexer.ElasticsearchException(e);
    }
    return result;
}

Example 5 with IndexingRequest

use of org.graylog2.indexer.messages.IndexingRequest in project graylog2-server by Graylog2.

the class MessagesAdapterES6 method indexingErrorsFrom.

private List<Messages.IndexingError> indexingErrorsFrom(List<BulkResult.BulkResultItem> failedItems, List<IndexingRequest> messageList) {
    if (failedItems.isEmpty()) {
        return Collections.emptyList();
    }
    final Map<String, Indexable> messageMap = messageList.stream().map(IndexingRequest::message).distinct().collect(Collectors.toMap(Indexable::getId, Function.identity()));
    final List<Messages.IndexingError> indexFailures = new ArrayList<>(failedItems.size());
    for (BulkResult.BulkResultItem item : failedItems) {
        LOG.warn("Failed to index message: index=<{}> id=<{}> error=<{}>", item.index, item.id, item.error);
        final Indexable messageEntry = messageMap.get(item.id);
        final Messages.IndexingError indexFailure = indexingErrorFromResultItem(item, messageEntry);
        indexFailures.add(indexFailure);
    }
    return indexFailures;
}