Examples with IndexingRequest org.graylog2.indexer.messages.IndexingRequest used on opensource projects

Example 6 with IndexingRequest

use of org.graylog2.indexer.messages.IndexingRequest in project graylog2-server by Graylog2.

the class MessagesAdapterES6 method bulkIndexChunked.

private List<Messages.IndexingError> bulkIndexChunked(ChunkedBulkIndexer.Chunk command) throws ChunkedBulkIndexer.EntityTooLargeException, IOException {
    final List<IndexingRequest> messageList = command.requests;
    final int offset = command.offset;
    int chunkSize = Math.min(messageList.size(), command.size);
    final List<BulkResult.BulkResultItem> failedItems = new ArrayList<>();
    final Iterable<List<IndexingRequest>> chunks = Iterables.partition(messageList.subList(offset, messageList.size()), chunkSize);
    int chunkCount = 1;
    int indexedSuccessfully = 0;
    for (List<IndexingRequest> chunk : chunks) {
        final BulkResult result = bulkIndexChunk(chunk);
        if (result.getResponseCode() == 413) {
            throw new ChunkedBulkIndexer.EntityTooLargeException(indexedSuccessfully, indexingErrorsFrom(failedItems, messageList));
        }
        if (result.getResponseCode() >= 400) {
            throw JestUtils.specificException(() -> "Error during bulk indexing: ", result.getJsonObject().get("error"));
        }
        indexedSuccessfully += chunk.size();
        final List<BulkResult.BulkResultItem> remainingFailures = result.getFailedItems();
        failedItems.addAll(remainingFailures);
        if (LOG.isDebugEnabled()) {
            String chunkInfo = "";
            if (chunkSize != messageList.size()) {
                chunkInfo = String.format(Locale.ROOT, " (chunk %d/%d offset %d)", chunkCount, (int) Math.ceil((double) messageList.size() / chunkSize), offset);
            }
            LOG.debug("Index: Bulk indexed {} messages{}, failures: {}", result.getItems().size(), chunkInfo, failedItems.size());
        }
        if (!remainingFailures.isEmpty()) {
            LOG.error("Failed to index [{}] messages. Please check the index error log in your web interface for the reason. Error: {}", remainingFailures.size(), result.getErrorMessage());
        }
        chunkCount++;
    }
    return indexingErrorsFrom(failedItems, messageList);
}

Example 7 with IndexingRequest

use of org.graylog2.indexer.messages.IndexingRequest in project graylog2-server by Graylog2.

the class MessagesTest method bulkIndexRequests_nothingPropagatedToFailureSubmissionServiceWhenThereAreNoIndexingErrors.

@Test
public void bulkIndexRequests_nothingPropagatedToFailureSubmissionServiceWhenThereAreNoIndexingErrors() throws Exception {
    // given
    final DateTime ts = Tools.nowUTC();
    final IndexSet indexSet = mock(IndexSet.class);
    final Message message1 = message("msg-1", ts);
    final Message message2 = message("msg-2", ts);
    final List<IndexingRequest> indexingRequest = ImmutableList.of(IndexingRequest.create(indexSet, message1), IndexingRequest.create(indexSet, message2));
    when(messagesAdapter.bulkIndex(indexingRequest)).thenReturn(ImmutableList.of());
    // when
    final List<String> failureIds = messages.bulkIndexRequests(indexingRequest, false);
    // then
    assertThat(failureIds).isEmpty();
    verifyNoInteractions(failureSubmissionService);
}

Example 8 with IndexingRequest

use of org.graylog2.indexer.messages.IndexingRequest in project graylog2-server by Graylog2.

the class MessagesTest method bulkIndexRequests_allNonIndexBlockErrorsPropagatedToTheFailureSubmissionService.

@Test
public void bulkIndexRequests_allNonIndexBlockErrorsPropagatedToTheFailureSubmissionService() throws Exception {
    // given
    final DateTime ts = Tools.nowUTC();
    final IndexSet indexSet = mock(IndexSet.class);
    final Message message1 = mock(Message.class);
    final Message message2 = message("msg-2", ts);
    final Message message3 = message("msg-3", ts);
    final Message message4 = message("msg-4", ts);
    final List<IndexingRequest> indexingRequest = ImmutableList.of(IndexingRequest.create(indexSet, message1), IndexingRequest.create(indexSet, message2), IndexingRequest.create(indexSet, message3));
    when(messagesAdapter.bulkIndex(indexingRequest)).thenReturn(ImmutableList.of(Messages.IndexingError.create(message2, "msg-index", Messages.IndexingError.ErrorType.MappingError, "Some error message"), Messages.IndexingError.create(message3, "msg-index", Messages.IndexingError.ErrorType.MappingError, "Some error message"), Messages.IndexingError.create(message4, "msg-index", Messages.IndexingError.ErrorType.IndexBlocked, "Index blocked error message")));
    // when
    final List<String> failureIds = messages.bulkIndexRequests(indexingRequest, false);
    // then
    assertThat(failureIds).hasSize(2).containsExactlyInAnyOrder("msg-2", "msg-3");
    verify(failureSubmissionService, times(1)).submitIndexingErrors(indexingErrorsArgumentCaptor.capture());
    assertThat(indexingErrorsArgumentCaptor.getValue().stream().sorted(Comparator.comparing(e -> e.message().getMessageId())).collect(Collectors.toList())).satisfies(indexingErrors -> {
        assertThat(indexingErrors.get(0)).satisfies(indexingError -> {
            assertThat(indexingError.errorType()).isEqualTo(Messages.IndexingError.ErrorType.MappingError);
            assertThat(indexingError.message()).isEqualTo(message2);
        });
        assertThat(indexingErrors.get(1)).satisfies(indexingError -> {
            assertThat(indexingError.errorType()).isEqualTo(Messages.IndexingError.ErrorType.MappingError);
            assertThat(indexingError.message()).isEqualTo(message3);
        });
    });
}

Example 9 with IndexingRequest

use of org.graylog2.indexer.messages.IndexingRequest in project graylog2-server by Graylog2.

the class MessagesAdapterES6 method bulkIndexChunk.

private BulkResult bulkIndexChunk(List<IndexingRequest> chunk) throws IOException {
    final Bulk.Builder bulk = new Bulk.Builder();
    for (IndexingRequest entry : chunk) {
        final Indexable message = entry.message();
        bulk.addAction(new Index.Builder(message.toElasticSearchObject(objectMapper, invalidTimestampMeter)).index(entry.indexSet().getWriteIndexAlias()).type(IndexMapping.TYPE_MESSAGE).id(message.getId()).build());
    }
    return runBulkRequest(bulk.build(), chunk.size());
}

Example 10 with IndexingRequest

use of org.graylog2.indexer.messages.IndexingRequest in project graylog2-server by Graylog2.

the class MessagesAdapterES6Test method bulkIndexingParsesPrimaryShardUnavailableErrors.

@Test
public void bulkIndexingParsesPrimaryShardUnavailableErrors() throws Exception {
    final String messageId = "BOOMID";
    final BulkResult failedJestResult = mock(BulkResult.class);
    final BulkResult.BulkResultItem bulkResultItem = new MockedBulkResult().createResultItem("index", "someindex", "message", messageId, 400, "{\"type\":\"unavailable_shards_exception\",\"reason\":\"primary shard is not active\"\"}}", null, "unavailable_shards_exception", "primary shard is not active");
    when(failedJestResult.isSucceeded()).thenReturn(false);
    when(failedJestResult.getFailedItems()).thenReturn(ImmutableList.of(bulkResultItem));
    when(jestClient.execute(any())).thenReturn(failedJestResult).thenThrow(new IllegalStateException("JestResult#execute should not be called twice."));
    final List<IndexingRequest> messageList = messageListWith(messageWithId(messageId));
    final List<Messages.IndexingError> result = messagesAdapter.bulkIndex(messageList);
    assertThat(result).hasSize(1).extracting(indexingError -> indexingError.message().getId(), Messages.IndexingError::errorType, Messages.IndexingError::errorMessage).containsExactly(tuple(messageId, Messages.IndexingError.ErrorType.IndexBlocked, "primary shard is not active"));
}