Examples with BulkItemResponse org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse used on opensource projects

Example 6 with BulkItemResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project elasticsearch by elastic.

the class IngestClientIT method testBulkWithIngestFailures.

public void testBulkWithIngestFailures() throws Exception {
    createIndex("index");
    BytesReference source = jsonBuilder().startObject().field("description", "my_pipeline").startArray("processors").startObject().startObject("test").endObject().endObject().endArray().endObject().bytes();
    PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON);
    client().admin().cluster().putPipeline(putPipelineRequest).get();
    int numRequests = scaledRandomIntBetween(32, 128);
    BulkRequest bulkRequest = new BulkRequest();
    for (int i = 0; i < numRequests; i++) {
        IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i)).setPipeline("_id");
        indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "fail", i % 2 == 0);
        bulkRequest.add(indexRequest);
    }
    BulkResponse response = client().bulk(bulkRequest).actionGet();
    assertThat(response.getItems().length, equalTo(bulkRequest.requests().size()));
    for (int i = 0; i < bulkRequest.requests().size(); i++) {
        BulkItemResponse itemResponse = response.getItems()[i];
        if (i % 2 == 0) {
            BulkItemResponse.Failure failure = itemResponse.getFailure();
            ElasticsearchException compoundProcessorException = (ElasticsearchException) failure.getCause();
            assertThat(compoundProcessorException.getRootCause().getMessage(), equalTo("test processor failed"));
        } else {
            IndexResponse indexResponse = itemResponse.getResponse();
            assertThat("Expected a successful response but found failure [" + itemResponse.getFailure() + "].", itemResponse.isFailed(), is(false));
            assertThat(indexResponse, notNullValue());
            assertThat(indexResponse.getId(), equalTo(Integer.toString(i)));
            assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
        }
    }
}

Example 7 with BulkItemResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project elasticsearch by elastic.

the class CrudIT method testBulk.

public void testBulk() throws IOException {
    int nbItems = randomIntBetween(10, 100);
    boolean[] errors = new boolean[nbItems];
    XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
    BulkRequest bulkRequest = new BulkRequest();
    for (int i = 0; i < nbItems; i++) {
        String id = String.valueOf(i);
        boolean erroneous = randomBoolean();
        errors[i] = erroneous;
        DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
        if (opType == DocWriteRequest.OpType.DELETE) {
            if (erroneous == false) {
                assertEquals(RestStatus.CREATED, highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status());
            }
            DeleteRequest deleteRequest = new DeleteRequest("index", "test", id);
            bulkRequest.add(deleteRequest);
        } else {
            BytesReference source = XContentBuilder.builder(xContentType.xContent()).startObject().field("id", i).endObject().bytes();
            if (opType == DocWriteRequest.OpType.INDEX) {
                IndexRequest indexRequest = new IndexRequest("index", "test", id).source(source, xContentType);
                if (erroneous) {
                    indexRequest.version(12L);
                }
                bulkRequest.add(indexRequest);
            } else if (opType == DocWriteRequest.OpType.CREATE) {
                IndexRequest createRequest = new IndexRequest("index", "test", id).source(source, xContentType).create(true);
                if (erroneous) {
                    assertEquals(RestStatus.CREATED, highLevelClient().index(createRequest).status());
                }
                bulkRequest.add(createRequest);
            } else if (opType == DocWriteRequest.OpType.UPDATE) {
                UpdateRequest updateRequest = new UpdateRequest("index", "test", id).doc(new IndexRequest().source(source, xContentType));
                if (erroneous == false) {
                    assertEquals(RestStatus.CREATED, highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status());
                }
                bulkRequest.add(updateRequest);
            }
        }
    }
    BulkResponse bulkResponse = execute(bulkRequest, highLevelClient()::bulk, highLevelClient()::bulkAsync);
    assertEquals(RestStatus.OK, bulkResponse.status());
    assertTrue(bulkResponse.getTookInMillis() > 0);
    assertEquals(nbItems, bulkResponse.getItems().length);
    for (int i = 0; i < nbItems; i++) {
        BulkItemResponse bulkItemResponse = bulkResponse.getItems()[i];
        assertEquals(i, bulkItemResponse.getItemId());
        assertEquals("index", bulkItemResponse.getIndex());
        assertEquals("test", bulkItemResponse.getType());
        assertEquals(String.valueOf(i), bulkItemResponse.getId());
        DocWriteRequest.OpType requestOpType = bulkRequest.requests().get(i).opType();
        if (requestOpType == DocWriteRequest.OpType.INDEX || requestOpType == DocWriteRequest.OpType.CREATE) {
            assertEquals(errors[i], bulkItemResponse.isFailed());
            assertEquals(errors[i] ? RestStatus.CONFLICT : RestStatus.CREATED, bulkItemResponse.status());
        } else if (requestOpType == DocWriteRequest.OpType.UPDATE) {
            assertEquals(errors[i], bulkItemResponse.isFailed());
            assertEquals(errors[i] ? RestStatus.NOT_FOUND : RestStatus.OK, bulkItemResponse.status());
        } else if (requestOpType == DocWriteRequest.OpType.DELETE) {
            assertFalse(bulkItemResponse.isFailed());
            assertEquals(errors[i] ? RestStatus.NOT_FOUND : RestStatus.OK, bulkItemResponse.status());
        }
    }
}

Example 8 with BulkItemResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project elasticsearch by elastic.

the class SimpleRoutingIT method testRequiredRoutingBulk.

public void testRequiredRoutingBulk() throws Exception {
    client().admin().indices().prepareCreate("test").addAlias(new Alias("alias")).addMapping("type1", XContentFactory.jsonBuilder().startObject().startObject("type1").startObject("_routing").field("required", true).endObject().endObject().endObject()).execute().actionGet();
    ensureGreen();
    {
        BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1").source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet();
        assertThat(bulkResponse.getItems().length, equalTo(1));
        assertThat(bulkResponse.hasFailures(), equalTo(true));
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            assertThat(bulkItemResponse.isFailed(), equalTo(true));
            assertThat(bulkItemResponse.getOpType(), equalTo(DocWriteRequest.OpType.INDEX));
            assertThat(bulkItemResponse.getFailure().getStatus(), equalTo(RestStatus.BAD_REQUEST));
            assertThat(bulkItemResponse.getFailure().getCause(), instanceOf(RoutingMissingException.class));
            assertThat(bulkItemResponse.getFailureMessage(), containsString("routing is required for [test]/[type1]/[1]"));
        }
    }
    {
        BulkResponse bulkResponse = client().prepareBulk().add(Requests.indexRequest(indexOrAlias()).type("type1").id("1").routing("0").source(Requests.INDEX_CONTENT_TYPE, "field", "value")).execute().actionGet();
        assertThat(bulkResponse.hasFailures(), equalTo(false));
    }
    {
        BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1").doc(Requests.INDEX_CONTENT_TYPE, "field", "value2")).execute().actionGet();
        assertThat(bulkResponse.getItems().length, equalTo(1));
        assertThat(bulkResponse.hasFailures(), equalTo(true));
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            assertThat(bulkItemResponse.isFailed(), equalTo(true));
            assertThat(bulkItemResponse.getOpType(), equalTo(DocWriteRequest.OpType.UPDATE));
            assertThat(bulkItemResponse.getFailure().getStatus(), equalTo(RestStatus.BAD_REQUEST));
            assertThat(bulkItemResponse.getFailure().getCause(), instanceOf(RoutingMissingException.class));
            assertThat(bulkItemResponse.getFailureMessage(), containsString("routing is required for [test]/[type1]/[1]"));
        }
    }
    {
        BulkResponse bulkResponse = client().prepareBulk().add(new UpdateRequest(indexOrAlias(), "type1", "1").doc(Requests.INDEX_CONTENT_TYPE, "field", "value2").routing("0")).execute().actionGet();
        assertThat(bulkResponse.hasFailures(), equalTo(false));
    }
    {
        BulkResponse bulkResponse = client().prepareBulk().add(Requests.deleteRequest(indexOrAlias()).type("type1").id("1")).execute().actionGet();
        assertThat(bulkResponse.getItems().length, equalTo(1));
        assertThat(bulkResponse.hasFailures(), equalTo(true));
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            assertThat(bulkItemResponse.isFailed(), equalTo(true));
            assertThat(bulkItemResponse.getOpType(), equalTo(DocWriteRequest.OpType.DELETE));
            assertThat(bulkItemResponse.getFailure().getStatus(), equalTo(RestStatus.BAD_REQUEST));
            assertThat(bulkItemResponse.getFailure().getCause(), instanceOf(RoutingMissingException.class));
            assertThat(bulkItemResponse.getFailureMessage(), containsString("routing is required for [test]/[type1]/[1]"));
        }
    }
    {
        BulkResponse bulkResponse = client().prepareBulk().add(Requests.deleteRequest(indexOrAlias()).type("type1").id("1").routing("0")).execute().actionGet();
        assertThat(bulkResponse.getItems().length, equalTo(1));
        assertThat(bulkResponse.hasFailures(), equalTo(false));
    }
}

Example 9 with BulkItemResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project graylog2-server by Graylog2.

the class Messages method propagateFailure.

private void propagateFailure(BulkItemResponse[] items, List<Map.Entry<IndexSet, Message>> messageList, String errorMessage) {
    final List<IndexFailure> indexFailures = new LinkedList<>();
    for (BulkItemResponse item : items) {
        if (item.isFailed()) {
            LOG.trace("Failed to index message: {}", item.getFailureMessage());
            // Write failure to index_failures.
            final BulkItemResponse.Failure f = item.getFailure();
            final Map.Entry<IndexSet, Message> messageEntry = messageList.get(item.getItemId());
            final Map<String, Object> doc = ImmutableMap.<String, Object>builder().put("letter_id", item.getId()).put("index", f.getIndex()).put("type", f.getType()).put("message", f.getMessage()).put("timestamp", messageEntry.getValue().getTimestamp()).build();
            indexFailures.add(new IndexFailureImpl(doc));
        }
    }
    LOG.error("Failed to index [{}] messages. Please check the index error log in your web interface for the reason. Error: {}", indexFailures.size(), errorMessage);
    try {
        // TODO: Magic number
        indexFailureQueue.offer(indexFailures, 25, TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        LOG.warn("Couldn't save index failures.", e);
    }
}

Example 10 with BulkItemResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project pentaho-kettle by pentaho.

the class ElasticSearchBulk method handleResponse.

/**
 * @param response
 * @return <code>true</code> if no errors
 */
private boolean handleResponse(BulkResponse response) {
    boolean hasErrors = response.hasFailures();
    if (hasErrors) {
        logError(response.buildFailureMessage());
    }
    int errorsInBatch = 0;
    if (hasErrors || useOutput) {
        for (BulkItemResponse item : response) {
            if (item.isFailed()) {
                // log
                logDetailed(item.getFailureMessage());
                errorsInBatch++;
                if (getStepMeta().isDoingErrorHandling()) {
                    rejectRow(item.getItemId(), item.getFailureMessage());
                }
            } else if (useOutput) {
                if (idOutFieldName != null) {
                    addIdToRow(item.getId(), item.getItemId());
                }
                echoRow(item.getItemId());
            }
        }
    }
    numberOfErrors += errorsInBatch;
    setErrors(numberOfErrors);
    int linesOK = currentRequest.numberOfActions() - errorsInBatch;
    if (useOutput) {
        setLinesOutput(getLinesOutput() + linesOK);
    } else {
        setLinesWritten(getLinesWritten() + linesOK);
    }
    return !hasErrors;
}