Examples with BulkResponse org.elasticsearch.action.bulk.BulkResponse used on opensource projects

Example 6 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project elasticsearch by elastic.

the class TokenCountFieldMapperIntegrationIT method init.

private void init() throws IOException {
    prepareCreate("test").addMapping("test", jsonBuilder().startObject().startObject("test").startObject("properties").startObject("foo").field("type", "text").field("store", storeCountedFields).field("analyzer", "simple").startObject("fields").startObject("token_count").field("type", "token_count").field("analyzer", "standard").field("store", true).endObject().startObject("token_count_unstored").field("type", "token_count").field("analyzer", "standard").endObject().startObject("token_count_with_doc_values").field("type", "token_count").field("analyzer", "standard").field("doc_values", true).endObject().endObject().endObject().endObject().endObject().endObject()).get();
    ensureGreen();
    assertEquals(DocWriteResponse.Result.CREATED, prepareIndex("single", "I have four terms").get().getResult());
    BulkResponse bulk = client().prepareBulk().add(prepareIndex("bulk1", "bulk three terms")).add(prepareIndex("bulk2", "this has five bulk terms")).get();
    assertFalse(bulk.buildFailureMessage(), bulk.hasFailures());
    assertEquals(DocWriteResponse.Result.CREATED, prepareIndex("multi", "two terms", "wow now I have seven lucky terms").get().getResult());
    bulk = client().prepareBulk().add(prepareIndex("multibulk1", "one", "oh wow now I have eight unlucky terms")).add(prepareIndex("multibulk2", "six is a bunch of terms", "ten!  ten terms is just crazy!  too many too count!")).get();
    assertFalse(bulk.buildFailureMessage(), bulk.hasFailures());
    assertThat(refresh().getFailedShards(), equalTo(0));
}

Example 7 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project elasticsearch by elastic.

the class ExceptionRetryIT method testRetryDueToExceptionOnNetworkLayer.

/**
     * Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally
     * failing. If auto generated ids are used this must not lead to duplicate ids
     * see https://github.com/elastic/elasticsearch/issues/8788
     */
public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {
    final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
    int numDocs = scaledRandomIntBetween(100, 1000);
    Client client = internalCluster().coordOnlyNodeClient();
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    NodeStats unluckyNode = randomFrom(nodeStats.getNodes().stream().filter((s) -> s.getNode().isDataNode()).collect(Collectors.toList()));
    assertAcked(client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 1).put("index.number_of_shards", 5)));
    ensureGreen("index");
    logger.info("unlucky node: {}", unluckyNode.getNode());
    //create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.
    for (NodeStats dataNode : nodeStats.getNodes()) {
        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
        mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {

            @Override
            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                super.sendRequest(connection, requestId, action, request, options);
                if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
                    logger.debug("Throw ConnectTransportException");
                    throw new ConnectTransportException(connection.getNode(), action);
                }
            }
        });
    }
    BulkRequestBuilder bulkBuilder = client.prepareBulk();
    for (int i = 0; i < numDocs; i++) {
        XContentBuilder doc = null;
        doc = jsonBuilder().startObject().field("foo", "bar").endObject();
        bulkBuilder.add(client.prepareIndex("index", "type").setSource(doc));
    }
    BulkResponse response = bulkBuilder.get();
    if (response.hasFailures()) {
        for (BulkItemResponse singleIndexRespons : response.getItems()) {
            if (singleIndexRespons.isFailed()) {
                fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage());
            }
        }
    }
    refresh();
    SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
    Set<String> uniqueIds = new HashSet();
    long dupCounter = 0;
    boolean found_duplicate_already = false;
    for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
        if (!uniqueIds.add(searchResponse.getHits().getHits()[i].getId())) {
            if (!found_duplicate_already) {
                SearchResponse dupIdResponse = client().prepareSearch("index").setQuery(termQuery("_id", searchResponse.getHits().getHits()[i].getId())).setExplain(true).get();
                assertThat(dupIdResponse.getHits().getTotalHits(), greaterThan(1L));
                logger.info("found a duplicate id:");
                for (SearchHit hit : dupIdResponse.getHits()) {
                    logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
                }
                logger.info("will not print anymore in case more duplicates are found.");
                found_duplicate_already = true;
            }
            dupCounter++;
        }
    }
    assertSearchResponse(searchResponse);
    assertThat(dupCounter, equalTo(0L));
    assertHitCount(searchResponse, numDocs);
    IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setSegments(true).get();
    IndexStats indexStats = index.getIndex("index");
    long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
    for (IndexShardStats indexShardStats : indexStats) {
        for (ShardStats shardStats : indexShardStats) {
            SegmentsStats segments = shardStats.getStats().getSegments();
            maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp());
        }
    }
    assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get());
    assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1);
}

Example 8 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project elasticsearch by elastic.

the class IngestClientIT method testBulkWithIngestFailures.

public void testBulkWithIngestFailures() throws Exception {
    createIndex("index");
    BytesReference source = jsonBuilder().startObject().field("description", "my_pipeline").startArray("processors").startObject().startObject("test").endObject().endObject().endArray().endObject().bytes();
    PutPipelineRequest putPipelineRequest = new PutPipelineRequest("_id", source, XContentType.JSON);
    client().admin().cluster().putPipeline(putPipelineRequest).get();
    int numRequests = scaledRandomIntBetween(32, 128);
    BulkRequest bulkRequest = new BulkRequest();
    for (int i = 0; i < numRequests; i++) {
        IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i)).setPipeline("_id");
        indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "fail", i % 2 == 0);
        bulkRequest.add(indexRequest);
    }
    BulkResponse response = client().bulk(bulkRequest).actionGet();
    assertThat(response.getItems().length, equalTo(bulkRequest.requests().size()));
    for (int i = 0; i < bulkRequest.requests().size(); i++) {
        BulkItemResponse itemResponse = response.getItems()[i];
        if (i % 2 == 0) {
            BulkItemResponse.Failure failure = itemResponse.getFailure();
            ElasticsearchException compoundProcessorException = (ElasticsearchException) failure.getCause();
            assertThat(compoundProcessorException.getRootCause().getMessage(), equalTo("test processor failed"));
        } else {
            IndexResponse indexResponse = itemResponse.getResponse();
            assertThat("Expected a successful response but found failure [" + itemResponse.getFailure() + "].", itemResponse.isFailed(), is(false));
            assertThat(indexResponse, notNullValue());
            assertThat(indexResponse.getId(), equalTo(Integer.toString(i)));
            assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
        }
    }
}

Example 9 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project elasticsearch by elastic.

the class SimpleIndexTemplateIT method testStrictAliasParsingInIndicesCreatedViaTemplates.

public void testStrictAliasParsingInIndicesCreatedViaTemplates() throws Exception {
    // Indexing into a should succeed, because the field mapping for field 'field' is defined in the test mapping.
    client().admin().indices().preparePutTemplate("template1").setPatterns(Collections.singletonList("a*")).setOrder(0).addMapping("test", "field", "type=text").addAlias(new Alias("alias1").filter(termQuery("field", "value"))).get();
    // Indexing into b should succeed, because the field mapping for field 'field' is defined in the _default_ mapping and
    //  the test type exists.
    client().admin().indices().preparePutTemplate("template2").setPatterns(Collections.singletonList("b*")).setOrder(0).addMapping("_default_", "field", "type=text").addMapping("test").addAlias(new Alias("alias2").filter(termQuery("field", "value"))).get();
    // Indexing into c should succeed, because the field mapping for field 'field' is defined in the _default_ mapping.
    client().admin().indices().preparePutTemplate("template3").setPatterns(Collections.singletonList("c*")).setOrder(0).addMapping("_default_", "field", "type=text").addAlias(new Alias("alias3").filter(termQuery("field", "value"))).get();
    // Indexing into d index should fail, since there is field with name 'field' in the mapping
    client().admin().indices().preparePutTemplate("template4").setPatterns(Collections.singletonList("d*")).setOrder(0).addAlias(new Alias("alias4").filter(termQuery("field", "value"))).get();
    client().prepareIndex("a1", "test", "test").setSource("{}", XContentType.JSON).get();
    BulkResponse response = client().prepareBulk().add(new IndexRequest("a2", "test", "test").source("{}", XContentType.JSON)).get();
    assertThat(response.hasFailures(), is(false));
    assertThat(response.getItems()[0].isFailed(), equalTo(false));
    assertThat(response.getItems()[0].getIndex(), equalTo("a2"));
    assertThat(response.getItems()[0].getType(), equalTo("test"));
    assertThat(response.getItems()[0].getId(), equalTo("test"));
    assertThat(response.getItems()[0].getVersion(), equalTo(1L));
    client().prepareIndex("b1", "test", "test").setSource("{}", XContentType.JSON).get();
    response = client().prepareBulk().add(new IndexRequest("b2", "test", "test").source("{}", XContentType.JSON)).get();
    assertThat(response.hasFailures(), is(false));
    assertThat(response.getItems()[0].isFailed(), equalTo(false));
    assertThat(response.getItems()[0].getIndex(), equalTo("b2"));
    assertThat(response.getItems()[0].getType(), equalTo("test"));
    assertThat(response.getItems()[0].getId(), equalTo("test"));
    assertThat(response.getItems()[0].getVersion(), equalTo(1L));
    client().prepareIndex("c1", "test", "test").setSource("{}", XContentType.JSON).get();
    response = client().prepareBulk().add(new IndexRequest("c2", "test", "test").source("{}", XContentType.JSON)).get();
    assertThat(response.hasFailures(), is(false));
    assertThat(response.getItems()[0].isFailed(), equalTo(false));
    assertThat(response.getItems()[0].getIndex(), equalTo("c2"));
    assertThat(response.getItems()[0].getType(), equalTo("test"));
    assertThat(response.getItems()[0].getId(), equalTo("test"));
    assertThat(response.getItems()[0].getVersion(), equalTo(1L));
    // Before 2.0 alias filters were parsed at alias creation time, in order
    // for filters to work correctly ES required that fields mentioned in those
    // filters exist in the mapping.
    // From 2.0 and higher alias filters are parsed at request time and therefor
    // fields mentioned in filters don't need to exist in the mapping.
    // So the aliases defined in the index template for this index will not fail
    // even though the fields in the alias fields don't exist yet and indexing into
    // an index that doesn't exist yet will succeed
    client().prepareIndex("d1", "test", "test").setSource("{}", XContentType.JSON).get();
    response = client().prepareBulk().add(new IndexRequest("d2", "test", "test").source("{}", XContentType.JSON)).get();
    assertThat(response.hasFailures(), is(false));
    assertThat(response.getItems()[0].isFailed(), equalTo(false));
    assertThat(response.getItems()[0].getId(), equalTo("test"));
    assertThat(response.getItems()[0].getVersion(), equalTo(1L));
}

Example 10 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project elasticsearch by elastic.

the class CrudIT method testBulk.

public void testBulk() throws IOException {
    int nbItems = randomIntBetween(10, 100);
    boolean[] errors = new boolean[nbItems];
    XContentType xContentType = randomFrom(XContentType.JSON, XContentType.SMILE);
    BulkRequest bulkRequest = new BulkRequest();
    for (int i = 0; i < nbItems; i++) {
        String id = String.valueOf(i);
        boolean erroneous = randomBoolean();
        errors[i] = erroneous;
        DocWriteRequest.OpType opType = randomFrom(DocWriteRequest.OpType.values());
        if (opType == DocWriteRequest.OpType.DELETE) {
            if (erroneous == false) {
                assertEquals(RestStatus.CREATED, highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status());
            }
            DeleteRequest deleteRequest = new DeleteRequest("index", "test", id);
            bulkRequest.add(deleteRequest);
        } else {
            BytesReference source = XContentBuilder.builder(xContentType.xContent()).startObject().field("id", i).endObject().bytes();
            if (opType == DocWriteRequest.OpType.INDEX) {
                IndexRequest indexRequest = new IndexRequest("index", "test", id).source(source, xContentType);
                if (erroneous) {
                    indexRequest.version(12L);
                }
                bulkRequest.add(indexRequest);
            } else if (opType == DocWriteRequest.OpType.CREATE) {
                IndexRequest createRequest = new IndexRequest("index", "test", id).source(source, xContentType).create(true);
                if (erroneous) {
                    assertEquals(RestStatus.CREATED, highLevelClient().index(createRequest).status());
                }
                bulkRequest.add(createRequest);
            } else if (opType == DocWriteRequest.OpType.UPDATE) {
                UpdateRequest updateRequest = new UpdateRequest("index", "test", id).doc(new IndexRequest().source(source, xContentType));
                if (erroneous == false) {
                    assertEquals(RestStatus.CREATED, highLevelClient().index(new IndexRequest("index", "test", id).source("field", -1)).status());
                }
                bulkRequest.add(updateRequest);
            }
        }
    }
    BulkResponse bulkResponse = execute(bulkRequest, highLevelClient()::bulk, highLevelClient()::bulkAsync);
    assertEquals(RestStatus.OK, bulkResponse.status());
    assertTrue(bulkResponse.getTookInMillis() > 0);
    assertEquals(nbItems, bulkResponse.getItems().length);
    for (int i = 0; i < nbItems; i++) {
        BulkItemResponse bulkItemResponse = bulkResponse.getItems()[i];
        assertEquals(i, bulkItemResponse.getItemId());
        assertEquals("index", bulkItemResponse.getIndex());
        assertEquals("test", bulkItemResponse.getType());
        assertEquals(String.valueOf(i), bulkItemResponse.getId());
        DocWriteRequest.OpType requestOpType = bulkRequest.requests().get(i).opType();
        if (requestOpType == DocWriteRequest.OpType.INDEX || requestOpType == DocWriteRequest.OpType.CREATE) {
            assertEquals(errors[i], bulkItemResponse.isFailed());
            assertEquals(errors[i] ? RestStatus.CONFLICT : RestStatus.CREATED, bulkItemResponse.status());
        } else if (requestOpType == DocWriteRequest.OpType.UPDATE) {
            assertEquals(errors[i], bulkItemResponse.isFailed());
            assertEquals(errors[i] ? RestStatus.NOT_FOUND : RestStatus.OK, bulkItemResponse.status());
        } else if (requestOpType == DocWriteRequest.OpType.DELETE) {
            assertFalse(bulkItemResponse.isFailed());
            assertEquals(errors[i] ? RestStatus.NOT_FOUND : RestStatus.OK, bulkItemResponse.status());
        }
    }
}