Examples with BulkResponse org.elasticsearch.action.bulk.BulkResponse used on opensource projects

Example 61 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project titan by thinkaurelius.

the class ElasticSearchIndex method mutate.

@Override
public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
    BulkRequestBuilder brb = client.prepareBulk();
    int bulkrequests = 0;
    try {
        for (Map.Entry<String, Map<String, IndexMutation>> stores : mutations.entrySet()) {
            String storename = stores.getKey();
            for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
                String docid = entry.getKey();
                IndexMutation mutation = entry.getValue();
                assert mutation.isConsolidated();
                Preconditions.checkArgument(!(mutation.isNew() && mutation.isDeleted()));
                Preconditions.checkArgument(!mutation.isNew() || !mutation.hasDeletions());
                Preconditions.checkArgument(!mutation.isDeleted() || !mutation.hasAdditions());
                //Deletions first
                if (mutation.hasDeletions()) {
                    if (mutation.isDeleted()) {
                        log.trace("Deleting entire document {}", docid);
                        brb.add(new DeleteRequest(indexName, storename, docid));
                    } else {
                        String script = getDeletionScript(informations, storename, mutation);
                        brb.add(client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE));
                        log.trace("Adding script {}", script);
                    }
                    bulkrequests++;
                }
                if (mutation.hasAdditions()) {
                    int ttl = mutation.determineTTL();
                    if (mutation.isNew()) {
                        //Index
                        log.trace("Adding entire document {}", docid);
                        brb.add(new IndexRequest(indexName, storename, docid).source(getNewDocument(mutation.getAdditions(), informations.get(storename), ttl)));
                    } else {
                        Preconditions.checkArgument(ttl == 0, "Elasticsearch only supports TTL on new documents [%s]", docid);
                        boolean needUpsert = !mutation.hasDeletions();
                        String script = getAdditionScript(informations, storename, mutation);
                        UpdateRequestBuilder update = client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE);
                        if (needUpsert) {
                            XContentBuilder doc = getNewDocument(mutation.getAdditions(), informations.get(storename), ttl);
                            update.setUpsert(doc);
                        }
                        brb.add(update);
                        log.trace("Adding script {}", script);
                    }
                    bulkrequests++;
                }
            }
        }
        if (bulkrequests > 0) {
            BulkResponse bulkItemResponses = brb.execute().actionGet();
            if (bulkItemResponses.hasFailures()) {
                boolean actualFailure = false;
                for (BulkItemResponse response : bulkItemResponses.getItems()) {
                    //The document may have been deleted, which is OK
                    if (response.isFailed() && response.getFailure().getStatus() != RestStatus.NOT_FOUND) {
                        log.error("Failed to execute ES query {}", response.getFailureMessage());
                        actualFailure = true;
                    }
                }
                if (actualFailure) {
                    throw new Exception(bulkItemResponses.buildFailureMessage());
                }
            }
        }
    } catch (Exception e) {
        log.error("Failed to execute ES query {}", brb.request().timeout(), e);
        throw convert(e);
    }
}

Example 62 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project graylog2-server by Graylog2.

the class Indices method move.

public void move(String source, String target) {
    SearchResponse scrollResp = c.prepareSearch(source).setScroll(TimeValue.timeValueSeconds(10L)).setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort(SortParseElement.DOC_FIELD_NAME)).setSize(350).execute().actionGet();
    while (true) {
        scrollResp = c.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet();
        // No more hits.
        if (scrollResp.getHits().hits().length == 0) {
            break;
        }
        final BulkRequestBuilder request = c.prepareBulk();
        for (SearchHit hit : scrollResp.getHits()) {
            Map<String, Object> doc = hit.getSource();
            String id = (String) doc.remove("_id");
            request.add(messages.buildIndexRequest(target, doc, id));
        }
        request.setConsistencyLevel(WriteConsistencyLevel.ONE);
        if (request.numberOfActions() > 0) {
            BulkResponse response = c.bulk(request.request()).actionGet();
            LOG.info("Moving index <{}> to <{}>: Bulk indexed {} messages, took {} ms, failures: {}", source, target, response.getItems().length, response.getTookInMillis(), response.hasFailures());
            if (response.hasFailures()) {
                throw new RuntimeException("Failed to move a message. Check your indexer log.");
            }
        }
    }
}

Example 63 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project sonarqube by SonarSource.

the class ProxyBulkRequestBuilderTest method testBulk.

private void testBulk() {
    BulkRequestBuilder req = esTester.client().prepareBulk();
    req.add(new UpdateRequest(FakeIndexDefinition.INDEX, FakeIndexDefinition.INDEX_TYPE_FAKE.getType(), "key1").doc(FakeIndexDefinition.newDoc(1).getFields()));
    req.add(new DeleteRequest(FakeIndexDefinition.INDEX, FakeIndexDefinition.INDEX_TYPE_FAKE.getType(), "key2"));
    req.add(new IndexRequest(FakeIndexDefinition.INDEX, FakeIndexDefinition.INDEX_TYPE_FAKE.getType(), "key3").source(FakeIndexDefinition.newDoc(3).getFields()));
    assertThat(req.toString()).isEqualTo("Bulk[1 update request(s) on index fakes and type fake, 1 delete request(s) on index fakes and type fake, 1 index request(s) on index fakes and type fake]");
    BulkResponse response = req.get();
    assertThat(response.getItems()).hasSize(3);
}

Example 64 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project metacat by Netflix.

the class ElasticSearchUtilImpl method updateDocs.

/**
     * Updates the documents with partial updates with the given fields.
     *
     * @param type                  index type
     * @param ids                   list of entity ids
     * @param metacatRequestContext context containing the user name
     * @param node                  json that represents the document source
     */
private void updateDocs(final String type, final List<String> ids, final MetacatRequestContext metacatRequestContext, final ObjectNode node) {
    try {
        RETRY_ES_PUBLISH.call(() -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            ids.forEach(id -> {
                node.put(ElasticSearchDoc.Field.USER, metacatRequestContext.getUserName());
                bulkRequest.add(client.prepareUpdate(esIndex, type, id).setRetryOnConflict(NO_OF_CONFLICT_RETRIES).setDoc(metacatJson.toJsonAsBytes(node)));
            });
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet();
            if (bulkResponse.hasFailures()) {
                for (BulkItemResponse item : bulkResponse.getItems()) {
                    if (item.isFailed()) {
                        log.error("Failed updating metadata of type {} with id {}. Message: {}", type, item.getId(), item.getFailureMessage());
                        registry.counter(registry.createId(Metrics.CounterElasticSearchUpdate.name()).withTags(Metrics.statusFailureMap)).increment();
                        log("ElasticSearchUtil.updateDocs.item", type, item.getId(), null, item.getFailureMessage(), null, true);
                    }
                }
            }
            return null;
        });
    } catch (Exception e) {
        log.error(String.format("Failed updating metadata of type %s with ids %s", type, ids), e);
        registry.counter(registry.createId(Metrics.CounterElasticSearchBulkUpdate.name()).withTags(Metrics.statusFailureMap)).increment();
        log("ElasticSearchUtil.updatDocs", type, ids.toString(), null, e.getMessage(), e, true);
    }
}

Example 65 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project camel by apache.

the class ElasticsearchBulkTest method bulkRequestBody.

@Test
public void bulkRequestBody() throws Exception {
    String prefix = createPrefix();
    // given
    BulkRequest request = new BulkRequest();
    request.add(new IndexRequest(prefix + "foo", prefix + "bar", prefix + "baz").source("{\"" + prefix + "content\": \"" + prefix + "hello\"}"));
    // when
    BulkResponse response = template.requestBody("direct:bulk", request, BulkResponse.class);
    // then
    assertThat(response, notNullValue());
    assertEquals(prefix + "baz", response.getItems()[0].getId());
}