Examples with BulkRequestBuilder org.elasticsearch.action.bulk.BulkRequestBuilder used on opensource projects

Example 36 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project storm by apache.

the class EsState method updateState.

/**
     * Store current state to ElasticSearch.
     *
     * @param tuples list of tuples for storing to ES.
     *               Each tuple should have relevant fields (source, index, type, id) for EsState's tupleMapper to extract ES document.
     */
public void updateState(List<TridentTuple> tuples) {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (TridentTuple tuple : tuples) {
        String source = tupleMapper.getSource(tuple);
        String index = tupleMapper.getIndex(tuple);
        String type = tupleMapper.getType(tuple);
        String id = tupleMapper.getId(tuple);
        bulkRequest.add(client.prepareIndex(index, type, id).setSource(source));
    }
    BulkResponse bulkResponse = bulkRequest.execute().actionGet();
    if (bulkResponse.hasFailures()) {
        LOG.warn("failed processing bulk index requests " + bulkResponse.buildFailureMessage());
        throw new FailedException();
    }
}

Example 37 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project titan by thinkaurelius.

the class ElasticSearchIndex method mutate.

@Override
public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
    BulkRequestBuilder brb = client.prepareBulk();
    int bulkrequests = 0;
    try {
        for (Map.Entry<String, Map<String, IndexMutation>> stores : mutations.entrySet()) {
            String storename = stores.getKey();
            for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
                String docid = entry.getKey();
                IndexMutation mutation = entry.getValue();
                assert mutation.isConsolidated();
                Preconditions.checkArgument(!(mutation.isNew() && mutation.isDeleted()));
                Preconditions.checkArgument(!mutation.isNew() || !mutation.hasDeletions());
                Preconditions.checkArgument(!mutation.isDeleted() || !mutation.hasAdditions());
                //Deletions first
                if (mutation.hasDeletions()) {
                    if (mutation.isDeleted()) {
                        log.trace("Deleting entire document {}", docid);
                        brb.add(new DeleteRequest(indexName, storename, docid));
                    } else {
                        String script = getDeletionScript(informations, storename, mutation);
                        brb.add(client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE));
                        log.trace("Adding script {}", script);
                    }
                    bulkrequests++;
                }
                if (mutation.hasAdditions()) {
                    int ttl = mutation.determineTTL();
                    if (mutation.isNew()) {
                        //Index
                        log.trace("Adding entire document {}", docid);
                        brb.add(new IndexRequest(indexName, storename, docid).source(getNewDocument(mutation.getAdditions(), informations.get(storename), ttl)));
                    } else {
                        Preconditions.checkArgument(ttl == 0, "Elasticsearch only supports TTL on new documents [%s]", docid);
                        boolean needUpsert = !mutation.hasDeletions();
                        String script = getAdditionScript(informations, storename, mutation);
                        UpdateRequestBuilder update = client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE);
                        if (needUpsert) {
                            XContentBuilder doc = getNewDocument(mutation.getAdditions(), informations.get(storename), ttl);
                            update.setUpsert(doc);
                        }
                        brb.add(update);
                        log.trace("Adding script {}", script);
                    }
                    bulkrequests++;
                }
            }
        }
        if (bulkrequests > 0) {
            BulkResponse bulkItemResponses = brb.execute().actionGet();
            if (bulkItemResponses.hasFailures()) {
                boolean actualFailure = false;
                for (BulkItemResponse response : bulkItemResponses.getItems()) {
                    //The document may have been deleted, which is OK
                    if (response.isFailed() && response.getFailure().getStatus() != RestStatus.NOT_FOUND) {
                        log.error("Failed to execute ES query {}", response.getFailureMessage());
                        actualFailure = true;
                    }
                }
                if (actualFailure) {
                    throw new Exception(bulkItemResponses.buildFailureMessage());
                }
            }
        }
    } catch (Exception e) {
        log.error("Failed to execute ES query {}", brb.request().timeout(), e);
        throw convert(e);
    }
}

Example 38 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project graylog2-server by Graylog2.

the class Indices method move.

public void move(String source, String target) {
    SearchResponse scrollResp = c.prepareSearch(source).setScroll(TimeValue.timeValueSeconds(10L)).setQuery(matchAllQuery()).addSort(SortBuilders.fieldSort(SortParseElement.DOC_FIELD_NAME)).setSize(350).execute().actionGet();
    while (true) {
        scrollResp = c.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet();
        // No more hits.
        if (scrollResp.getHits().hits().length == 0) {
            break;
        }
        final BulkRequestBuilder request = c.prepareBulk();
        for (SearchHit hit : scrollResp.getHits()) {
            Map<String, Object> doc = hit.getSource();
            String id = (String) doc.remove("_id");
            request.add(messages.buildIndexRequest(target, doc, id));
        }
        request.setConsistencyLevel(WriteConsistencyLevel.ONE);
        if (request.numberOfActions() > 0) {
            BulkResponse response = c.bulk(request.request()).actionGet();
            LOG.info("Moving index <{}> to <{}>: Bulk indexed {} messages, took {} ms, failures: {}", source, target, response.getItems().length, response.getTookInMillis(), response.hasFailures());
            if (response.hasFailures()) {
                throw new RuntimeException("Failed to move a message. Check your indexer log.");
            }
        }
    }
}

Example 39 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project zipkin by openzipkin.

the class ElasticsearchDependenciesTest method writeDependencyLinks.

protected void writeDependencyLinks(List<DependencyLink> links, long timestampMillis) {
    long midnight = Util.midnightUTC(timestampMillis);
    TransportClient client = ((NativeClient) storage().client()).client;
    BulkRequestBuilder request = client.prepareBulk();
    for (DependencyLink link : links) {
        request.add(client.prepareIndex(storage().indexNameFormatter.indexNameForTimestamp(midnight), ElasticsearchConstants.DEPENDENCY_LINK).setId(// Unique constraint
        link.parent + "|" + link.child).setSource("parent", link.parent, "child", link.child, "callCount", link.callCount));
    }
    request.execute().actionGet();
    client.admin().indices().flush(new FlushRequest()).actionGet();
}

Example 40 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project sonarqube by SonarSource.

the class ProxyBulkRequestBuilderTest method testBulk.

private void testBulk() {
    BulkRequestBuilder req = esTester.client().prepareBulk();
    req.add(new UpdateRequest(FakeIndexDefinition.INDEX, FakeIndexDefinition.INDEX_TYPE_FAKE.getType(), "key1").doc(FakeIndexDefinition.newDoc(1).getFields()));
    req.add(new DeleteRequest(FakeIndexDefinition.INDEX, FakeIndexDefinition.INDEX_TYPE_FAKE.getType(), "key2"));
    req.add(new IndexRequest(FakeIndexDefinition.INDEX, FakeIndexDefinition.INDEX_TYPE_FAKE.getType(), "key3").source(FakeIndexDefinition.newDoc(3).getFields()));
    assertThat(req.toString()).isEqualTo("Bulk[1 update request(s) on index fakes and type fake, 1 delete request(s) on index fakes and type fake, 1 index request(s) on index fakes and type fake]");
    BulkResponse response = req.get();
    assertThat(response.getItems()).hasSize(3);
}