Examples with BulkRequestBuilder org.elasticsearch.action.bulk.BulkRequestBuilder used on opensource projects

Example 11 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project sonarqube by SonarSource.

the class EsTester method putDocuments.

public void putDocuments(IndexType indexType, BaseDoc... docs) {
    try {
        BulkRequestBuilder bulk = client.prepareBulk().setRefresh(true);
        for (BaseDoc doc : docs) {
            bulk.add(new IndexRequest(indexType.getIndex(), indexType.getType(), doc.getId()).parent(doc.getParent()).routing(doc.getRouting()).source(doc.getFields()));
        }
        EsUtils.executeBulkRequest(bulk, "");
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 12 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project MSEC by Tencent.

the class ESClientThread method run.

@Override
public void run() {
    // TODO Auto-generated method stub
    try {
        while (true) {
            ESClientThread.ESThreadRequest request = queue.take();
            BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (int i = 0; i < request.sourceList.size(); i++) {
                //System.out.println("take from queue: " + source);
                bulkRequest.add(client.prepareIndex(request.indexNameList.get(i), request.indexTypeList.get(i)).setSource(request.sourceList.get(i)));
                LOG.info("taken source: " + request.sourceList.get(i));
            }
            BulkResponse bulkResponse = bulkRequest.execute().actionGet();
            if (bulkResponse.hasFailures()) {
                System.out.println("bulk response errorss!" + bulkResponse.buildFailureMessage());
                bulkResponse = bulkRequest.execute().actionGet();
            }
        //Thread.sleep(10);
        }
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}

Example 13 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project storm-elastic-search by hmsonline.

the class ElasticSearchState method createIndices.

public void createIndices(TridentElasticSearchMapper mapper, List<TridentTuple> tuples) {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    Set<String> existingIndex = new HashSet<String>();
    for (TridentTuple tuple : tuples) {
        String indexName = mapper.mapToIndex(tuple);
        String type = mapper.mapToType(tuple);
        String key = mapper.mapToKey(tuple);
        Map<String, Object> data = mapper.mapToData(tuple);
        String parentId = mapper.mapToParentId(tuple);
        if (!existingIndex.contains(indexName) && !client.admin().indices().exists(new IndicesExistsRequest(indexName)).actionGet().isExists()) {
            createIndex(bulkRequest, indexName, mapper.mapToIndexSettings(tuple));
            createMapping(bulkRequest, indexName, type, mapper.mapToMappingSettings(tuple));
            existingIndex.add(indexName);
        }
        if (StringUtils.isBlank(parentId)) {
            bulkRequest.add(client.prepareIndex(indexName, type, key).setSource(data));
        } else {
            LOGGER.debug("parent: " + parentId);
            bulkRequest.add(client.prepareIndex(indexName, type, key).setSource(data).setParent(parentId));
        }
    }
    try {
        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
        if (bulkResponse.hasFailures()) {
            // Index failed. Retry!
            throw new FailedException("Cannot create index via ES: " + bulkResponse.buildFailureMessage());
        }
    } catch (ElasticSearchException e) {
        StormElasticSearchUtils.handleElasticSearchException(getClass(), e);
    }
}

Example 14 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project zipkin by openzipkin.

the class NativeClient method bulkSpanIndexer.

@Override
protected BulkSpanIndexer bulkSpanIndexer() {
    return new SpanBytesBulkSpanIndexer() {

        final List<IndexRequestBuilder> indexRequests = new LinkedList<>();

        final Set<String> indicesToFlush = new LinkedHashSet<>();

        @Override
        protected void add(String index, byte[] spanBytes) {
            indexRequests.add(client.prepareIndex(index, SPAN).setSource(spanBytes));
            if (flushOnWrites)
                indicesToFlush.add(index);
        }

        // Creates a bulk request when there is more than one span to store
        @Override
        public void execute(final Callback<Void> callback) {
            ActionListener callbackAdapter = new ActionListener() {

                @Override
                public void onResponse(Object input) {
                    callback.onSuccess(null);
                }

                @Override
                public void onFailure(Throwable throwable) {
                    callback.onError(throwable);
                }
            };
            // Conditionally create a bulk action depending on the count of index requests
            ListenableActionFuture<? extends ActionResponse> future;
            if (indexRequests.size() == 1) {
                future = indexRequests.get(0).execute();
            } else {
                BulkRequestBuilder request = client.prepareBulk();
                for (IndexRequestBuilder span : indexRequests) {
                    request.add(span);
                }
                future = request.execute();
            }
            // Unless we are in a unit test, this should always be true
            if (indicesToFlush.isEmpty()) {
                future.addListener(callbackAdapter);
                return;
            }
            // If we are in a unit test, we need to flush so that we can read our writes
            future.addListener(new ActionListener() {

                @Override
                public void onResponse(Object input) {
                    client.admin().indices().prepareFlush(indicesToFlush.toArray(new String[indicesToFlush.size()])).execute().addListener(callbackAdapter);
                }

                @Override
                public void onFailure(Throwable throwable) {
                    callbackAdapter.onFailure(throwable);
                }
            });
        }
    };
}

Example 15 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project stash-codesearch-plugin by palantir.

the class SearchUpdateJobImpl method doReindex.

@Override
public void doReindex(Client client, GitScm gitScm, GlobalSettings globalSettings) {
    if (!globalSettings.getIndexingEnabled()) {
        return;
    }
    deleteLatestIndexedNote(client);
    while (true) {
        try {
            SearchRequestBuilder esReq = client.prepareSearch(ES_UPDATEALIAS).setSize(1000).setFetchSource(false).setRouting(getRepoDesc()).setQuery(filteredQuery(matchAllQuery(), andFilter(sfu.projectRepositoryFilter(repository.getProject().getKey(), repository.getSlug()), sfu.exactRefFilter(ref))));
            BulkRequestBuilder bulkDelete = client.prepareBulk().setRefresh(true);
            for (SearchHit hit : esReq.get().getHits().getHits()) {
                bulkDelete.add(buildDeleteFromRef(client, hit.getType(), hit.getId()));
            }
            if (bulkDelete.numberOfActions() == 0) {
                break;
            }
            bulkDelete.get();
        } catch (Exception e) {
            log.error("Could not delete documents for {}, aborting", toString(), e);
            return;
        }
    }
    doUpdate(client, gitScm, globalSettings);
}