Examples with BulkResponse org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse used on opensource projects

Example 86 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project metacat by Netflix.

the class ElasticSearchUtilImpl method softDeleteDoc.

/* Use elasticSearch bulk API to mark the documents as deleted
    * @param type index type
    * @param ids list of entity ids
    * @param metacatRequestContext context containing the user name
    */
private void softDeleteDoc(final String type, final List<String> ids, final MetacatRequestContext metacatRequestContext) {
    try {
        RETRY_ES_PUBLISH.call(() -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            final XContentBuilder builder = XContentFactory.contentBuilder(contentType);
            builder.startObject().field(ElasticSearchDoc.Field.DELETED, true).field(ElasticSearchDoc.Field.TIMESTAMP, java.time.Instant.now().toEpochMilli()).field(ElasticSearchDoc.Field.USER, metacatRequestContext.getUserName()).endObject();
            ids.forEach(id -> bulkRequest.add(client.prepareUpdate(esIndex, type, id).setRetryOnConflict(NO_OF_CONFLICT_RETRIES).setDoc(builder)));
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
            if (bulkResponse.hasFailures()) {
                for (BulkItemResponse item : bulkResponse.getItems()) {
                    if (item.isFailed()) {
                        handleException("ElasticSearchUtil.bulkSoftDelete.item", type, item.getId(), item.getFailure().getCause(), Metrics.CounterElasticSearchDelete.getMetricName());
                    }
                }
            }
            return null;
        });
    } catch (Exception e) {
        handleException("ElasticSearchUtil.bulkSoftDelete", type, ids, e, Metrics.CounterElasticSearchBulkDelete.getMetricName());
    }
}

Example 87 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project metacat by Netflix.

the class ElasticSearchUtilImpl method bulkSaveToIndex.

/**
 * Bulk save of the entities.
 *
 * @param type index type
 * @param docs metacat documents
 */
private void bulkSaveToIndex(final String type, final List<ElasticSearchDoc> docs, final String index) {
    if (docs != null && !docs.isEmpty()) {
        try {
            RETRY_ES_PUBLISH.call(() -> {
                final BulkRequestBuilder bulkRequest = client.prepareBulk();
                for (ElasticSearchDoc doc : docs) {
                    final IndexRequestBuilder indexRequestBuilder = prepareIndexRequest(index, type, doc);
                    if (indexRequestBuilder != null) {
                        bulkRequest.add(indexRequestBuilder);
                    }
                }
                if (bulkRequest.numberOfActions() > 0) {
                    final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
                    log.info("Bulk saving metadata of index {} type {} with size {}.", index, type, docs.size());
                    if (bulkResponse.hasFailures()) {
                        for (BulkItemResponse item : bulkResponse.getItems()) {
                            if (item.isFailed()) {
                                handleException("ElasticSearchUtil.bulkSaveToIndex.index", type, item.getId(), item.getFailure().getCause(), Metrics.CounterElasticSearchSave.getMetricName());
                            }
                        }
                    }
                }
                return null;
            });
        } catch (Exception e) {
            final List<String> docIds = docs.stream().map(ElasticSearchDoc::getId).collect(Collectors.toList());
            handleException("ElasticSearchUtil.bulkSaveToIndex", type, docIds, e, Metrics.CounterElasticSearchBulkSave.getMetricName());
        }
    }
}

Example 88 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project wonderdog by infochimps-labs.

the class ElasticSearchStreamingRecordWriter method sendBulkRequestIfMoreThan.

private void sendBulkRequestIfMoreThan(int size) {
    totalBulkItems.incrementAndGet();
    if (currentRequest.numberOfActions() > size) {
        long startTime = System.currentTimeMillis();
        BulkResponse response = currentRequest.execute().actionGet();
        totalBulkTime.addAndGet(System.currentTimeMillis() - startTime);
        if (randgen.nextDouble() < 0.1) {
            LOG.info("Indexed [" + totalBulkItems.get() + "] in [" + (totalBulkTime.get() / 1000) + "s] of indexing" + "[" + ((System.currentTimeMillis() - runStartTime) / 1000) + "s] of wall clock" + " for [" + (float) (1000.0 * totalBulkItems.get()) / (System.currentTimeMillis() - runStartTime) + "rec/s]");
        }
        currentRequest = client.prepareBulk();
    }
}

Example 89 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project apex-malhar by apache.

the class AbstractElasticSearchOutputOperator method processBatch.

/**
 * This will flush all the tuples from queue to ElasticSearch.
 * It uses BulkRequestBuilder API for sending batch.
 */
private void processBatch() {
    BulkRequestBuilder bulkRequestBuilder = new BulkRequestBuilder(store.client);
    while (!tupleBatch.isEmpty()) {
        T tuple = tupleBatch.remove();
        IndexRequestBuilder indexRequestBuilder = getIndexRequestBuilder(tuple);
        bulkRequestBuilder.add(indexRequestBuilder);
    }
    BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
    if (bulkResponse.hasFailures()) {
        DTThrowable.rethrow(new Exception(bulkResponse.buildFailureMessage()));
    }
}

Example 90 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project sonarqube by SonarSource.

the class EsTester method putDocuments.

public void putDocuments(IndexType indexType, Map<String, Object>... docs) {
    try {
        BulkRequest bulk = new BulkRequest().setRefreshPolicy(REFRESH_IMMEDIATE);
        for (Map<String, Object> doc : docs) {
            IndexType.IndexMainType mainType = indexType.getMainType();
            bulk.add(new IndexRequest(mainType.getIndex().getName(), mainType.getType()).source(doc));
        }
        BulkResponse bulkResponse = ES_REST_CLIENT.bulk(bulk);
        if (bulkResponse.hasFailures()) {
            throw new IllegalStateException(bulkResponse.buildFailureMessage());
        }
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}