Examples with BulkItemResponse org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse used on opensource projects

Example 41 with BulkItemResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project metacat by Netflix.

the class ElasticSearchUtilImpl method bulkSaveToIndex.

/**
 * Bulk save of the entities.
 *
 * @param type index type
 * @param docs metacat documents
 */
private void bulkSaveToIndex(final String type, final List<ElasticSearchDoc> docs, final String index) {
    if (docs != null && !docs.isEmpty()) {
        try {
            RETRY_ES_PUBLISH.call(() -> {
                final BulkRequestBuilder bulkRequest = client.prepareBulk();
                for (ElasticSearchDoc doc : docs) {
                    final IndexRequestBuilder indexRequestBuilder = prepareIndexRequest(index, type, doc);
                    if (indexRequestBuilder != null) {
                        bulkRequest.add(indexRequestBuilder);
                    }
                }
                if (bulkRequest.numberOfActions() > 0) {
                    final BulkResponse bulkResponse = bulkRequest.execute().actionGet();
                    log.info("Bulk saving metadata of index {} type {} with size {}.", index, type, docs.size());
                    if (bulkResponse.hasFailures()) {
                        for (BulkItemResponse item : bulkResponse.getItems()) {
                            if (item.isFailed()) {
                                log.error("Failed saving metadata of {} index type {} with id {}. Message: {}", index, type, item.getId(), item.getFailureMessage());
                                this.elasticSearchMetric.getElasticSearchSaveFailureCounter().increment();
                                log("ElasticSearchUtil.bulkSaveToIndex.index", type, item.getId(), null, item.getFailureMessage(), null, true, index);
                            }
                        }
                    }
                }
                return null;
            });
        } catch (Exception e) {
            log.error("Failed saving metadatas of index {} type {}. {}", index, type, e);
            this.elasticSearchMetric.getElasticSearchBulkSaveFailureCounter().increment();
            final List<String> docIds = docs.stream().map(ElasticSearchDoc::getId).collect(Collectors.toList());
            log("ElasticSearchUtil.bulkSaveToIndex", type, docIds.toString(), null, e.getMessage(), e, true, index);
        }
    }
}

Example 42 with BulkItemResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project metacat by Netflix.

the class ElasticSearchUtilImpl method softDeleteDoc.

/* Use elasticSearch bulk API to mark the documents as deleted
    * @param type index type
    * @param ids list of entity ids
    * @param metacatRequestContext context containing the user name
    */
private void softDeleteDoc(final String type, final List<String> ids, final MetacatRequestContext metacatRequestContext) {
    try {
        RETRY_ES_PUBLISH.call(() -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            final XContentBuilder builder = XContentFactory.contentBuilder(contentType);
            builder.startObject().field(ElasticSearchDoc.Field.DELETED, true).field(ElasticSearchDoc.Field.TIMESTAMP, java.time.Instant.now().toEpochMilli()).field(ElasticSearchDoc.Field.USER, metacatRequestContext.getUserName()).endObject();
            ids.forEach(id -> bulkRequest.add(client.prepareUpdate(esIndex, type, id).setRetryOnConflict(NO_OF_CONFLICT_RETRIES).setDoc(builder)));
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet();
            if (bulkResponse.hasFailures()) {
                for (BulkItemResponse item : bulkResponse.getItems()) {
                    if (item.isFailed()) {
                        log.error("Failed soft deleting metadata of type {} with id {}. Message: {}", type, item.getId(), item.getFailureMessage());
                        this.elasticSearchMetric.getElasticSearchDeleteFailureCounter().increment();
                        log("ElasticSearchUtil.bulkSoftDelete.item", type, item.getId(), null, item.getFailureMessage(), null, true);
                    }
                }
            }
            return null;
        });
    } catch (Exception e) {
        log.error("Failed soft deleting metadata of type {} with ids {}. {}", type, ids, e);
        this.elasticSearchMetric.getElasticSearchBulkDeleteFailureCounter().increment();
        log("ElasticSearchUtil.bulkSoftDelete", type, ids.toString(), null, e.getMessage(), e, true);
    }
}

Example 43 with BulkItemResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project incubator-skywalking by apache.

the class BatchEsDAO method batchPersistence.

@Override
public void batchPersistence(List<?> batchCollection) {
    BulkRequestBuilder bulkRequest = getClient().prepareBulk();
    logger.debug("bulk data size: {}", batchCollection.size());
    if (CollectionUtils.isNotEmpty(batchCollection)) {
        batchCollection.forEach(builder -> {
            if (builder instanceof IndexRequestBuilder) {
                bulkRequest.add((IndexRequestBuilder) builder);
            }
            if (builder instanceof UpdateRequestBuilder) {
                bulkRequest.add((UpdateRequestBuilder) builder);
            }
        });
        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
        if (bulkResponse.hasFailures()) {
            logger.error(bulkResponse.buildFailureMessage());
            for (BulkItemResponse itemResponse : bulkResponse.getItems()) {
                logger.error("Bulk request failure, index: {}, id: {}", itemResponse.getIndex(), itemResponse.getId());
            }
        }
    }
}

Example 44 with BulkItemResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project yacy_grid_mcp by yacy.

the class ElasticsearchClient method writeMapBulk.

/**
 * bulk message write
 * @param jsonMapList
 *            a list of json documents to be indexed
 * @param indexName
 *            the name of the index
 * @param typeName
 *            the type of the index
 * @return a list with error messages.
 *            The key is the id of the document, the value is an error string.
 *            The method was only successful if this list is empty.
 *            This must be a list, because keys may appear several times.
 */
public BulkWriteResult writeMapBulk(final String indexName, final List<BulkEntry> jsonMapList) {
    long start = System.currentTimeMillis();
    BulkRequestBuilder bulkRequest = elasticsearchClient.prepareBulk();
    for (BulkEntry be : jsonMapList) {
        if (be.id == null)
            continue;
        bulkRequest.add(elasticsearchClient.prepareIndex(indexName, be.type, be.id).setSource(be.jsonMap).setVersion(1).setCreate(// enforces OpType.INDEX
        false).setVersionType(VersionType.EXTERNAL_GTE));
    }
    BulkResponse bulkResponse = bulkRequest.get();
    BulkWriteResult result = new BulkWriteResult();
    for (BulkItemResponse r : bulkResponse.getItems()) {
        String id = r.getId();
        DocWriteResponse response = r.getResponse();
        if (response.getResult() == DocWriteResponse.Result.CREATED)
            result.created.add(id);
        String err = r.getFailureMessage();
        if (err != null) {
            result.errors.put(id, err);
        }
    }
    long duration = Math.max(1, System.currentTimeMillis() - start);
    long regulator = 0;
    int created = result.created.size();
    long ops = created * 1000 / duration;
    if (duration > throttling_time_threshold && ops < throttling_ops_threshold) {
        regulator = (long) (throttling_factor * duration);
        try {
            Thread.sleep(regulator);
        } catch (InterruptedException e) {
        }
    }
    Data.logger.info("elastic write bulk to index " + indexName + ": " + jsonMapList.size() + " entries, " + result.created.size() + " created, " + result.errors.size() + " errors, " + duration + " ms" + (regulator == 0 ? "" : ", throttled with " + regulator + " ms") + ", " + ops + " objects/second");
    return result;
}

Example 45 with BulkItemResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project elasticsearch-river-couchdb by elastic.

the class CouchdbRiver method start.

@Override
public void start() {
    logger.info("starting couchdb stream: host [{}], port [{}], filter [{}], db [{}], indexing to [{}]/[{}]", couchHost, couchPort, couchFilter, couchDb, indexName, typeName);
    // Creating bulk processor
    this.bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
            logger.debug("Going to execute new bulk composed of {} actions", request.numberOfActions());
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            logger.debug("Executed bulk composed of {} actions", request.numberOfActions());
            if (response.hasFailures()) {
                logger.warn("There was failures while executing bulk", response.buildFailureMessage());
                if (logger.isDebugEnabled()) {
                    for (BulkItemResponse item : response.getItems()) {
                        if (item.isFailed()) {
                            logger.debug("Error for {}/{}/{} for {} operation: {}", item.getIndex(), item.getType(), item.getId(), item.getOpType(), item.getFailureMessage());
                        }
                    }
                }
            }
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            logger.warn("Error executing bulk", failure);
        }
    }).setBulkActions(bulkSize).setConcurrentRequests(maxConcurrentBulk).setFlushInterval(bulkFlushInterval).build();
    slurperThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "couchdb_river_slurper").newThread(new Slurper());
    indexerThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "couchdb_river_indexer").newThread(new Indexer());
    indexerThread.start();
    slurperThread.start();
}