Examples with BulkRequest org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest used on opensource projects

Example 16 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project Zpider by zeroized.

the class ElasticClient method bulkIndex.

public List<String> bulkIndex(List<Map<String, ?>> docs) throws IOException {
    BulkRequest bulkRequest = new BulkRequest();
    for (Map<String, ?> doc : docs) {
        IndexRequest indexRequest = new IndexRequest(index, type);
        indexRequest.source(doc);
        bulkRequest.add(indexRequest);
    }
    BulkResponse bulkResponse = highLevelClient.bulk(bulkRequest);
    return Arrays.stream(bulkResponse.getItems()).map(BulkItemResponse::getId).collect(Collectors.toList());
}

Example 17 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project jnosql-diana-driver by eclipse.

the class DefaultElasticsearchDocumentCollectionManager method delete.

@Override
public void delete(DocumentDeleteQuery query) throws NullPointerException {
    requireNonNull(query, "query is required");
    query.getCondition().orElseThrow(() -> new IllegalArgumentException("condition is required"));
    DocumentQuery select = new ElasticsearchDocumentQuery(query);
    List<DocumentEntity> entities = select(select);
    if (entities.isEmpty()) {
        return;
    }
    BulkRequest bulk = new BulkRequest();
    entities.stream().map(entity -> entity.find(ID_FIELD).get().get(String.class)).map(id -> new DeleteRequest(index, query.getDocumentCollection(), id)).forEach(bulk::add);
    try {
        client.bulk(bulk);
    } catch (IOException e) {
        throw new ElasticsearchException("An error to delete entities on elasticsearch", e);
    }
}

Example 18 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project jnosql-diana-driver by eclipse.

the class DefaultElasticsearchDocumentCollectionManagerAsync method delete.

@Override
public void delete(DocumentDeleteQuery query, Consumer<Void> callBack) {
    requireNonNull(query, "query is required");
    requireNonNull(callBack, "callBack is required");
    query.getCondition().orElseThrow(() -> new IllegalArgumentException("condition is required"));
    DocumentQuery select = new ElasticsearchDocumentQuery(query);
    List<DocumentEntity> entities = EntityConverter.query(select, client, index);
    if (entities.isEmpty()) {
        callBack.accept(null);
        return;
    }
    BulkRequest bulk = new BulkRequest();
    entities.stream().map(entity -> entity.find(ID_FIELD).get().get(String.class)).map(id -> new DeleteRequest(index, query.getDocumentCollection(), id)).forEach(bulk::add);
    ActionListener<BulkResponse> listener = new ActionListener<BulkResponse>() {

        @Override
        public void onResponse(BulkResponse bulkItemResponses) {
            callBack.accept(null);
        }

        @Override
        public void onFailure(Exception e) {
            throw new ExecuteAsyncQueryException("An error when delete on elasticsearch", e);
        }
    };
    client.bulkAsync(bulk, listener);
}

Example 19 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project vertexium by visallo.

the class Elasticsearch5SearchIndex method bulkUpdate.

private void bulkUpdate(Graph graph, Iterable<UpdateRequest> updateRequests) {
    int totalCount = 0;
    Map<String, BulkRequest> bulkRequests = new HashMap<>();
    for (UpdateRequest updateRequest : updateRequests) {
        BulkRequest bulkRequest = bulkRequests.computeIfAbsent(updateRequest.index(), k -> new BulkRequest());
        bulkRequest.add(updateRequest);
        if (bulkRequest.numberOfActions() >= MAX_BATCH_COUNT || bulkRequest.estimatedSizeInBytes() > MAX_BATCH_SIZE) {
            LOGGER.debug("adding elements... %d (est size %d)", bulkRequest.numberOfActions(), bulkRequest.estimatedSizeInBytes());
            totalCount += bulkRequest.numberOfActions();
            doBulkRequest(bulkRequest);
            bulkRequest = new BulkRequest();
            bulkRequests.put(updateRequest.index(), bulkRequest);
        }
    }
    for (BulkRequest bulkRequest : bulkRequests.values()) {
        if (bulkRequest.numberOfActions() > 0) {
            LOGGER.debug("adding elements... %d (est size %d)", bulkRequest.numberOfActions(), bulkRequest.estimatedSizeInBytes());
            totalCount += bulkRequest.numberOfActions();
            doBulkRequest(bulkRequest);
        }
    }
    LOGGER.debug("added %d elements", totalCount);
    if (getConfig().isAutoFlush()) {
        flush(graph);
    }
}

Example 20 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project samza by apache.

the class ElasticsearchSystemProducer method register.

@Override
public void register(final String source) {
    BulkProcessor.Listener listener = new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
        // Nothing to do.
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            boolean hasFatalError = false;
            // Do not consider version conficts to be errors. Ignore old versions
            if (response.hasFailures()) {
                for (BulkItemResponse itemResp : response.getItems()) {
                    if (itemResp.isFailed()) {
                        if (itemResp.getFailure().getStatus().equals(RestStatus.CONFLICT)) {
                            LOGGER.info("Failed to index document in Elasticsearch: " + itemResp.getFailureMessage());
                        } else {
                            hasFatalError = true;
                            LOGGER.error("Failed to index document in Elasticsearch: " + itemResp.getFailureMessage());
                        }
                    }
                }
            }
            if (hasFatalError) {
                sendFailed.set(true);
            } else {
                updateSuccessMetrics(response);
            }
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            LOGGER.error(failure.getMessage());
            thrown.compareAndSet(null, failure);
            sendFailed.set(true);
        }

        private void updateSuccessMetrics(BulkResponse response) {
            metrics.bulkSendSuccess.inc();
            int writes = 0;
            for (BulkItemResponse itemResp : response.getItems()) {
                if (itemResp.isFailed()) {
                    if (itemResp.getFailure().getStatus().equals(RestStatus.CONFLICT)) {
                        metrics.conflicts.inc();
                    }
                } else {
                    ActionResponse resp = itemResp.getResponse();
                    if (resp instanceof IndexResponse) {
                        writes += 1;
                        if (((IndexResponse) resp).isCreated()) {
                            metrics.inserts.inc();
                        } else {
                            metrics.updates.inc();
                        }
                    } else {
                        LOGGER.error("Unexpected Elasticsearch action response type: " + resp.getClass().getSimpleName());
                    }
                }
            }
            LOGGER.info(String.format("Wrote %s messages from %s to %s.", writes, source, system));
        }
    };
    sourceBulkProcessor.put(source, bulkProcessorFactory.getBulkProcessor(client, listener));
}