Examples with BulkRequest org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest used on opensource projects

Example 61 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project tmail-backend by linagora.

the class ElasticSearchIndexer method update.

public Mono<BulkResponse> update(List<UpdatedRepresentation> updatedDocumentParts, RoutingKey routingKey) {
    Preconditions.checkNotNull(updatedDocumentParts);
    Preconditions.checkNotNull(routingKey);
    BulkRequest request = new BulkRequest();
    updatedDocumentParts.forEach(updatedDocumentPart -> request.add(new UpdateRequest(aliasName.getValue(), NodeMappingFactory.DEFAULT_MAPPING_NAME, updatedDocumentPart.getId().asString()).doc(updatedDocumentPart.getUpdatedDocumentPart(), XContentType.JSON).routing(routingKey.asString())));
    return client.bulk(request, RequestOptions.DEFAULT).onErrorResume(ValidationException.class, exception -> {
        LOGGER.warn("Error while updating index", exception);
        return Mono.empty();
    });
}

Example 62 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project flink by splunk.

the class Elasticsearch6SinkBuilder method getBulkProcessorBuilderFactory.

@Override
protected BulkProcessorBuilderFactory getBulkProcessorBuilderFactory() {
    return new BulkProcessorBuilderFactory() {

        @Override
        public BulkProcessor.Builder apply(RestHighLevelClient client, BulkProcessorConfig bulkProcessorConfig, BulkProcessor.Listener listener) {
            BulkProcessor.Builder builder = BulkProcessor.builder(new // This cannot be inlined as a
            BulkRequestConsumerFactory() {

                // lambda because then
                // deserialization fails
                @Override
                public void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
                    client.bulkAsync(bulkRequest, bulkResponseActionListener);
                }
            }, listener);
            if (bulkProcessorConfig.getBulkFlushMaxActions() != -1) {
                builder.setBulkActions(bulkProcessorConfig.getBulkFlushMaxActions());
            }
            if (bulkProcessorConfig.getBulkFlushMaxMb() != -1) {
                builder.setBulkSize(new ByteSizeValue(bulkProcessorConfig.getBulkFlushMaxMb(), ByteSizeUnit.MB));
            }
            if (bulkProcessorConfig.getBulkFlushInterval() != -1) {
                builder.setFlushInterval(new TimeValue(bulkProcessorConfig.getBulkFlushInterval()));
            }
            BackoffPolicy backoffPolicy;
            final TimeValue backoffDelay = new TimeValue(bulkProcessorConfig.getBulkFlushBackOffDelay());
            final int maxRetryCount = bulkProcessorConfig.getBulkFlushBackoffRetries();
            switch(bulkProcessorConfig.getFlushBackoffType()) {
                case CONSTANT:
                    backoffPolicy = BackoffPolicy.constantBackoff(backoffDelay, maxRetryCount);
                    break;
                case EXPONENTIAL:
                    backoffPolicy = BackoffPolicy.exponentialBackoff(backoffDelay, maxRetryCount);
                    break;
                case NONE:
                    backoffPolicy = BackoffPolicy.noBackoff();
                    break;
                default:
                    throw new IllegalArgumentException("Received unknown backoff policy type " + bulkProcessorConfig.getFlushBackoffType());
            }
            builder.setBackoffPolicy(backoffPolicy);
            return builder;
        }
    };
}

Example 63 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project flink by splunk.

the class Elasticsearch7SinkBuilder method getBulkProcessorBuilderFactory.

@Override
protected BulkProcessorBuilderFactory getBulkProcessorBuilderFactory() {
    return new BulkProcessorBuilderFactory() {

        @Override
        public BulkProcessor.Builder apply(RestHighLevelClient client, BulkProcessorConfig bulkProcessorConfig, BulkProcessor.Listener listener) {
            BulkProcessor.Builder builder = BulkProcessor.builder(new // This cannot be inlined as a
            BulkRequestConsumerFactory() {

                // lambda because then
                // deserialization fails
                @Override
                public void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
                    client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener);
                }
            }, listener);
            if (bulkProcessorConfig.getBulkFlushMaxActions() != -1) {
                builder.setBulkActions(bulkProcessorConfig.getBulkFlushMaxActions());
            }
            if (bulkProcessorConfig.getBulkFlushMaxMb() != -1) {
                builder.setBulkSize(new ByteSizeValue(bulkProcessorConfig.getBulkFlushMaxMb(), ByteSizeUnit.MB));
            }
            if (bulkProcessorConfig.getBulkFlushInterval() != -1) {
                builder.setFlushInterval(new TimeValue(bulkProcessorConfig.getBulkFlushInterval()));
            }
            BackoffPolicy backoffPolicy;
            final TimeValue backoffDelay = new TimeValue(bulkProcessorConfig.getBulkFlushBackOffDelay());
            final int maxRetryCount = bulkProcessorConfig.getBulkFlushBackoffRetries();
            switch(bulkProcessorConfig.getFlushBackoffType()) {
                case CONSTANT:
                    backoffPolicy = BackoffPolicy.constantBackoff(backoffDelay, maxRetryCount);
                    break;
                case EXPONENTIAL:
                    backoffPolicy = BackoffPolicy.exponentialBackoff(backoffDelay, maxRetryCount);
                    break;
                case NONE:
                    backoffPolicy = BackoffPolicy.noBackoff();
                    break;
                default:
                    throw new IllegalArgumentException("Received unknown backoff policy type " + bulkProcessorConfig.getFlushBackoffType());
            }
            builder.setBackoffPolicy(backoffPolicy);
            return builder;
        }
    };
}

Example 64 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project open-commerce-search by CommerceExperts.

the class ElasticsearchIndexClient method deleteDocuments.

public List<DeleteResponse> deleteDocuments(String index, List<String> ids) throws IOException {
    BulkRequest bulkRequest = new BulkRequest(index);
    for (String id : ids) {
        bulkRequest.add(new DeleteRequest().id(id));
    }
    BulkResponse bulkResponse = highLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
    List<DeleteResponse> responses = new ArrayList<>();
    for (BulkItemResponse item : bulkResponse.getItems()) {
        responses.add((DeleteResponse) item.getResponse());
    }
    return responses;
}

Example 65 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project open-commerce-search by CommerceExperts.

the class ElasticsearchIndexClient method indexRecordsChunkwise.

/**
 * Will split (if necessary) the given records into several bulk requests
 * each with the specified maximum size. The target index will be delete
 * before if already present.
 *
 * @param records
 * @param maxBulkSize
 * @return
 * @throws IOException
 */
public List<BulkResponse> indexRecordsChunkwise(String indexName, Iterator<IndexableItem> records, int maxBulkSize) throws IOException {
    List<BulkResponse> responses = new ArrayList<>();
    BulkRequest bulkIndexRequest = new BulkRequest();
    int i = 0;
    int indexedTotal = 0;
    while (records.hasNext()) {
        IndexRequest indexRequest;
        try {
            IndexableItem nextRecord = records.next();
            if (nextRecord != null) {
                indexRequest = asIndexRequest(indexName, nextRecord);
                bulkIndexRequest.add(indexRequest);
                i++;
                if (i == maxBulkSize) {
                    indexedTotal += i;
                    i = 0;
                    responses.add(highLevelClient.bulk(bulkIndexRequest, RequestOptions.DEFAULT));
                    log.info("Indexed {} records", indexedTotal);
                    bulkIndexRequest = new BulkRequest();
                }
            }
        } catch (JsonProcessingException e) {
            log.warn("failed to add record to bulk request", e);
        }
    }
    if (i > 0) {
        responses.add(highLevelClient.bulk(bulkIndexRequest, RequestOptions.DEFAULT));
    }
    return responses;
}