Examples with BulkRequest org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest used on opensource projects

Example 51 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project vertx-zero by silentbalanceyh.

the class EsAmbitDelete method process.

@Override
public Boolean process(final JsonArray data, final String idField) {
    return this.doBatch(data, idField, () -> {
        final BulkRequest request = new BulkRequest();
        Ut.itJString(data).forEach(documentId -> request.add(new DeleteRequest().index(this.index).id(documentId)));
        return request;
    });
}

Example 52 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project incubator-sdap-mudrod by apache.

the class ESDriver method createBulkProcessor.

public void createBulkProcessor() {
    LOG.debug("Creating BulkProcessor with maxBulkDocs={}, maxBulkLength={}", 1000, 2500500);
    setBulkProcessor(BulkProcessor.builder(getClient(), new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
            LOG.debug("ESDriver#createBulkProcessor @Override #beforeBulk is not implemented yet!");
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            LOG.debug("ESDriver#createBulkProcessor @Override #afterBulk is not implemented yet!");
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            LOG.error("Bulk request has failed!");
            throw new RuntimeException("Caught exception in bulk: " + request.getDescription() + ", failure: " + failure, failure);
        }
    }).setBulkActions(1000).setBulkSize(new ByteSizeValue(2500500, ByteSizeUnit.GB)).setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 10)).setConcurrentRequests(1).build());
}

Example 53 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project apiman by apiman.

the class EsMetrics method processQueue.

/**
 * Process the next item in the queue.
 */
protected void processQueue() {
    try {
        RestHighLevelClient client = getClient();
        Collection<RequestMetric> batch = new ArrayList<>(this.batchSize);
        RequestMetric rm = queue.take();
        batch.add(rm);
        queue.drainTo(batch, this.batchSize - 1);
        BulkRequest request = new BulkRequest();
        for (RequestMetric metric : batch) {
            IndexRequest index = new IndexRequest(getDefaultIndexPrefix());
            index.source(JSON_MAPPER.writeValueAsString(metric), XContentType.JSON);
            request.add(index);
        }
        ActionListener<BulkResponse> listener = new ActionListener<BulkResponse>() {

            @Override
            public void onResponse(BulkResponse bulkItemResponses) {
            // Do nothing
            }

            @Override
            public void onFailure(Exception e) {
                // $NON-NLS-1$
                LOGGER.error("Failed to add metric(s) to ES", e);
            }
        };
        client.bulkAsync(request, RequestOptions.DEFAULT, listener);
    } catch (InterruptedException | JsonProcessingException e) {
        // $NON-NLS-1$
        LOGGER.error("Failed to add metric(s) to ES", e);
    }
}

Example 54 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project flink-mirror by flink-ci.

the class Elasticsearch7SinkBuilder method getBulkProcessorBuilderFactory.

@Override
protected BulkProcessorBuilderFactory getBulkProcessorBuilderFactory() {
    return new BulkProcessorBuilderFactory() {

        @Override
        public BulkProcessor.Builder apply(RestHighLevelClient client, BulkProcessorConfig bulkProcessorConfig, BulkProcessor.Listener listener) {
            BulkProcessor.Builder builder = BulkProcessor.builder(new // This cannot be inlined as a
            BulkRequestConsumerFactory() {

                // lambda because then
                // deserialization fails
                @Override
                public void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
                    client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener);
                }
            }, listener);
            if (bulkProcessorConfig.getBulkFlushMaxActions() != -1) {
                builder.setBulkActions(bulkProcessorConfig.getBulkFlushMaxActions());
            }
            if (bulkProcessorConfig.getBulkFlushMaxMb() != -1) {
                builder.setBulkSize(new ByteSizeValue(bulkProcessorConfig.getBulkFlushMaxMb(), ByteSizeUnit.MB));
            }
            if (bulkProcessorConfig.getBulkFlushInterval() != -1) {
                builder.setFlushInterval(new TimeValue(bulkProcessorConfig.getBulkFlushInterval()));
            }
            BackoffPolicy backoffPolicy;
            final TimeValue backoffDelay = new TimeValue(bulkProcessorConfig.getBulkFlushBackOffDelay());
            final int maxRetryCount = bulkProcessorConfig.getBulkFlushBackoffRetries();
            switch(bulkProcessorConfig.getFlushBackoffType()) {
                case CONSTANT:
                    backoffPolicy = BackoffPolicy.constantBackoff(backoffDelay, maxRetryCount);
                    break;
                case EXPONENTIAL:
                    backoffPolicy = BackoffPolicy.exponentialBackoff(backoffDelay, maxRetryCount);
                    break;
                case NONE:
                    backoffPolicy = BackoffPolicy.noBackoff();
                    break;
                default:
                    throw new IllegalArgumentException("Received unknown backoff policy type " + bulkProcessorConfig.getFlushBackoffType());
            }
            builder.setBackoffPolicy(backoffPolicy);
            return builder;
        }
    };
}

Example 55 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project geo-platform by geosdi.

the class ElasticSearchRestDAO method persist.

/**
 * @param documents
 * @return {@link BulkResponse}
 * @throws Exception
 */
@Override
public BulkResponse persist(@Nonnull(when = NEVER) Iterable<D> documents) throws Exception {
    checkArgument((documents != null) && (Iterables.size(documents) > 0));
    BulkRequest bulkRequest = new BulkRequest();
    stream(documents.spliterator(), FALSE).filter(Objects::nonNull).map(this::prepareIndexRequest).filter(Objects::nonNull).forEach(bulkRequest::add);
    BulkResponse bulkResponse = this.elasticSearchRestHighLevelClient.bulk(bulkRequest, DEFAULT);
    if (bulkResponse.hasFailures()) {
        throw new IllegalStateException(bulkResponse.buildFailureMessage());
    }
    return bulkResponse;
}