Examples with BulkRequest org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest used on opensource projects

Example 11 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project nutch by apache.

the class TestElasticIndexWriter method setup.

@Before
public void setup() {
    conf = NutchConfiguration.create();
    conf.addResource("nutch-site-test.xml");
    bulkRequestSuccessful = false;
    clusterSaturated = false;
    curNumFailures = 0;
    maxNumFailures = 0;
    Settings settings = Settings.builder().build();
    ThreadPool threadPool = new ThreadPool(settings);
    // customize the ES client to simulate responses from an ES cluster
    client = new AbstractClient(settings, threadPool) {

        @Override
        public void close() {
        }

        @Override
        protected <Request extends ActionRequest, Response extends ActionResponse, RequestBuilder extends ActionRequestBuilder<Request, Response, RequestBuilder>> void doExecute(Action<Request, Response, RequestBuilder> action, Request request, ActionListener<Response> listener) {
            BulkResponse response = null;
            if (clusterSaturated) {
                // pretend the cluster is saturated
                curNumFailures++;
                if (curNumFailures >= maxNumFailures) {
                    // pretend the cluster is suddenly no longer saturated
                    clusterSaturated = false;
                }
                // respond with a failure
                BulkItemResponse failed = new BulkItemResponse(0, OpType.INDEX, new BulkItemResponse.Failure("nutch", "index", "failure0", new EsRejectedExecutionException("saturated")));
                response = new BulkResponse(new BulkItemResponse[] { failed }, 0);
            } else {
                // respond successfully
                BulkItemResponse success = new BulkItemResponse(0, OpType.INDEX, new IndexResponse(new ShardId("nutch", UUID.randomUUID().toString(), 0), "index", "index0", 0, true));
                response = new BulkResponse(new BulkItemResponse[] { success }, 0);
            }
            listener.onResponse((Response) response);
        }
    };
    // customize the plugin to signal successful bulk operations
    testIndexWriter = new ElasticIndexWriter() {

        @Override
        protected Client makeClient(Configuration conf) {
            return client;
        }

        @Override
        protected BulkProcessor.Listener bulkProcessorListener() {
            return new BulkProcessor.Listener() {

                @Override
                public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
                    if (!response.hasFailures()) {
                        bulkRequestSuccessful = true;
                    }
                }

                @Override
                public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
                }

                @Override
                public void beforeBulk(long executionId, BulkRequest request) {
                }
            };
        }
    };
}

Example 12 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project fabric8 by jboss-fuse.

the class AbstractElasticsearchStorage method run.

public void run() {
    while (running) {
        try {
            ActionRequest req = queue.take();
            // Send data
            BulkRequest bulk = new BulkRequest();
            int nb = 0;
            while (req != null && (nb == 0 || nb < max)) {
                bulk.add(req);
                nb++;
                req = queue.poll();
            }
            if (bulk.numberOfActions() > 0) {
                BulkResponse rep = getNode().client().bulk(bulk).actionGet();
                for (BulkItemResponse bir : rep.getItems()) {
                    if (bir.isFailed()) {
                        LOGGER.warn("Error executing request: {}", bir.getFailureMessage());
                    }
                }
            }
        } catch (Exception e) {
            if (running) {
                LOGGER.warn("Error while sending requests", e);
            }
        }
    }
}

Example 13 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project incubator-sdap-mudrod by apache.

the class ESDriver method createBulkProcessor.

public void createBulkProcessor() {
    LOG.debug("Creating BulkProcessor with maxBulkDocs={}, maxBulkLength={}", 1000, 2500500);
    setBulkProcessor(BulkProcessor.builder(getClient(), new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
            LOG.debug("ESDriver#createBulkProcessor @Override #beforeBulk is not implemented yet!");
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            LOG.debug("ESDriver#createBulkProcessor @Override #afterBulk is not implemented yet!");
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            LOG.error("Bulk request has failed!");
            throw new RuntimeException("Caught exception in bulk: " + request.getDescription() + ", failure: " + failure, failure);
        }
    }).setBulkActions(1000).setBulkSize(new ByteSizeValue(2500500, ByteSizeUnit.GB)).setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 10)).setConcurrentRequests(1).build());
}

Example 14 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project elasticsearch-indexing-proxy by codelibs.

the class ProxyActionFilter method getExecutor.

@SuppressWarnings("unchecked")
private <Request extends ActionRequest, Response extends ActionResponse> Supplier<Response> getExecutor(final Task task, final String action, final Request request) {
    if (BulkAction.NAME.equals(action)) {
        final long startTime = System.nanoTime();
        int count = 0;
        final BulkRequest req = (BulkRequest) request;
        for (final DocWriteRequest<?> subReq : req.requests()) {
            if (indexingProxyService.isTargetIndex(subReq.index())) {
                count++;
            }
        }
        if (count == 0) {
            return null;
        } else if (count != req.requests().size()) {
            throw new ElasticsearchException("Mixed target requests. ({} != {})", count, req.requests().size());
        }
        return () -> {
            final List<BulkItemResponse> responseList = new ArrayList<>(req.requests().size());
            for (int i = 0; i < req.requests().size(); i++) {
                final DocWriteRequest<?> dwr = req.requests().get(i);
                if (dwr instanceof IndexRequest) {
                    final IndexRequest r = (IndexRequest) dwr;
                    final String id = r.id() == null ? INDEX_UUID : r.id();
                    final IndexResponse response = new IndexResponse(new ShardId(new Index(r.index(), INDEX_UUID), 0), r.type(), id, r.version(), true);
                    responseList.add(new BulkItemResponse(i, r.opType(), response));
                } else if (dwr instanceof UpdateRequest) {
                    final UpdateRequest r = (UpdateRequest) dwr;
                    final String id = r.id() == null ? INDEX_UUID : r.id();
                    final UpdateResponse response = new UpdateResponse(new ShardId(new Index(r.index(), INDEX_UUID), 0), r.type(), id, r.version(), Result.CREATED);
                    responseList.add(new BulkItemResponse(i, r.opType(), response));
                } else if (dwr instanceof DeleteRequest) {
                    final DeleteRequest r = (DeleteRequest) dwr;
                    final String id = r.id() == null ? INDEX_UUID : r.id();
                    final DeleteResponse response = new DeleteResponse(new ShardId(new Index(r.index(), INDEX_UUID), 0), r.type(), id, r.version(), true);
                    response.setShardInfo(new ReplicationResponse.ShardInfo(1, 1, ReplicationResponse.EMPTY));
                    responseList.add(new BulkItemResponse(i, r.opType(), response));
                } else {
                    responseList.add(new BulkItemResponse(i, dwr.opType(), new BulkItemResponse.Failure(dwr.index(), dwr.type(), dwr.id(), new ElasticsearchException("Unknown request: " + dwr))));
                }
            }
            return (Response) new BulkResponse(responseList.toArray(new BulkItemResponse[responseList.size()]), (System.nanoTime() - startTime) / 1000000);
        };
    } else if (DeleteAction.NAME.equals(action)) {
        final DeleteRequest req = (DeleteRequest) request;
        if (!indexingProxyService.isTargetIndex(req.index())) {
            return null;
        }
        return () -> {
            final String id = req.id() == null ? INDEX_UUID : req.id();
            final DeleteResponse res = new DeleteResponse(new ShardId(new Index(req.index(), INDEX_UUID), 0), req.type(), id, req.version(), true);
            res.setShardInfo(new ReplicationResponse.ShardInfo(1, 1, ReplicationResponse.EMPTY));
            return (Response) res;
        };
    } else if (DeleteByQueryAction.NAME.equals(action)) {
        final long startTime = System.nanoTime();
        int count = 0;
        final DeleteByQueryRequest req = (DeleteByQueryRequest) request;
        for (final String index : req.indices()) {
            if (indexingProxyService.isTargetIndex(index)) {
                count++;
            }
        }
        if (count == 0) {
            return null;
        } else if (count != req.indices().length) {
            throw new ElasticsearchException("Mixed target requests. ({} != {})", count, req.indices().length);
        }
        return () -> {
            return (Response) new BulkByScrollResponse(TimeValue.timeValueNanos(System.nanoTime() - startTime), new BulkByScrollTask.Status(null, 0, 0, 0, 0, 0, 0, 0, 0, 0, TimeValue.ZERO, 0, null, TimeValue.ZERO), Collections.emptyList(), Collections.emptyList(), false);
        };
    } else if (IndexAction.NAME.equals(action)) {
        final IndexRequest req = (IndexRequest) request;
        if (!indexingProxyService.isTargetIndex(req.index())) {
            return null;
        }
        return () -> {
            final String id = req.id() == null ? INDEX_UUID : req.id();
            return (Response) new IndexResponse(new ShardId(new Index(req.index(), INDEX_UUID), 0), req.type(), id, req.version(), true);
        };
    } else if (UpdateAction.NAME.equals(action)) {
        final UpdateRequest req = (UpdateRequest) request;
        if (!indexingProxyService.isTargetIndex(req.index())) {
            return null;
        }
        return () -> {
            final String id = req.id() == null ? INDEX_UUID : req.id();
            return (Response) new UpdateResponse(new ShardId(new Index(req.index(), INDEX_UUID), 0), req.type(), id, req.version(), Result.CREATED);
        };
    } else if (UpdateByQueryAction.NAME.equals(action)) {
        final long startTime = System.nanoTime();
        int count = 0;
        final UpdateByQueryRequest req = (UpdateByQueryRequest) request;
        for (final String index : req.indices()) {
            if (indexingProxyService.isTargetIndex(index)) {
                count++;
            }
        }
        if (count == 0) {
            return null;
        } else if (count != req.indices().length) {
            throw new ElasticsearchException("Mixed target requests. ({} != {})", count, req.indices().length);
        }
        return () -> {
            return (Response) new BulkByScrollResponse(TimeValue.timeValueNanos(System.nanoTime() - startTime), new BulkByScrollTask.Status(null, 0, 0, 0, 0, 0, 0, 0, 0, 0, TimeValue.ZERO, 0, null, TimeValue.ZERO), Collections.emptyList(), Collections.emptyList(), false);
        };
    }
    return null;
}

Example 15 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project elasticsearch-indexing-proxy by codelibs.

the class RequestUtils method createBulkRequest.

public static BulkRequestBuilder createBulkRequest(final Client client, final StreamInput streamInput, final String index) throws IOException {
    final BulkRequestBuilder builder = client.prepareBulk();
    final BulkRequest request = builder.request();
    request.readFrom(streamInput);
    if (index != null) {
        request.requests().stream().forEach(req -> {
            if (req instanceof DeleteRequest) {
                ((DeleteRequest) req).index(index);
            } else if (req instanceof DeleteByQueryRequest) {
                ((DeleteByQueryRequest) req).indices(index);
            } else if (req instanceof IndexRequest) {
                ((IndexRequest) req).index(index);
            } else if (req instanceof UpdateRequest) {
                ((UpdateRequest) req).index(index);
            } else if (req instanceof UpdateByQueryRequest) {
                ((UpdateByQueryRequest) req).indices(index);
            } else {
                throw new ElasticsearchException("Unsupported request in bulk: " + req);
            }
        });
    }
    return builder;
}