Examples with BulkResponse org.elasticsearch.action.bulk.BulkResponse used on opensource projects

Example 26 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project ecs-dashboard by carone1.

the class ElasticBillingDAO method insert.

/**
 * {@inheritDoc}
 */
@Override
public void insert(ObjectBuckets objectBuckets, Date collectionTime) {
    if (objectBuckets == null || objectBuckets.getObjectBucket() == null || objectBuckets.getObjectBucket().isEmpty()) {
        // nothing to insert
        return;
    }
    BulkRequestBuilder requestBuilder = elasticClient.prepareBulk();
    // Generate JSON for object buckets info
    for (ObjectBucket objectBucket : objectBuckets.getObjectBucket()) {
        XContentBuilder objectBucketBuilder = toJsonFormat(objectBucket, collectionTime);
        IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(objectBucketIndexDayName).setType(OBJECT_BUCKET_INDEX_TYPE).setSource(objectBucketBuilder);
        requestBuilder.add(request);
    }
    BulkResponse bulkResponse = requestBuilder.execute().actionGet();
    int items = bulkResponse.getItems().length;
    LOGGER.info("Took " + bulkResponse.getTookInMillis() + " ms to index [" + items + "] items in ElasticSearch" + "index: " + objectBucketIndexDayName + " index type: " + OBJECT_BUCKET_INDEX_TYPE);
    if (bulkResponse.hasFailures()) {
        LOGGER.error("Failures occured while items in ElasticSearch " + "index: " + objectBucketIndexDayName + " index type: " + OBJECT_BUCKET_INDEX_TYPE);
    }
}

Example 27 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project ecs-dashboard by carone1.

the class ElasticS3ObjectDAO method insert.

/**
 * {@inheritDoc}
 */
@Override
public void insert(ListObjectsResult listObjectsResult, String namespace, String bucket, Date collectionTime) {
    if (listObjectsResult == null || listObjectsResult.getObjects() == null || listObjectsResult.getObjects().isEmpty()) {
        // nothing to insert
        return;
    }
    BulkRequestBuilder requestBuilder = elasticClient.prepareBulk();
    // Generate JSON for object buckets info
    for (S3Object s3Object : listObjectsResult.getObjects()) {
        XContentBuilder s3ObjectBuilder = toJsonFormat(s3Object, namespace, bucket, collectionTime);
        IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(s3ObjectIndexDayName).setType(S3_OBJECT_INDEX_TYPE).setSource(s3ObjectBuilder);
        requestBuilder.add(request);
    }
    BulkResponse bulkResponse = requestBuilder.execute().actionGet();
    int items = bulkResponse.getItems().length;
    LOGGER.info("Took " + bulkResponse.getTookInMillis() + " ms to index [" + items + "] items in Elasticsearch " + "index: " + s3ObjectIndexDayName + " index type: " + S3_OBJECT_INDEX_TYPE);
    if (bulkResponse.hasFailures()) {
        LOGGER.error("Failure(s) occured while items in Elasticsearch " + "index: " + s3ObjectIndexDayName + " index type: " + S3_OBJECT_INDEX_TYPE);
    }
}

Example 28 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project incubator-sdap-mudrod by apache.

the class ESDriver method createBulkProcessor.

public void createBulkProcessor() {
    LOG.debug("Creating BulkProcessor with maxBulkDocs={}, maxBulkLength={}", 1000, 2500500);
    setBulkProcessor(BulkProcessor.builder(getClient(), new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
            LOG.debug("ESDriver#createBulkProcessor @Override #beforeBulk is not implemented yet!");
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            LOG.debug("ESDriver#createBulkProcessor @Override #afterBulk is not implemented yet!");
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            LOG.error("Bulk request has failed!");
            throw new RuntimeException("Caught exception in bulk: " + request.getDescription() + ", failure: " + failure, failure);
        }
    }).setBulkActions(1000).setBulkSize(new ByteSizeValue(2500500, ByteSizeUnit.GB)).setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 10)).setConcurrentRequests(1).build());
}

Example 29 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project elasticsearch-indexing-proxy by codelibs.

the class ProxyActionFilter method getExecutor.

@SuppressWarnings("unchecked")
private <Request extends ActionRequest, Response extends ActionResponse> Supplier<Response> getExecutor(final Task task, final String action, final Request request) {
    if (BulkAction.NAME.equals(action)) {
        final long startTime = System.nanoTime();
        int count = 0;
        final BulkRequest req = (BulkRequest) request;
        for (final DocWriteRequest<?> subReq : req.requests()) {
            if (indexingProxyService.isTargetIndex(subReq.index())) {
                count++;
            }
        }
        if (count == 0) {
            return null;
        } else if (count != req.requests().size()) {
            throw new ElasticsearchException("Mixed target requests. ({} != {})", count, req.requests().size());
        }
        return () -> {
            final List<BulkItemResponse> responseList = new ArrayList<>(req.requests().size());
            for (int i = 0; i < req.requests().size(); i++) {
                final DocWriteRequest<?> dwr = req.requests().get(i);
                if (dwr instanceof IndexRequest) {
                    final IndexRequest r = (IndexRequest) dwr;
                    final String id = r.id() == null ? INDEX_UUID : r.id();
                    final IndexResponse response = new IndexResponse(new ShardId(new Index(r.index(), INDEX_UUID), 0), r.type(), id, r.version(), true);
                    responseList.add(new BulkItemResponse(i, r.opType(), response));
                } else if (dwr instanceof UpdateRequest) {
                    final UpdateRequest r = (UpdateRequest) dwr;
                    final String id = r.id() == null ? INDEX_UUID : r.id();
                    final UpdateResponse response = new UpdateResponse(new ShardId(new Index(r.index(), INDEX_UUID), 0), r.type(), id, r.version(), Result.CREATED);
                    responseList.add(new BulkItemResponse(i, r.opType(), response));
                } else if (dwr instanceof DeleteRequest) {
                    final DeleteRequest r = (DeleteRequest) dwr;
                    final String id = r.id() == null ? INDEX_UUID : r.id();
                    final DeleteResponse response = new DeleteResponse(new ShardId(new Index(r.index(), INDEX_UUID), 0), r.type(), id, r.version(), true);
                    response.setShardInfo(new ReplicationResponse.ShardInfo(1, 1, ReplicationResponse.EMPTY));
                    responseList.add(new BulkItemResponse(i, r.opType(), response));
                } else {
                    responseList.add(new BulkItemResponse(i, dwr.opType(), new BulkItemResponse.Failure(dwr.index(), dwr.type(), dwr.id(), new ElasticsearchException("Unknown request: " + dwr))));
                }
            }
            return (Response) new BulkResponse(responseList.toArray(new BulkItemResponse[responseList.size()]), (System.nanoTime() - startTime) / 1000000);
        };
    } else if (DeleteAction.NAME.equals(action)) {
        final DeleteRequest req = (DeleteRequest) request;
        if (!indexingProxyService.isTargetIndex(req.index())) {
            return null;
        }
        return () -> {
            final String id = req.id() == null ? INDEX_UUID : req.id();
            final DeleteResponse res = new DeleteResponse(new ShardId(new Index(req.index(), INDEX_UUID), 0), req.type(), id, req.version(), true);
            res.setShardInfo(new ReplicationResponse.ShardInfo(1, 1, ReplicationResponse.EMPTY));
            return (Response) res;
        };
    } else if (DeleteByQueryAction.NAME.equals(action)) {
        final long startTime = System.nanoTime();
        int count = 0;
        final DeleteByQueryRequest req = (DeleteByQueryRequest) request;
        for (final String index : req.indices()) {
            if (indexingProxyService.isTargetIndex(index)) {
                count++;
            }
        }
        if (count == 0) {
            return null;
        } else if (count != req.indices().length) {
            throw new ElasticsearchException("Mixed target requests. ({} != {})", count, req.indices().length);
        }
        return () -> {
            return (Response) new BulkByScrollResponse(TimeValue.timeValueNanos(System.nanoTime() - startTime), new BulkByScrollTask.Status(null, 0, 0, 0, 0, 0, 0, 0, 0, 0, TimeValue.ZERO, 0, null, TimeValue.ZERO), Collections.emptyList(), Collections.emptyList(), false);
        };
    } else if (IndexAction.NAME.equals(action)) {
        final IndexRequest req = (IndexRequest) request;
        if (!indexingProxyService.isTargetIndex(req.index())) {
            return null;
        }
        return () -> {
            final String id = req.id() == null ? INDEX_UUID : req.id();
            return (Response) new IndexResponse(new ShardId(new Index(req.index(), INDEX_UUID), 0), req.type(), id, req.version(), true);
        };
    } else if (UpdateAction.NAME.equals(action)) {
        final UpdateRequest req = (UpdateRequest) request;
        if (!indexingProxyService.isTargetIndex(req.index())) {
            return null;
        }
        return () -> {
            final String id = req.id() == null ? INDEX_UUID : req.id();
            return (Response) new UpdateResponse(new ShardId(new Index(req.index(), INDEX_UUID), 0), req.type(), id, req.version(), Result.CREATED);
        };
    } else if (UpdateByQueryAction.NAME.equals(action)) {
        final long startTime = System.nanoTime();
        int count = 0;
        final UpdateByQueryRequest req = (UpdateByQueryRequest) request;
        for (final String index : req.indices()) {
            if (indexingProxyService.isTargetIndex(index)) {
                count++;
            }
        }
        if (count == 0) {
            return null;
        } else if (count != req.indices().length) {
            throw new ElasticsearchException("Mixed target requests. ({} != {})", count, req.indices().length);
        }
        return () -> {
            return (Response) new BulkByScrollResponse(TimeValue.timeValueNanos(System.nanoTime() - startTime), new BulkByScrollTask.Status(null, 0, 0, 0, 0, 0, 0, 0, 0, 0, TimeValue.ZERO, 0, null, TimeValue.ZERO), Collections.emptyList(), Collections.emptyList(), false);
        };
    }
    return null;
}

Example 30 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project opencast by opencast.

the class AbstractElasticsearchIndex method update.

/**
 * Posts the input document to the search index.
 *
 * @param documents
 *          the input documents
 * @return the query response
 * @throws SearchIndexException
 *           if posting to the index fails
 */
protected BulkResponse update(ElasticsearchDocument... documents) throws SearchIndexException {
    BulkRequestBuilder bulkRequest = nodeClient.prepareBulk();
    for (ElasticsearchDocument doc : documents) {
        String type = doc.getType();
        String uid = doc.getUID();
        bulkRequest.add(nodeClient.prepareIndex(index, type, uid).setSource(doc));
    }
    // Make sure the operations are searchable immediately
    bulkRequest.setRefresh(true);
    try {
        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
        // Check for errors
        if (bulkResponse.hasFailures()) {
            for (BulkItemResponse item : bulkResponse.getItems()) {
                if (item.isFailed()) {
                    logger.warn("Error updating {}: {}", item, item.getFailureMessage());
                    throw new SearchIndexException(item.getFailureMessage());
                }
            }
        }
        return bulkResponse;
    } catch (Throwable t) {
        throw new SearchIndexException("Cannot update documents in index " + index, t);
    }
}