Examples with BulkResponse org.elasticsearch.action.bulk.BulkResponse used on opensource projects

Example 91 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project elasticsearch-river-couchdb by elastic.

the class CouchdbRiver method start.

@Override
public void start() {
    logger.info("starting couchdb stream: host [{}], port [{}], filter [{}], db [{}], indexing to [{}]/[{}]", couchHost, couchPort, couchFilter, couchDb, indexName, typeName);
    // Creating bulk processor
    this.bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
            logger.debug("Going to execute new bulk composed of {} actions", request.numberOfActions());
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            logger.debug("Executed bulk composed of {} actions", request.numberOfActions());
            if (response.hasFailures()) {
                logger.warn("There was failures while executing bulk", response.buildFailureMessage());
                if (logger.isDebugEnabled()) {
                    for (BulkItemResponse item : response.getItems()) {
                        if (item.isFailed()) {
                            logger.debug("Error for {}/{}/{} for {} operation: {}", item.getIndex(), item.getType(), item.getId(), item.getOpType(), item.getFailureMessage());
                        }
                    }
                }
            }
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            logger.warn("Error executing bulk", failure);
        }
    }).setBulkActions(bulkSize).setConcurrentRequests(maxConcurrentBulk).setFlushInterval(bulkFlushInterval).build();
    slurperThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "couchdb_river_slurper").newThread(new Slurper());
    indexerThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "couchdb_river_indexer").newThread(new Indexer());
    indexerThread.start();
    slurperThread.start();
}

Example 92 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project fess-crawler by codelibs.

the class AbstractCrawlerService method delete.

public void delete(final Consumer<SearchRequestBuilder> callback) {
    SearchResponse response = null;
    while (true) {
        if (response == null) {
            response = getClient().get(c -> {
                final SearchRequestBuilder builder = c.prepareSearch(index).setTypes(type).setScroll(new TimeValue(scrollTimeout)).setSize(scrollSize);
                callback.accept(builder);
                return builder.execute();
            });
        } else {
            final String scrollId = response.getScrollId();
            response = getClient().get(c -> c.prepareSearchScroll(scrollId).setScroll(new TimeValue(scrollTimeout)).execute());
        }
        final SearchHits searchHits = response.getHits();
        if (searchHits.getHits().length == 0) {
            break;
        }
        final BulkResponse bulkResponse = getClient().get(c -> {
            final BulkRequestBuilder bulkBuilder = c.prepareBulk();
            for (final SearchHit searchHit : searchHits) {
                bulkBuilder.add(c.prepareDelete(index, type, searchHit.getId()));
            }
            return bulkBuilder.execute();
        });
        if (bulkResponse.hasFailures()) {
            throw new EsAccessException(bulkResponse.buildFailureMessage());
        }
    }
    refresh();
}

Example 93 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project fess-crawler by codelibs.

the class AbstractCrawlerService method insertAll.

protected <T> void insertAll(final List<T> list, final OpType opType, final boolean ignoreAlreadyExists) {
    final List<T> bufferedList = new ArrayList<>(bulkBufferSize);
    final StringBuilder failureBuf = new StringBuilder(100);
    list.stream().forEach(target -> {
        bufferedList.add(target);
        if (bufferedList.size() >= bulkBufferSize) {
            final BulkResponse response = doInsertAll(bufferedList, opType);
            if (response.hasFailures()) {
                final String failureMessage = buildFailureMessage(response, ignoreAlreadyExists);
                if (failureMessage.length() > 0) {
                    failureBuf.append(response.buildFailureMessage()).append('\n');
                }
            }
            bufferedList.clear();
        }
    });
    if (!bufferedList.isEmpty()) {
        final BulkResponse response = doInsertAll(bufferedList, opType);
        if (response.hasFailures()) {
            final String failureMessage = buildFailureMessage(response, ignoreAlreadyExists);
            if (failureMessage.length() > 0) {
                failureBuf.append(response.buildFailureMessage()).append('\n');
            }
        }
    }
    if (failureBuf.length() > 0) {
        throw new EsAccessException(failureBuf.toString());
    }
}

Example 94 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project fess-crawler by codelibs.

the class EsClient method deleteByQuery.

public int deleteByQuery(final String index, final String type, final QueryBuilder queryBuilder) {
    int count = 0;
    String scrollId = null;
    while (true) {
        final SearchResponse scrollResponse;
        if (scrollId == null) {
            scrollResponse = get(c -> c.prepareSearch(index).setTypes(type).setScroll(scrollForDelete).setSize(sizeForDelete).setQuery(queryBuilder).execute());
        } else {
            final String sid = scrollId;
            scrollResponse = get(c -> c.prepareSearchScroll(sid).setScroll(scrollForDelete).execute());
        }
        final SearchHit[] hits = scrollResponse.getHits().getHits();
        if (hits.length == 0) {
            break;
        }
        scrollId = scrollResponse.getScrollId();
        count += hits.length;
        final BulkResponse bulkResponse = get(c -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                bulkRequest.add(client.prepareDelete(hit.getIndex(), hit.getType(), hit.getId()));
            }
            return bulkRequest.execute();
        });
        if (bulkResponse.hasFailures()) {
            throw new EsAccessException(bulkResponse.buildFailureMessage());
        }
    }
    return count;
}

Example 95 with BulkResponse

use of org.elasticsearch.action.bulk.BulkResponse in project samza by apache.

the class ElasticsearchSystemProducerTest method getRespWithFailedDocument.

private BulkResponse getRespWithFailedDocument(RestStatus status) {
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(true);
    BulkItemResponse itemResp = mock(BulkItemResponse.class);
    when(itemResp.isFailed()).thenReturn(true);
    BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class);
    when(failure.getStatus()).thenReturn(status);
    when(itemResp.getFailure()).thenReturn(failure);
    BulkItemResponse[] itemResponses = new BulkItemResponse[] { itemResp };
    when(response.getItems()).thenReturn(itemResponses);
    return response;
}