Examples with BulkByScrollResponse org.elasticsearch.index.reindex.BulkByScrollResponse used on opensource projects

Example 21 with BulkByScrollResponse

use of org.elasticsearch.index.reindex.BulkByScrollResponse in project incubator-skywalking by apache.

the class GlobalTraceEsPersistenceDAO method deleteHistory.

@Override
public void deleteHistory(Long startTimestamp, Long endTimestamp) {
    long startTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(startTimestamp);
    long endTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(endTimestamp);
    BulkByScrollResponse response = getClient().prepareDelete(QueryBuilders.rangeQuery(GlobalTraceTable.COLUMN_TIME_BUCKET).gte(startTimeBucket).lte(endTimeBucket), GlobalTraceTable.TABLE).get();
    long deleted = response.getDeleted();
    logger.info("Delete {} rows history from {} index.", deleted, GlobalTraceTable.TABLE);
}

Example 22 with BulkByScrollResponse

use of org.elasticsearch.index.reindex.BulkByScrollResponse in project incubator-skywalking by apache.

the class ApplicationAlarmEsPersistenceDAO method deleteHistory.

@Override
public void deleteHistory(Long startTimestamp, Long endTimestamp) {
    long startTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(startTimestamp);
    long endTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(endTimestamp);
    BulkByScrollResponse response = getClient().prepareDelete(QueryBuilders.rangeQuery(ApplicationAlarmTable.COLUMN_LAST_TIME_BUCKET).gte(startTimeBucket).lte(endTimeBucket), ApplicationAlarmTable.TABLE).get();
    long deleted = response.getDeleted();
    logger.info("Delete {} rows history from {} index.", deleted, ApplicationAlarmTable.TABLE);
}

Example 23 with BulkByScrollResponse

use of org.elasticsearch.index.reindex.BulkByScrollResponse in project incubator-skywalking by apache.

the class InstanceAlarmListEsPersistenceDAO method deleteHistory.

@Override
public void deleteHistory(Long startTimestamp, Long endTimestamp) {
    long startTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(startTimestamp);
    long endTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(endTimestamp);
    BulkByScrollResponse response = getClient().prepareDelete(QueryBuilders.rangeQuery(InstanceAlarmListTable.COLUMN_TIME_BUCKET).gte(startTimeBucket).lte(endTimeBucket), InstanceAlarmListTable.TABLE).get();
    long deleted = response.getDeleted();
    logger.info("Delete {} rows history from {} index.", deleted, InstanceAlarmListTable.TABLE);
}

Example 24 with BulkByScrollResponse

use of org.elasticsearch.index.reindex.BulkByScrollResponse in project snow-owl by b2ihealthcare.

the class EsIndexAdmin method bulkIndexByScroll.

private boolean bulkIndexByScroll(final EsClient client, final DocumentMapping mapping, final Expression filter, final String command, final org.elasticsearch.script.Script script, final String operationDescription) {
    final QueryBuilder query = new EsQueryBuilder(mapping, settings, log).build(filter);
    boolean needsRefresh = false;
    long versionConflicts = 0;
    int attempts = DEFAULT_MAX_NUMBER_OF_VERSION_CONFLICT_RETRIES;
    do {
        try {
            final BulkByScrollResponse response;
            final int batchSize = Integer.parseInt((String) settings.get(IndexClientFactory.RESULT_WINDOW_KEY));
            if ("update".equals(command)) {
                response = client.updateByQuery(getTypeIndex(mapping), batchSize, script, query);
            } else if ("delete".equals(command)) {
                response = client.deleteByQuery(getTypeIndex(mapping), batchSize, query);
            } else {
                throw new UnsupportedOperationException("Not implemented command: " + command);
            }
            final long updateCount = response.getUpdated();
            final long deleteCount = response.getDeleted();
            final long noops = response.getNoops();
            final List<Failure> failures = response.getBulkFailures();
            versionConflicts = response.getVersionConflicts();
            boolean updated = updateCount > 0;
            if (updated) {
                log().info("Updated {} {} documents with bulk {}", updateCount, mapping.typeAsString(), operationDescription);
                needsRefresh = true;
            }
            boolean deleted = deleteCount > 0;
            if (deleted) {
                log().info("Deleted {} {} documents with bulk {}", deleteCount, mapping.typeAsString(), operationDescription);
                needsRefresh = true;
            }
            if (!updated && !deleted) {
                log().warn("Bulk {} could not be applied to {} documents, no-ops ({}), conflicts ({})", operationDescription, mapping.typeAsString(), noops, versionConflicts);
            }
            if (failures.size() > 0) {
                boolean versionConflictsOnly = true;
                for (Failure failure : failures) {
                    final String failureMessage = failure.getCause().getMessage();
                    final int failureStatus = failure.getStatus().getStatus();
                    if (failureStatus != RestStatus.CONFLICT.getStatus()) {
                        versionConflictsOnly = false;
                        log().error("Index failure during bulk update: {}", failureMessage);
                    } else {
                        log().warn("Version conflict reason: {}", failureMessage);
                    }
                }
                if (!versionConflictsOnly) {
                    throw new IllegalStateException("There were indexing failures during bulk updates. See logs for all failures.");
                }
            }
            if (attempts <= 0) {
                throw new IndexException("There were indexing failures during bulk updates. See logs for all failures.", null);
            }
            if (versionConflicts > 0) {
                --attempts;
                try {
                    Thread.sleep(100 + random.nextInt(900));
                    refresh(Collections.singleton(mapping));
                } catch (InterruptedException e) {
                    throw new IndexException("Interrupted", e);
                }
            }
        } catch (IOException e) {
            throw new IndexException("Could not execute bulk update.", e);
        }
    } while (versionConflicts > 0);
    return needsRefresh;
}

Example 25 with BulkByScrollResponse

use of org.elasticsearch.index.reindex.BulkByScrollResponse in project pancm_project by xuwujing.

the class EsHighLevelRestTest1 method deleteByQuery.

/**
 * 根据查询条件删除
 *
 * @throws IOException
 */
private static void deleteByQuery() throws IOException {
    String type = "_doc";
    String index = "test1";
    DeleteByQueryRequest request = new DeleteByQueryRequest(index, type);
    SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
    sourceBuilder.sort("sendtime", SortOrder.ASC);
    // 设置索引库表达式
    BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
    boolQueryBuilder.must(QueryBuilders.termQuery("uid", 123456));
    sourceBuilder.query(boolQueryBuilder);
    // 设置查询条件
    request.setQuery(sourceBuilder.query());
    request.setSize(1);
    // 同步执行
    BulkByScrollResponse bulkResponse = client.deleteByQuery(request, RequestOptions.DEFAULT);
    // 异步执行
    // client.updateByQueryAsync(request, RequestOptions.DEFAULT, listener);
    // 返回结果
    TimeValue timeTaken = bulkResponse.getTook();
    boolean timedOut = bulkResponse.isTimedOut();
    long totalDocs = bulkResponse.getTotal();
    long updatedDocs = bulkResponse.getUpdated();
    long deletedDocs = bulkResponse.getDeleted();
    long batches = bulkResponse.getBatches();
    long noops = bulkResponse.getNoops();
    long versionConflicts = bulkResponse.getVersionConflicts();
    long bulkRetries = bulkResponse.getBulkRetries();
    long searchRetries = bulkResponse.getSearchRetries();
    TimeValue throttledMillis = bulkResponse.getStatus().getThrottled();
    TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil();
    List<ScrollableHitSource.SearchFailure> searchFailures = bulkResponse.getSearchFailures();
    List<BulkItemResponse.Failure> bulkFailures = bulkResponse.getBulkFailures();
    System.out.println("查询更新总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",更新数:" + updatedDocs);
}