Examples with BulkByScrollResponse org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.BulkByScrollResponse used on opensource projects

Example 21 with BulkByScrollResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.BulkByScrollResponse in project incubator-skywalking by apache.

the class AbstractPersistenceEsDAO method deleteHistory.

@Override
public final void deleteHistory(Long startTimestamp, Long endTimestamp) {
    long startTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(startTimestamp);
    long endTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(endTimestamp);
    BulkByScrollResponse response = getClient().prepareDelete(QueryBuilders.rangeQuery(timeBucketColumnNameForDelete()).gte(startTimeBucket).lte(endTimeBucket), tableName()).get();
    long deleted = response.getDeleted();
    logger.info("Delete {} rows history from {} index.", deleted, tableName());
}

Example 22 with BulkByScrollResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.BulkByScrollResponse in project incubator-skywalking by apache.

the class GlobalTraceEsPersistenceDAO method deleteHistory.

@Override
public void deleteHistory(Long startTimestamp, Long endTimestamp) {
    long startTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(startTimestamp);
    long endTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(endTimestamp);
    BulkByScrollResponse response = getClient().prepareDelete(QueryBuilders.rangeQuery(GlobalTraceTable.COLUMN_TIME_BUCKET).gte(startTimeBucket).lte(endTimeBucket), GlobalTraceTable.TABLE).get();
    long deleted = response.getDeleted();
    logger.info("Delete {} rows history from {} index.", deleted, GlobalTraceTable.TABLE);
}

Example 23 with BulkByScrollResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.BulkByScrollResponse in project incubator-skywalking by apache.

the class ApplicationAlarmEsPersistenceDAO method deleteHistory.

@Override
public void deleteHistory(Long startTimestamp, Long endTimestamp) {
    long startTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(startTimestamp);
    long endTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(endTimestamp);
    BulkByScrollResponse response = getClient().prepareDelete(QueryBuilders.rangeQuery(ApplicationAlarmTable.COLUMN_LAST_TIME_BUCKET).gte(startTimeBucket).lte(endTimeBucket), ApplicationAlarmTable.TABLE).get();
    long deleted = response.getDeleted();
    logger.info("Delete {} rows history from {} index.", deleted, ApplicationAlarmTable.TABLE);
}

Example 24 with BulkByScrollResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.BulkByScrollResponse in project incubator-skywalking by apache.

the class InstanceAlarmListEsPersistenceDAO method deleteHistory.

@Override
public void deleteHistory(Long startTimestamp, Long endTimestamp) {
    long startTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(startTimestamp);
    long endTimeBucket = TimeBucketUtils.INSTANCE.getMinuteTimeBucket(endTimestamp);
    BulkByScrollResponse response = getClient().prepareDelete(QueryBuilders.rangeQuery(InstanceAlarmListTable.COLUMN_TIME_BUCKET).gte(startTimeBucket).lte(endTimeBucket), InstanceAlarmListTable.TABLE).get();
    long deleted = response.getDeleted();
    logger.info("Delete {} rows history from {} index.", deleted, InstanceAlarmListTable.TABLE);
}

Example 25 with BulkByScrollResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.BulkByScrollResponse in project snow-owl by b2ihealthcare.

the class EsIndexAdmin method bulkIndexByScroll.

private boolean bulkIndexByScroll(final EsClient client, final DocumentMapping mapping, final Expression filter, final String command, final org.elasticsearch.script.Script script, final String operationDescription) {
    final QueryBuilder query = new EsQueryBuilder(mapping, settings, log).build(filter);
    boolean needsRefresh = false;
    long versionConflicts = 0;
    int attempts = DEFAULT_MAX_NUMBER_OF_VERSION_CONFLICT_RETRIES;
    do {
        try {
            final BulkByScrollResponse response;
            final int batchSize = Integer.parseInt((String) settings.get(IndexClientFactory.RESULT_WINDOW_KEY));
            if ("update".equals(command)) {
                response = client.updateByQuery(getTypeIndex(mapping), batchSize, script, query);
            } else if ("delete".equals(command)) {
                response = client.deleteByQuery(getTypeIndex(mapping), batchSize, query);
            } else {
                throw new UnsupportedOperationException("Not implemented command: " + command);
            }
            final long updateCount = response.getUpdated();
            final long deleteCount = response.getDeleted();
            final long noops = response.getNoops();
            final List<Failure> failures = response.getBulkFailures();
            versionConflicts = response.getVersionConflicts();
            boolean updated = updateCount > 0;
            if (updated) {
                log().info("Updated {} {} documents with bulk {}", updateCount, mapping.typeAsString(), operationDescription);
                needsRefresh = true;
            }
            boolean deleted = deleteCount > 0;
            if (deleted) {
                log().info("Deleted {} {} documents with bulk {}", deleteCount, mapping.typeAsString(), operationDescription);
                needsRefresh = true;
            }
            if (!updated && !deleted) {
                log().warn("Bulk {} could not be applied to {} documents, no-ops ({}), conflicts ({})", operationDescription, mapping.typeAsString(), noops, versionConflicts);
            }
            if (failures.size() > 0) {
                boolean versionConflictsOnly = true;
                for (Failure failure : failures) {
                    final String failureMessage = failure.getCause().getMessage();
                    final int failureStatus = failure.getStatus().getStatus();
                    if (failureStatus != RestStatus.CONFLICT.getStatus()) {
                        versionConflictsOnly = false;
                        log().error("Index failure during bulk update: {}", failureMessage);
                    } else {
                        log().warn("Version conflict reason: {}", failureMessage);
                    }
                }
                if (!versionConflictsOnly) {
                    throw new IllegalStateException("There were indexing failures during bulk updates. See logs for all failures.");
                }
            }
            if (attempts <= 0) {
                throw new IndexException("There were indexing failures during bulk updates. See logs for all failures.", null);
            }
            if (versionConflicts > 0) {
                --attempts;
                try {
                    Thread.sleep(100 + random.nextInt(900));
                    refresh(Collections.singleton(mapping));
                } catch (InterruptedException e) {
                    throw new IndexException("Interrupted", e);
                }
            }
        } catch (IOException e) {
            throw new IndexException("Could not execute bulk update.", e);
        }
    } while (versionConflicts > 0);
    return needsRefresh;
}