Examples with ReindexRequest org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.ReindexRequest used on opensource projects

Example 1 with ReindexRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.ReindexRequest in project pancm_project by xuwujing.

the class EsHighLevelRestTest2 method reindex.

/**
 * 索引复制
 *
 * @throws IOException
 */
private static void reindex() throws IOException {
    // 创建索引复制请求并进行索引复制
    ReindexRequest request = new ReindexRequest();
    // 需要复制的索引
    request.setSourceIndices("user");
    // 复制的目标索引
    request.setDestIndex("dest_test");
    // 表示如果在复制索引的时候有缺失的文档的话会进行创建,默认是index
    request.setDestOpType("create");
    // 如果在复制的过程中发现版本冲突，那么会继续进行复制
    request.setConflicts("proceed");
    // 只复制文档类型为 userindex 的数据
    request.setSourceDocTypes("userindex");
    // 只复制 pancm 用户的数据
    request.setSourceQuery(new TermQueryBuilder("user", "pancm"));
    // 设置复制文档的数量
    request.setSize(10);
    // 设置一次批量处理的条数，默认是1000
    request.setSourceBatchSize(100);
    // 进行排序
    // request.addSortField("postDate", SortOrder.DESC);
    // 指定切片大小
    request.setSlices(2);
    // 设置超时时间
    request.setTimeout(TimeValue.timeValueMinutes(2));
    // 允许刷新
    request.setRefresh(true);
    // 同步执行
    BulkByScrollResponse bulkResponse = client.reindex(request, RequestOptions.DEFAULT);
    // 异步执行
    // client.reindexAsync(request, RequestOptions.DEFAULT, listener);
    // 响应结果处理
    TimeValue timeTaken = bulkResponse.getTook();
    boolean timedOut = bulkResponse.isTimedOut();
    long totalDocs = bulkResponse.getTotal();
    long updatedDocs = bulkResponse.getUpdated();
    long createdDocs = bulkResponse.getCreated();
    long deletedDocs = bulkResponse.getDeleted();
    long batches = bulkResponse.getBatches();
    long noops = bulkResponse.getNoops();
    long versionConflicts = bulkResponse.getVersionConflicts();
    long bulkRetries = bulkResponse.getBulkRetries();
    long searchRetries = bulkResponse.getSearchRetries();
    TimeValue throttledMillis = bulkResponse.getStatus().getThrottled();
    TimeValue throttledUntilMillis = bulkResponse.getStatus().getThrottledUntil();
    List<ScrollableHitSource.SearchFailure> searchFailures = bulkResponse.getSearchFailures();
    List<BulkItemResponse.Failure> bulkFailures = bulkResponse.getBulkFailures();
    System.out.println("索引复制总共花费了:" + timeTaken.getMillis() + " 毫秒，总条数:" + totalDocs + ",创建数:" + createdDocs + ",更新数:" + updatedDocs);
}

Example 2 with ReindexRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.ReindexRequest in project graylog2-server by Graylog2.

the class IndicesAdapterES7 method move.

@Override
public void move(String source, String target, Consumer<IndexMoveResult> resultCallback) {
    final ReindexRequest request = new ReindexRequest();
    request.setSourceIndices(source);
    request.setDestIndex(target);
    final BulkByScrollResponse result = client.execute((c, requestOptions) -> c.reindex(request, requestOptions));
    final IndexMoveResult indexMoveResult = IndexMoveResult.create(Math.toIntExact(result.getTotal()), result.getTook().millis(), !result.getBulkFailures().isEmpty());
    resultCallback.accept(indexMoveResult);
}

Example 3 with ReindexRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.ReindexRequest in project pancm_project by xuwujing.

the class IpHandler method reindexByQuery.

/**
 * @return Map
 * @Author pancm
 * @Description //重索引
 * @Date 2019/3/21
 * @Param []
 */
public static Map<String, Object> reindexByQuery(String index, String destIndex, QueryBuilder[] queryBuilders) throws IOException {
    if (index == null || destIndex == null) {
        return null;
    }
    Map<String, Object> map = new HashMap<>();
    try {
        // 创建索引复制请求并进行索引复制
        ReindexRequest request = new ReindexRequest();
        // 需要复制的索引
        request.setSourceIndices(index);
        /* 复制的目标索引 */
        request.setDestIndex(destIndex);
        if (queryBuilders != null) {
            for (QueryBuilder queryBuilder : queryBuilders) {
                request.setSourceQuery(queryBuilder);
            }
        }
        // 表示如果在复制索引的时候有缺失的文档的话会进行创建,默认是index
        request.setDestOpType("create");
        // 如果在复制的过程中发现版本冲突，那么会继续进行复制
        request.setConflicts("proceed");
        // 设置复制文档的数量
        // request.setSize(10);
        // 设置一次批量处理的条数，默认是1000
        // request.setSourceBatchSize(10000);
        // 设置超时时间
        request.setTimeout(TimeValue.timeValueMinutes(2));
        // 同步执行
        BulkByScrollResponse bulkResponse = client.reindex(request, RequestOptions.DEFAULT);
        // 响应结果处理
        map.put("time", bulkResponse.getTook().getMillis());
        map.put("total", bulkResponse.getTotal());
        map.put("createdDocs", bulkResponse.getCreated());
        map.put("updatedDocs", bulkResponse.getUpdated());
    } finally {
        if (isAutoClose) {
            close();
        }
    }
    return map;
}

Example 4 with ReindexRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.index.reindex.ReindexRequest in project ranger by apache.

the class RequestUtils method getIndexFromRequest.

// To support all kinds of request in elasticsearch
public static <Request extends ActionRequest> List<String> getIndexFromRequest(Request request) {
    List<String> indexs = new ArrayList<>();
    if (request instanceof SingleShardRequest) {
        indexs.add(((SingleShardRequest<?>) request).index());
        return indexs;
    }
    if (request instanceof ReplicationRequest) {
        indexs.add(((ReplicationRequest<?>) request).index());
        return indexs;
    }
    if (request instanceof InstanceShardOperationRequest) {
        indexs.add(((InstanceShardOperationRequest<?>) request).index());
        return indexs;
    }
    if (request instanceof CreateIndexRequest) {
        indexs.add(((CreateIndexRequest) request).index());
        return indexs;
    }
    if (request instanceof PutMappingRequest) {
        if (((PutMappingRequest) request).getConcreteIndex() != null) {
            indexs.add(((PutMappingRequest) request).getConcreteIndex().getName());
            return indexs;
        } else {
            return Arrays.asList(((PutMappingRequest) request).indices());
        }
    }
    if (request instanceof SearchRequest) {
        return Arrays.asList(((SearchRequest) request).indices());
    }
    if (request instanceof IndicesStatsRequest) {
        return Arrays.asList(((IndicesStatsRequest) request).indices());
    }
    if (request instanceof OpenIndexRequest) {
        return Arrays.asList(((OpenIndexRequest) request).indices());
    }
    if (request instanceof DeleteIndexRequest) {
        return Arrays.asList(((DeleteIndexRequest) request).indices());
    }
    if (request instanceof BulkRequest) {
        @SuppressWarnings("rawtypes") List<DocWriteRequest<?>> requests = ((BulkRequest) request).requests();
        if (CollectionUtils.isNotEmpty(requests)) {
            for (DocWriteRequest<?> docWriteRequest : requests) {
                indexs.add(docWriteRequest.index());
            }
            return indexs;
        }
    }
    if (request instanceof MultiGetRequest) {
        List<Item> items = ((MultiGetRequest) request).getItems();
        if (CollectionUtils.isNotEmpty(items)) {
            for (Item item : items) {
                indexs.add(item.index());
            }
            return indexs;
        }
    }
    if (request instanceof GetMappingsRequest) {
        return Arrays.asList(((GetMappingsRequest) request).indices());
    }
    if (request instanceof GetSettingsRequest) {
        return Arrays.asList(((GetSettingsRequest) request).indices());
    }
    if (request instanceof IndicesExistsRequest) {
        return Arrays.asList(((IndicesExistsRequest) request).indices());
    }
    if (request instanceof GetAliasesRequest) {
        return Arrays.asList(((GetAliasesRequest) request).indices());
    }
    if (request instanceof GetIndexRequest) {
        return Arrays.asList(((GetIndexRequest) request).indices());
    }
    if (request instanceof GetFieldMappingsRequest) {
        return Arrays.asList(((GetFieldMappingsRequest) request).indices());
    }
    if (request instanceof TypesExistsRequest) {
        return Arrays.asList(((TypesExistsRequest) request).indices());
    }
    if (request instanceof ValidateQueryRequest) {
        return Arrays.asList(((ValidateQueryRequest) request).indices());
    }
    if (request instanceof RecoveryRequest) {
        return Arrays.asList(((RecoveryRequest) request).indices());
    }
    if (request instanceof IndicesSegmentsRequest) {
        return Arrays.asList(((IndicesSegmentsRequest) request).indices());
    }
    if (request instanceof IndicesShardStoresRequest) {
        return Arrays.asList(((IndicesShardStoresRequest) request).indices());
    }
    if (request instanceof UpgradeStatusRequest) {
        return Arrays.asList(((UpgradeStatusRequest) request).indices());
    }
    if (request instanceof ClusterSearchShardsRequest) {
        return Arrays.asList(((ClusterSearchShardsRequest) request).indices());
    }
    if (request instanceof IndicesAliasesRequest) {
        List<IndicesAliasesRequest.AliasActions> aliasActions = ((IndicesAliasesRequest) request).getAliasActions();
        if (CollectionUtils.isNotEmpty(aliasActions)) {
            for (IndicesAliasesRequest.AliasActions action : aliasActions) {
                indexs.addAll(Arrays.asList(action.indices()));
            }
            return indexs;
        }
    }
    if (request instanceof ClearIndicesCacheRequest) {
        return Arrays.asList(((ClearIndicesCacheRequest) request).indices());
    }
    if (request instanceof CloseIndexRequest) {
        return Arrays.asList(((CloseIndexRequest) request).indices());
    }
    if (request instanceof FlushRequest) {
        return Arrays.asList(((FlushRequest) request).indices());
    }
    if (request instanceof SyncedFlushRequest) {
        return Arrays.asList(((SyncedFlushRequest) request).indices());
    }
    if (request instanceof ForceMergeRequest) {
        return Arrays.asList(((ForceMergeRequest) request).indices());
    }
    if (request instanceof RefreshRequest) {
        return Arrays.asList(((RefreshRequest) request).indices());
    }
    if (request instanceof RolloverRequest) {
        return Arrays.asList(((RolloverRequest) request).indices());
    }
    if (request instanceof UpdateSettingsRequest) {
        return Arrays.asList(((UpdateSettingsRequest) request).indices());
    }
    if (request instanceof ResizeRequest) {
        return Arrays.asList(((ResizeRequest) request).indices());
    }
    if (request instanceof DeleteIndexTemplateRequest) {
        indexs.add(((DeleteIndexTemplateRequest) request).name());
        return indexs;
    }
    if (request instanceof GetIndexTemplatesRequest) {
        return Arrays.asList(((GetIndexTemplatesRequest) request).names());
    }
    if (request instanceof PutIndexTemplateRequest) {
        indexs.add(((PutIndexTemplateRequest) request).name());
        return indexs;
    }
    if (request instanceof UpgradeRequest) {
        return Arrays.asList(((UpgradeRequest) request).indices());
    }
    if (request instanceof FieldCapabilitiesRequest) {
        return Arrays.asList(((FieldCapabilitiesRequest) request).indices());
    }
    if (request instanceof MultiSearchRequest) {
        List<SearchRequest> searchRequests = ((MultiSearchRequest) request).requests();
        if (CollectionUtils.isNotEmpty(searchRequests)) {
            for (SearchRequest singleRequest : searchRequests) {
                indexs.addAll(Arrays.asList(singleRequest.indices()));
            }
            return indexs;
        }
    }
    if (request instanceof MultiTermVectorsRequest) {
        List<TermVectorsRequest> termVectorsRequests = ((MultiTermVectorsRequest) request).getRequests();
        if (CollectionUtils.isNotEmpty(termVectorsRequests)) {
            for (TermVectorsRequest singleRequest : termVectorsRequests) {
                indexs.addAll(Arrays.asList(singleRequest.indices()));
            }
            return indexs;
        }
    }
    if (request instanceof UpdateByQueryRequest) {
        return Arrays.asList(((UpdateByQueryRequest) request).indices());
    }
    if (request instanceof DeleteByQueryRequest) {
        return Arrays.asList(((DeleteByQueryRequest) request).indices());
    }
    if (request instanceof ReindexRequest) {
        indexs.addAll(Arrays.asList(((ReindexRequest) request).getSearchRequest().indices()));
        indexs.addAll(Arrays.asList(((ReindexRequest) request).getDestination().indices()));
        return indexs;
    }
    // ClearScrollRequest does not carry any index, so return empty List
    if (request instanceof ClearScrollRequest) {
        return indexs;
    }
    // No matched request type to find specific index , set default value *
    indexs.add("*");
    return indexs;
}