Examples with DeleteRequest org.elasticsearch.action.delete.DeleteRequest used on opensource projects

Example 51 with DeleteRequest

use of org.elasticsearch.action.delete.DeleteRequest in project snow-owl by b2ihealthcare.

the class EsDocumentWriter method commit.

@Override
public void commit() throws IOException {
    if (isEmpty()) {
        return;
    }
    final Set<DocumentMapping> mappingsToRefresh = Collections.synchronizedSet(newHashSet());
    final EsClient client = admin.client();
    // apply bulk updates first
    final ListeningExecutorService executor;
    if (bulkUpdateOperations.size() > 1 || bulkDeleteOperations.size() > 1) {
        final int threads = Math.min(4, Math.max(bulkUpdateOperations.size(), bulkDeleteOperations.size()));
        executor = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(threads));
    } else {
        executor = MoreExecutors.newDirectExecutorService();
    }
    final List<ListenableFuture<?>> updateFutures = newArrayList();
    for (BulkUpdate<?> update : bulkUpdateOperations) {
        updateFutures.add(executor.submit(() -> {
            if (admin.bulkUpdate(update)) {
                mappingsToRefresh.add(admin.mappings().getMapping(update.getType()));
            }
        }));
    }
    for (BulkDelete<?> delete : bulkDeleteOperations) {
        updateFutures.add(executor.submit(() -> {
            if (admin.bulkDelete(delete)) {
                mappingsToRefresh.add(admin.mappings().getMapping(delete.getType()));
            }
        }));
    }
    try {
        executor.shutdown();
        Futures.allAsList(updateFutures).get();
        executor.awaitTermination(10, TimeUnit.SECONDS);
    } catch (InterruptedException | ExecutionException e) {
        admin.log().error("Couldn't execute bulk updates", e);
        throw new IndexException("Couldn't execute bulk updates", e);
    }
    // then bulk indexes/deletes
    if (!indexOperations.isEmpty() || !deleteOperations.isEmpty()) {
        final BulkProcessor processor = client.bulk(new BulkProcessor.Listener() {

            @Override
            public void beforeBulk(long executionId, BulkRequest request) {
                admin.log().debug("Sending bulk request {}", request.numberOfActions());
            }

            @Override
            public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
                admin.log().error("Failed bulk request", failure);
            }

            @Override
            public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
                admin.log().debug("Successfully processed bulk request ({}) in {}.", request.numberOfActions(), response.getTook());
                if (response.hasFailures()) {
                    for (BulkItemResponse itemResponse : response.getItems()) {
                        checkState(!itemResponse.isFailed(), "Failed to commit bulk request in index '%s', %s", admin.name(), itemResponse.getFailureMessage());
                    }
                }
            }
        }).setConcurrentRequests(getConcurrencyLevel()).setBulkActions((int) admin.settings().get(IndexClientFactory.BULK_ACTIONS_SIZE)).setBulkSize(new ByteSizeValue((int) admin.settings().get(IndexClientFactory.BULK_ACTIONS_SIZE_IN_MB), ByteSizeUnit.MB)).build();
        for (Class<?> type : ImmutableSet.copyOf(indexOperations.rowKeySet())) {
            final Map<String, Object> indexOperationsForType = indexOperations.row(type);
            final DocumentMapping mapping = admin.mappings().getMapping(type);
            final String typeIndex = admin.getTypeIndex(mapping);
            mappingsToRefresh.add(mapping);
            for (Entry<String, Object> entry : Iterables.consumingIterable(indexOperationsForType.entrySet())) {
                final String id = entry.getKey();
                if (!deleteOperations.containsValue(id)) {
                    final Object obj = entry.getValue();
                    final byte[] _source = mapper.writeValueAsBytes(obj);
                    IndexRequest indexRequest = new IndexRequest().index(typeIndex).opType(OpType.INDEX).source(_source, XContentType.JSON);
                    // XXX revisions has their special local ID, but that's not needed when sending them to ES, ES will autogenerate a non-conflicting ID for them
                    if (!mapping.isAutoGeneratedId()) {
                        indexRequest.id(id);
                    }
                    processor.add(indexRequest);
                }
            }
            for (String id : deleteOperations.removeAll(type)) {
                processor.add(new DeleteRequest(typeIndex, id));
            }
            // Flush processor between index boundaries
            processor.flush();
        }
        // Remaining delete operations can be executed on their own
        for (Class<?> type : ImmutableSet.copyOf(deleteOperations.keySet())) {
            final DocumentMapping mapping = admin.mappings().getMapping(type);
            final String typeIndex = admin.getTypeIndex(mapping);
            mappingsToRefresh.add(mapping);
            for (String id : deleteOperations.removeAll(type)) {
                processor.add(new DeleteRequest(typeIndex, id));
            }
            // Flush processor between index boundaries
            processor.flush();
        }
        try {
            processor.awaitClose(5, TimeUnit.MINUTES);
        } catch (InterruptedException e) {
            throw new IndexException("Interrupted bulk processing part of the commit", e);
        }
    }
    // refresh the index if there were only updates
    admin.refresh(mappingsToRefresh);
}

Example 52 with DeleteRequest

use of org.elasticsearch.action.delete.DeleteRequest in project pancm_project by xuwujing.

the class EsHighLevelRestTest1 method delete.

/**
 * 删除
 *
 * @throws IOException
 */
private static void delete() throws IOException {
    String type = "_doc";
    String index = "test1";
    // 唯一编号
    String id = "1";
    DeleteRequest deleteRequest = new DeleteRequest();
    deleteRequest.id(id);
    deleteRequest.index(index);
    deleteRequest.type(type);
    // 设置超时时间
    deleteRequest.timeout(TimeValue.timeValueMinutes(2));
    // 设置刷新策略"wait_for"
    // 保持此请求打开，直到刷新使此请求的内容可以搜索为止。此刷新策略与高索引和搜索吞吐量兼容，但它会导致请求等待响应，直到发生刷新
    deleteRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
    // 同步删除
    DeleteResponse deleteResponse = client.delete(deleteRequest, RequestOptions.DEFAULT);
    /*
		 * 异步删除操作
		 */
    // 进行监听
    ActionListener<DeleteResponse> listener = new ActionListener<DeleteResponse>() {

        @Override
        public void onResponse(DeleteResponse deleteResponse) {
            System.out.println("响应:" + deleteResponse);
        }

        @Override
        public void onFailure(Exception e) {
            System.out.println("删除监听异常:" + e.getMessage());
        }
    };
    // 异步删除
    // client.deleteAsync(deleteRequest, RequestOptions.DEFAULT, listener);
    ReplicationResponse.ShardInfo shardInfo = deleteResponse.getShardInfo();
    // 如果处理成功碎片的数量少于总碎片的情况,说明还在处理或者处理发生异常
    if (shardInfo.getTotal() != shardInfo.getSuccessful()) {
        System.out.println("需要处理的碎片总量:" + shardInfo.getTotal());
        System.out.println("处理成功的碎片总量:" + shardInfo.getSuccessful());
    }
    if (shardInfo.getFailed() > 0) {
        for (ReplicationResponse.ShardInfo.Failure failure : shardInfo.getFailures()) {
            String reason = failure.reason();
        }
    }
    System.out.println("删除成功!");
}

Example 53 with DeleteRequest

use of org.elasticsearch.action.delete.DeleteRequest in project pancm_project by xuwujing.

the class EsHighLevelRestTest1 method bulk.

/**
 * 批量操作示例
 *
 * @throws InterruptedException
 */
private static void bulk() throws IOException, InterruptedException {
    // 类型
    String type = "_doc";
    String index = "student";
    BulkRequest request = new BulkRequest();
    Map<String, Object> map = new HashMap<>();
    map.put("uid", 123);
    map.put("age", 11);
    map.put("name", "虚无境");
    map.put("class", 9);
    map.put("grade", 400);
    map.put("createtm", "2019-11-04");
    map.put("updatetm", "2019-11-05 21:04:55.268");
    // 批量新增,存在会直接覆盖
    request.add(new IndexRequest(index, type, "1").source(XContentType.JSON, "field", "foo"));
    request.add(new IndexRequest(index, type, "2").source(XContentType.JSON, "field", "bar"));
    request.add(new IndexRequest(index, type, "3").source(XContentType.JSON, "field", "baz"));
    // 可以进行修改/删除/新增 操作
    // docAsUpsert 为true表示存在更新，不存在插入，为false表示不存在就是不做更新
    request.add(new UpdateRequest(index, type, "2").doc(XContentType.JSON, "field", "test").docAsUpsert(true));
    request.add(new DeleteRequest(index, type, "3"));
    request.add(new IndexRequest(index, type, "4").source(XContentType.JSON, "field", "baz"));
    BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
    ActionListener<BulkResponse> listener3 = new ActionListener<BulkResponse>() {

        @Override
        public void onResponse(BulkResponse response) {
            System.out.println("====" + response.buildFailureMessage());
        }

        @Override
        public void onFailure(Exception e) {
            System.out.println("====---" + e.getMessage());
        }
    };
    client.bulkAsync(request, RequestOptions.DEFAULT, listener3);
    // 可以快速检查一个或多个操作是否失败 true是有至少一个失败！
    if (bulkResponse.hasFailures()) {
        System.out.println("有一个操作失败!");
    }
    // 对处理结果进行遍历操作并根据不同的操作进行处理
    for (BulkItemResponse bulkItemResponse : bulkResponse) {
        DocWriteResponse itemResponse = bulkItemResponse.getResponse();
        // 操作失败的进行处理
        if (bulkItemResponse.isFailed()) {
            BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
        }
        if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.INDEX || bulkItemResponse.getOpType() == DocWriteRequest.OpType.CREATE) {
            IndexResponse indexResponse = (IndexResponse) itemResponse;
            System.out.println("新增失败!" + indexResponse.toString());
        } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.UPDATE) {
            UpdateResponse updateResponse = (UpdateResponse) itemResponse;
            System.out.println("更新失败!" + updateResponse.toString());
        } else if (bulkItemResponse.getOpType() == DocWriteRequest.OpType.DELETE) {
            DeleteResponse deleteResponse = (DeleteResponse) itemResponse;
            System.out.println("删除失败!" + deleteResponse.toString());
        }
    }
    System.out.println("批量执行成功！");
    /*
		 * 批量执行处理器相关示例代码
		 */
    // 批量处理器的监听器设置
    BulkProcessor.Listener listener = new BulkProcessor.Listener() {

        // 在执行BulkRequest的每次执行之前调用，这个方法允许知道将要在BulkRequest中执行的操作的数量
        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
            int numberOfActions = request.numberOfActions();
            logger.debug("Executing bulk [{}] with {} requests", executionId, numberOfActions);
        }

        // 在每次执行BulkRequest之后调用，这个方法允许知道BulkResponse是否包含错误
        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            if (response.hasFailures()) {
                logger.warn("Bulk [{}] executed with failures", executionId);
            } else {
                logger.debug("Bulk [{}] completed in {} milliseconds", executionId, response.getTook().getMillis());
            }
        }

        // 如果BulkRequest失败，则调用该方法，该方法允许知道失败
        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            logger.error("Failed to execute bulk", failure);
        }
    };
    BiConsumer<BulkRequest, ActionListener<BulkResponse>> bulkConsumer = (request2, bulkListener) -> client.bulkAsync(request, RequestOptions.DEFAULT, bulkListener);
    // 创建一个批量执行的处理器
    BulkProcessor bulkProcessor = BulkProcessor.builder(bulkConsumer, listener).build();
    BulkProcessor.Builder builder = BulkProcessor.builder(bulkConsumer, listener);
    // 根据当前添加的操作数量设置刷新新批量请求的时间(默认为1000，使用-1禁用它)
    builder.setBulkActions(500);
    // 根据当前添加的操作大小设置刷新新批量请求的时间(默认为5Mb，使用-1禁用)
    builder.setBulkSize(new ByteSizeValue(1L, ByteSizeUnit.MB));
    // 设置允许执行的并发请求数量(默认为1，使用0只允许执行单个请求)
    builder.setConcurrentRequests(0);
    // 设置刷新间隔如果间隔通过，则刷新任何挂起的BulkRequest(默认为未设置)
    builder.setFlushInterval(TimeValue.timeValueSeconds(10L));
    // 设置一个常量后退策略，该策略最初等待1秒并重试最多3次。
    builder.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3));
    IndexRequest one = new IndexRequest(index, type, "1").source(XContentType.JSON, "title", "In which order are my Elasticsearch queries executed?");
    IndexRequest two = new IndexRequest(index, type, "2").source(XContentType.JSON, "title", "Current status and upcoming changes in Elasticsearch");
    IndexRequest three = new IndexRequest(index, type, "3").source(XContentType.JSON, "title", "The Future of Federated Search in Elasticsearch");
    bulkProcessor.add(one);
    bulkProcessor.add(two);
    bulkProcessor.add(three);
    // 如果所有大容量请求都已完成，则该方法返回true;如果在所有大容量请求完成之前的等待时间已经过去，则返回false
    boolean terminated = bulkProcessor.awaitClose(30L, TimeUnit.SECONDS);
    System.out.println("请求的响应结果:" + terminated);
}

Example 54 with DeleteRequest

use of org.elasticsearch.action.delete.DeleteRequest in project pancm_project by xuwujing.

the class IpHandler method deleteByIds.

/**
 * @return boolean
 * @Author pancm
 * @Description //批量删除数据
 * 根据ID进行批量删除
 * @Date 2019/3/21
 * @Param []
 */
public static boolean deleteByIds(String index, String type, Set<String> ids) throws IOException {
    if (index == null || type == null || ids == null) {
        return true;
    }
    try {
        BulkRequest requestBulk = new BulkRequest();
        ids.forEach(id -> {
            DeleteRequest deleteRequest = new DeleteRequest(index, type, id);
            requestBulk.add(deleteRequest);
        });
        // 同步删除
        client.bulk(requestBulk, RequestOptions.DEFAULT);
    } finally {
        if (isAutoClose) {
            close();
        }
    }
    return false;
}

Example 55 with DeleteRequest

use of org.elasticsearch.action.delete.DeleteRequest in project elasticsearch-indexing-proxy by codelibs.

the class IndexingProxyService method dumpRequests.

public void dumpRequests(final int filePosition, ActionListener<String> listener) {
    final Path path = dataPath.resolve(String.format(dataFileFormat, filePosition) + IndexingProxyPlugin.DATA_EXTENTION);
    if (FileAccessUtils.existsFile(path)) {
        try (IndexingProxyStreamInput streamInput = AccessController.doPrivileged((PrivilegedAction<IndexingProxyStreamInput>) () -> {
            try {
                return new IndexingProxyStreamInput(Files.newInputStream(path), namedWriteableRegistry);
            } catch (final IOException e) {
                throw new ElasticsearchException("Failed to read " + path.toAbsolutePath(), e);
            }
        })) {
            final StringBuilder buf = new StringBuilder(10000);
            while (streamInput.available() > 0) {
                final short classType = streamInput.readShort();
                switch(classType) {
                    case RequestUtils.TYPE_DELETE:
                        DeleteRequest deleteRequest = RequestUtils.createDeleteRequest(client, streamInput, null).request();
                        buf.append(deleteRequest.toString());
                        break;
                    case RequestUtils.TYPE_DELETE_BY_QUERY:
                        DeleteByQueryRequest deleteByQueryRequest = RequestUtils.createDeleteByQueryRequest(client, streamInput, null).request();
                        buf.append(deleteByQueryRequest.toString());
                        buf.append(' ');
                        buf.append(deleteByQueryRequest.getSearchRequest().toString().replace("\n", ""));
                        break;
                    case RequestUtils.TYPE_INDEX:
                        IndexRequest indexRequest = RequestUtils.createIndexRequest(client, streamInput, null).request();
                        buf.append(indexRequest.toString());
                        break;
                    case RequestUtils.TYPE_UPDATE:
                        UpdateRequest updateRequest = RequestUtils.createUpdateRequest(client, streamInput, null).request();
                        buf.append("update {[").append(updateRequest.index()).append("][").append(updateRequest.type()).append("][").append(updateRequest.id()).append("] source[").append(updateRequest.toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS).string()).append("]}");
                        break;
                    case RequestUtils.TYPE_UPDATE_BY_QUERY:
                        UpdateByQueryRequest updateByQueryRequest = RequestUtils.createUpdateByQueryRequest(client, streamInput, null).request();
                        buf.append(updateByQueryRequest.toString());
                        buf.append(' ');
                        buf.append(updateByQueryRequest.getSearchRequest().toString().replace("\n", ""));
                        break;
                    case RequestUtils.TYPE_BULK:
                        BulkRequest bulkRequest = RequestUtils.createBulkRequest(client, streamInput, null).request();
                        buf.append("bulk [");
                        buf.append(bulkRequest.requests().stream().map(req -> {
                            if (req instanceof UpdateRequest) {
                                UpdateRequest upreq = (UpdateRequest) req;
                                try {
                                    return "update {[" + upreq.index() + "][" + upreq.type() + "][" + upreq.id() + "] source[" + upreq.toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS).string() + "]}";
                                } catch (IOException e) {
                                    return e.getMessage();
                                }
                            } else {
                                return req.toString();
                            }
                        }).collect(Collectors.joining(",")));
                        buf.append("]");
                        break;
                    default:
                        listener.onFailure(new ElasticsearchException("Unknown request type: " + classType));
                }
                buf.append('\n');
            }
            listener.onResponse(buf.toString());
        } catch (IOException e) {
            listener.onFailure(e);
        }
    } else {
        listener.onFailure(new ElasticsearchException("The data file does not exist: " + dataPath));
    }
}