Examples with BulkRequest org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest used on opensource projects

Example 41 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project camel by apache.

the class ElasticsearchBulkTest method bulkIndexRequestBody.

@Test
public void bulkIndexRequestBody() throws Exception {
    String prefix = createPrefix();
    // given
    BulkRequest request = new BulkRequest();
    request.add(new IndexRequest(prefix + "foo", prefix + "bar", prefix + "baz").source("{\"" + prefix + "content\": \"" + prefix + "hello\"}"));
    // when
    @SuppressWarnings("unchecked") List<String> indexedDocumentIds = template.requestBody("direct:bulk_index", request, List.class);
    // then
    assertThat(indexedDocumentIds, notNullValue());
    assertThat(indexedDocumentIds.size(), equalTo(1));
    assertThat(indexedDocumentIds, hasItem(prefix + "baz"));
}

Example 42 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project camel by apache.

the class ElasticsearchProducer method process.

public void process(Exchange exchange) throws Exception {
    // 2. Index and type will be set by:
    // a. If the incoming body is already an action request
    // b. If the body is not an action request we will use headers if they
    // are set.
    // c. If the body is not an action request and the headers aren't set we
    // will use the configuration.
    // No error is thrown by the component in the event none of the above
    // conditions are met. The java es client
    // will throw.
    Message message = exchange.getIn();
    final String operation = resolveOperation(exchange);
    // Set the index/type headers on the exchange if necessary. This is used
    // for type conversion.
    boolean configIndexName = false;
    String indexName = message.getHeader(ElasticsearchConstants.PARAM_INDEX_NAME, String.class);
    if (indexName == null) {
        message.setHeader(ElasticsearchConstants.PARAM_INDEX_NAME, getEndpoint().getConfig().getIndexName());
        configIndexName = true;
    }
    boolean configIndexType = false;
    String indexType = message.getHeader(ElasticsearchConstants.PARAM_INDEX_TYPE, String.class);
    if (indexType == null) {
        message.setHeader(ElasticsearchConstants.PARAM_INDEX_TYPE, getEndpoint().getConfig().getIndexType());
        configIndexType = true;
    }
    boolean configConsistencyLevel = false;
    String consistencyLevel = message.getHeader(ElasticsearchConstants.PARAM_CONSISTENCY_LEVEL, String.class);
    if (consistencyLevel == null) {
        message.setHeader(ElasticsearchConstants.PARAM_CONSISTENCY_LEVEL, getEndpoint().getConfig().getConsistencyLevel());
        configConsistencyLevel = true;
    }
    Client client = getEndpoint().getClient();
    if (ElasticsearchConstants.OPERATION_INDEX.equals(operation)) {
        IndexRequest indexRequest = message.getBody(IndexRequest.class);
        message.setBody(client.index(indexRequest).actionGet().getId());
    } else if (ElasticsearchConstants.OPERATION_UPDATE.equals(operation)) {
        UpdateRequest updateRequest = message.getBody(UpdateRequest.class);
        message.setBody(client.update(updateRequest).actionGet().getId());
    } else if (ElasticsearchConstants.OPERATION_GET_BY_ID.equals(operation)) {
        GetRequest getRequest = message.getBody(GetRequest.class);
        message.setBody(client.get(getRequest));
    } else if (ElasticsearchConstants.OPERATION_MULTIGET.equals(operation)) {
        MultiGetRequest multiGetRequest = message.getBody(MultiGetRequest.class);
        message.setBody(client.multiGet(multiGetRequest));
    } else if (ElasticsearchConstants.OPERATION_BULK.equals(operation)) {
        BulkRequest bulkRequest = message.getBody(BulkRequest.class);
        message.setBody(client.bulk(bulkRequest).actionGet());
    } else if (ElasticsearchConstants.OPERATION_BULK_INDEX.equals(operation)) {
        BulkRequest bulkRequest = message.getBody(BulkRequest.class);
        List<String> indexedIds = new ArrayList<String>();
        for (BulkItemResponse response : client.bulk(bulkRequest).actionGet().getItems()) {
            indexedIds.add(response.getId());
        }
        message.setBody(indexedIds);
    } else if (ElasticsearchConstants.OPERATION_DELETE.equals(operation)) {
        DeleteRequest deleteRequest = message.getBody(DeleteRequest.class);
        message.setBody(client.delete(deleteRequest).actionGet());
    } else if (ElasticsearchConstants.OPERATION_EXISTS.equals(operation)) {
        ExistsRequest existsRequest = message.getBody(ExistsRequest.class);
        message.setBody(client.admin().indices().prepareExists(existsRequest.indices()).get().isExists());
    } else if (ElasticsearchConstants.OPERATION_SEARCH.equals(operation)) {
        SearchRequest searchRequest = message.getBody(SearchRequest.class);
        message.setBody(client.search(searchRequest).actionGet());
    } else if (ElasticsearchConstants.OPERATION_MULTISEARCH.equals(operation)) {
        MultiSearchRequest multiSearchRequest = message.getBody(MultiSearchRequest.class);
        message.setBody(client.multiSearch(multiSearchRequest));
    } else if (ElasticsearchConstants.OPERATION_DELETE_INDEX.equals(operation)) {
        DeleteIndexRequest deleteIndexRequest = message.getBody(DeleteIndexRequest.class);
        message.setBody(client.admin().indices().delete(deleteIndexRequest).actionGet());
    } else {
        throw new IllegalArgumentException(ElasticsearchConstants.PARAM_OPERATION + " value '" + operation + "' is not supported");
    }
    // subsequent endpoint index/type with the first endpoint index/type.
    if (configIndexName) {
        message.removeHeader(ElasticsearchConstants.PARAM_INDEX_NAME);
    }
    if (configIndexType) {
        message.removeHeader(ElasticsearchConstants.PARAM_INDEX_TYPE);
    }
    if (configConsistencyLevel) {
        message.removeHeader(ElasticsearchConstants.PARAM_CONSISTENCY_LEVEL);
    }
}

Example 43 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project elasticsearch-indexing-proxy by codelibs.

the class IndexingProxyService method dumpRequests.

public void dumpRequests(final int filePosition, ActionListener<String> listener) {
    final Path path = dataPath.resolve(String.format(dataFileFormat, filePosition) + IndexingProxyPlugin.DATA_EXTENTION);
    if (FileAccessUtils.existsFile(path)) {
        try (IndexingProxyStreamInput streamInput = AccessController.doPrivileged((PrivilegedAction<IndexingProxyStreamInput>) () -> {
            try {
                return new IndexingProxyStreamInput(Files.newInputStream(path), namedWriteableRegistry);
            } catch (final IOException e) {
                throw new ElasticsearchException("Failed to read " + path.toAbsolutePath(), e);
            }
        })) {
            final StringBuilder buf = new StringBuilder(10000);
            while (streamInput.available() > 0) {
                final short classType = streamInput.readShort();
                switch(classType) {
                    case RequestUtils.TYPE_DELETE:
                        DeleteRequest deleteRequest = RequestUtils.createDeleteRequest(client, streamInput, null).request();
                        buf.append(deleteRequest.toString());
                        break;
                    case RequestUtils.TYPE_DELETE_BY_QUERY:
                        DeleteByQueryRequest deleteByQueryRequest = RequestUtils.createDeleteByQueryRequest(client, streamInput, null).request();
                        buf.append(deleteByQueryRequest.toString());
                        buf.append(' ');
                        buf.append(deleteByQueryRequest.getSearchRequest().toString().replace("\n", ""));
                        break;
                    case RequestUtils.TYPE_INDEX:
                        IndexRequest indexRequest = RequestUtils.createIndexRequest(client, streamInput, null).request();
                        buf.append(indexRequest.toString());
                        break;
                    case RequestUtils.TYPE_UPDATE:
                        UpdateRequest updateRequest = RequestUtils.createUpdateRequest(client, streamInput, null).request();
                        buf.append("update {[").append(updateRequest.index()).append("][").append(updateRequest.type()).append("][").append(updateRequest.id()).append("] source[").append(updateRequest.toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS).string()).append("]}");
                        break;
                    case RequestUtils.TYPE_UPDATE_BY_QUERY:
                        UpdateByQueryRequest updateByQueryRequest = RequestUtils.createUpdateByQueryRequest(client, streamInput, null).request();
                        buf.append(updateByQueryRequest.toString());
                        buf.append(' ');
                        buf.append(updateByQueryRequest.getSearchRequest().toString().replace("\n", ""));
                        break;
                    case RequestUtils.TYPE_BULK:
                        BulkRequest bulkRequest = RequestUtils.createBulkRequest(client, streamInput, null).request();
                        buf.append("bulk [");
                        buf.append(bulkRequest.requests().stream().map(req -> {
                            if (req instanceof UpdateRequest) {
                                UpdateRequest upreq = (UpdateRequest) req;
                                try {
                                    return "update {[" + upreq.index() + "][" + upreq.type() + "][" + upreq.id() + "] source[" + upreq.toXContent(JsonXContent.contentBuilder(), ToXContent.EMPTY_PARAMS).string() + "]}";
                                } catch (IOException e) {
                                    return e.getMessage();
                                }
                            } else {
                                return req.toString();
                            }
                        }).collect(Collectors.joining(",")));
                        buf.append("]");
                        break;
                    default:
                        listener.onFailure(new ElasticsearchException("Unknown request type: " + classType));
                }
                buf.append('\n');
            }
            listener.onResponse(buf.toString());
        } catch (IOException e) {
            listener.onFailure(e);
        }
    } else {
        listener.onFailure(new ElasticsearchException("The data file does not exist: " + dataPath));
    }
}

Example 44 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project elasticsearch-river-couchdb by elastic.

the class CouchdbRiver method start.

@Override
public void start() {
    logger.info("starting couchdb stream: host [{}], port [{}], filter [{}], db [{}], indexing to [{}]/[{}]", couchHost, couchPort, couchFilter, couchDb, indexName, typeName);
    // Creating bulk processor
    this.bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
            logger.debug("Going to execute new bulk composed of {} actions", request.numberOfActions());
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            logger.debug("Executed bulk composed of {} actions", request.numberOfActions());
            if (response.hasFailures()) {
                logger.warn("There was failures while executing bulk", response.buildFailureMessage());
                if (logger.isDebugEnabled()) {
                    for (BulkItemResponse item : response.getItems()) {
                        if (item.isFailed()) {
                            logger.debug("Error for {}/{}/{} for {} operation: {}", item.getIndex(), item.getType(), item.getId(), item.getOpType(), item.getFailureMessage());
                        }
                    }
                }
            }
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            logger.warn("Error executing bulk", failure);
        }
    }).setBulkActions(bulkSize).setConcurrentRequests(maxConcurrentBulk).setFlushInterval(bulkFlushInterval).build();
    slurperThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "couchdb_river_slurper").newThread(new Slurper());
    indexerThread = EsExecutors.daemonThreadFactory(settings.globalSettings(), "couchdb_river_indexer").newThread(new Indexer());
    indexerThread.start();
    slurperThread.start();
}

Example 45 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project fess-crawler by codelibs.

the class EsClient method deleteByQuery.

public int deleteByQuery(final String index, final String type, final QueryBuilder queryBuilder) {
    int count = 0;
    String scrollId = null;
    while (true) {
        final SearchResponse scrollResponse;
        if (scrollId == null) {
            scrollResponse = get(c -> c.prepareSearch(index).setTypes(type).setScroll(scrollForDelete).setSize(sizeForDelete).setQuery(queryBuilder).execute());
        } else {
            final String sid = scrollId;
            scrollResponse = get(c -> c.prepareSearchScroll(sid).setScroll(scrollForDelete).execute());
        }
        final SearchHit[] hits = scrollResponse.getHits().getHits();
        if (hits.length == 0) {
            break;
        }
        scrollId = scrollResponse.getScrollId();
        count += hits.length;
        final BulkResponse bulkResponse = get(c -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                bulkRequest.add(client.prepareDelete(hit.getIndex(), hit.getType(), hit.getId()));
            }
            return bulkRequest.execute();
        });
        if (bulkResponse.hasFailures()) {
            throw new EsAccessException(bulkResponse.buildFailureMessage());
        }
    }
    return count;
}