use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project elasticsearch by elastic.
the class CircuitBreakerServiceIT method testLimitsRequestSize.
public void testLimitsRequestSize() throws Exception {
ByteSizeValue inFlightRequestsLimit = new ByteSizeValue(8, ByteSizeUnit.KB);
if (noopBreakerUsed()) {
logger.info("--> noop breakers used, skipping test");
return;
}
internalCluster().ensureAtLeastNumDataNodes(2);
NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
List<NodeStats> dataNodeStats = new ArrayList<>();
for (NodeStats stat : nodeStats.getNodes()) {
if (stat.getNode().isDataNode()) {
dataNodeStats.add(stat);
}
}
assertThat(dataNodeStats.size(), greaterThanOrEqualTo(2));
Collections.shuffle(dataNodeStats, random());
// send bulk request from source node to target node later. The sole shard is bound to the target node.
NodeStats targetNode = dataNodeStats.get(0);
NodeStats sourceNode = dataNodeStats.get(1);
assertAcked(prepareCreate("index").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put("index.routing.allocation.include._name", targetNode.getNode().getName()).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)));
Client client = client(sourceNode.getNode().getName());
// we use the limit size as a (very) rough indication on how many requests we should sent to hit the limit
int numRequests = inFlightRequestsLimit.bytesAsInt();
BulkRequest bulkRequest = new BulkRequest();
for (int i = 0; i < numRequests; i++) {
IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i));
indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "num", i);
bulkRequest.add(indexRequest);
}
Settings limitSettings = Settings.builder().put(IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), inFlightRequestsLimit).build();
assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(limitSettings));
// can either fail directly with an exception or the response contains exceptions (depending on client)
try {
BulkResponse response = client.bulk(bulkRequest).actionGet();
if (!response.hasFailures()) {
fail("Should have thrown CircuitBreakingException");
} else {
// each item must have failed with CircuitBreakingException
for (BulkItemResponse bulkItemResponse : response) {
Throwable cause = ExceptionsHelper.unwrapCause(bulkItemResponse.getFailure().getCause());
assertThat(cause, instanceOf(CircuitBreakingException.class));
assertEquals(((CircuitBreakingException) cause).getByteLimit(), inFlightRequestsLimit.getBytes());
}
}
} catch (CircuitBreakingException ex) {
assertEquals(ex.getByteLimit(), inFlightRequestsLimit.getBytes());
}
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project titan by thinkaurelius.
the class ElasticSearchIndex method mutate.
@Override
public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever informations, BaseTransaction tx) throws BackendException {
BulkRequestBuilder brb = client.prepareBulk();
int bulkrequests = 0;
try {
for (Map.Entry<String, Map<String, IndexMutation>> stores : mutations.entrySet()) {
String storename = stores.getKey();
for (Map.Entry<String, IndexMutation> entry : stores.getValue().entrySet()) {
String docid = entry.getKey();
IndexMutation mutation = entry.getValue();
assert mutation.isConsolidated();
Preconditions.checkArgument(!(mutation.isNew() && mutation.isDeleted()));
Preconditions.checkArgument(!mutation.isNew() || !mutation.hasDeletions());
Preconditions.checkArgument(!mutation.isDeleted() || !mutation.hasAdditions());
//Deletions first
if (mutation.hasDeletions()) {
if (mutation.isDeleted()) {
log.trace("Deleting entire document {}", docid);
brb.add(new DeleteRequest(indexName, storename, docid));
} else {
String script = getDeletionScript(informations, storename, mutation);
brb.add(client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE));
log.trace("Adding script {}", script);
}
bulkrequests++;
}
if (mutation.hasAdditions()) {
int ttl = mutation.determineTTL();
if (mutation.isNew()) {
//Index
log.trace("Adding entire document {}", docid);
brb.add(new IndexRequest(indexName, storename, docid).source(getNewDocument(mutation.getAdditions(), informations.get(storename), ttl)));
} else {
Preconditions.checkArgument(ttl == 0, "Elasticsearch only supports TTL on new documents [%s]", docid);
boolean needUpsert = !mutation.hasDeletions();
String script = getAdditionScript(informations, storename, mutation);
UpdateRequestBuilder update = client.prepareUpdate(indexName, storename, docid).setScript(script, ScriptService.ScriptType.INLINE);
if (needUpsert) {
XContentBuilder doc = getNewDocument(mutation.getAdditions(), informations.get(storename), ttl);
update.setUpsert(doc);
}
brb.add(update);
log.trace("Adding script {}", script);
}
bulkrequests++;
}
}
}
if (bulkrequests > 0) {
BulkResponse bulkItemResponses = brb.execute().actionGet();
if (bulkItemResponses.hasFailures()) {
boolean actualFailure = false;
for (BulkItemResponse response : bulkItemResponses.getItems()) {
//The document may have been deleted, which is OK
if (response.isFailed() && response.getFailure().getStatus() != RestStatus.NOT_FOUND) {
log.error("Failed to execute ES query {}", response.getFailureMessage());
actualFailure = true;
}
}
if (actualFailure) {
throw new Exception(bulkItemResponses.buildFailureMessage());
}
}
}
} catch (Exception e) {
log.error("Failed to execute ES query {}", brb.request().timeout(), e);
throw convert(e);
}
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project metacat by Netflix.
the class ElasticSearchUtilImpl method updateDocs.
/**
* Updates the documents with partial updates with the given fields.
*
* @param type index type
* @param ids list of entity ids
* @param metacatRequestContext context containing the user name
* @param node json that represents the document source
*/
private void updateDocs(final String type, final List<String> ids, final MetacatRequestContext metacatRequestContext, final ObjectNode node) {
try {
RETRY_ES_PUBLISH.call(() -> {
final BulkRequestBuilder bulkRequest = client.prepareBulk();
ids.forEach(id -> {
node.put(ElasticSearchDoc.Field.USER, metacatRequestContext.getUserName());
bulkRequest.add(client.prepareUpdate(esIndex, type, id).setRetryOnConflict(NO_OF_CONFLICT_RETRIES).setDoc(metacatJson.toJsonAsBytes(node)));
});
final BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
for (BulkItemResponse item : bulkResponse.getItems()) {
if (item.isFailed()) {
log.error("Failed updating metadata of type {} with id {}. Message: {}", type, item.getId(), item.getFailureMessage());
registry.counter(registry.createId(Metrics.CounterElasticSearchUpdate.name()).withTags(Metrics.statusFailureMap)).increment();
log("ElasticSearchUtil.updateDocs.item", type, item.getId(), null, item.getFailureMessage(), null, true);
}
}
}
return null;
});
} catch (Exception e) {
log.error(String.format("Failed updating metadata of type %s with ids %s", type, ids), e);
registry.counter(registry.createId(Metrics.CounterElasticSearchBulkUpdate.name()).withTags(Metrics.statusFailureMap)).increment();
log("ElasticSearchUtil.updatDocs", type, ids.toString(), null, e.getMessage(), e, true);
}
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project camel by apache.
the class ElasticsearchProducer method process.
public void process(Exchange exchange) throws Exception {
// 2. Index and type will be set by:
// a. If the incoming body is already an action request
// b. If the body is not an action request we will use headers if they
// are set.
// c. If the body is not an action request and the headers aren't set we
// will use the configuration.
// No error is thrown by the component in the event none of the above
// conditions are met. The java es client
// will throw.
Message message = exchange.getIn();
final ElasticsearchOperation operation = resolveOperation(exchange);
// Set the index/type headers on the exchange if necessary. This is used
// for type conversion.
boolean configIndexName = false;
String indexName = message.getHeader(ElasticsearchConstants.PARAM_INDEX_NAME, String.class);
if (indexName == null) {
message.setHeader(ElasticsearchConstants.PARAM_INDEX_NAME, configuration.getIndexName());
configIndexName = true;
}
boolean configIndexType = false;
String indexType = message.getHeader(ElasticsearchConstants.PARAM_INDEX_TYPE, String.class);
if (indexType == null) {
message.setHeader(ElasticsearchConstants.PARAM_INDEX_TYPE, configuration.getIndexType());
configIndexType = true;
}
boolean configWaitForActiveShards = false;
Integer waitForActiveShards = message.getHeader(ElasticsearchConstants.PARAM_WAIT_FOR_ACTIVE_SHARDS, Integer.class);
if (waitForActiveShards == null) {
message.setHeader(ElasticsearchConstants.PARAM_WAIT_FOR_ACTIVE_SHARDS, configuration.getWaitForActiveShards());
configWaitForActiveShards = true;
}
if (operation == ElasticsearchOperation.INDEX) {
IndexRequest indexRequest = message.getBody(IndexRequest.class);
message.setBody(client.index(indexRequest).actionGet().getId());
} else if (operation == ElasticsearchOperation.UPDATE) {
UpdateRequest updateRequest = message.getBody(UpdateRequest.class);
message.setBody(client.update(updateRequest).actionGet().getId());
} else if (operation == ElasticsearchOperation.GET_BY_ID) {
GetRequest getRequest = message.getBody(GetRequest.class);
message.setBody(client.get(getRequest));
} else if (operation == ElasticsearchOperation.MULTIGET) {
MultiGetRequest multiGetRequest = message.getBody(MultiGetRequest.class);
message.setBody(client.multiGet(multiGetRequest));
} else if (operation == ElasticsearchOperation.BULK) {
BulkRequest bulkRequest = message.getBody(BulkRequest.class);
message.setBody(client.bulk(bulkRequest).actionGet());
} else if (operation == ElasticsearchOperation.BULK_INDEX) {
BulkRequest bulkRequest = message.getBody(BulkRequest.class);
List<String> indexedIds = new ArrayList<String>();
for (BulkItemResponse response : client.bulk(bulkRequest).actionGet().getItems()) {
indexedIds.add(response.getId());
}
message.setBody(indexedIds);
} else if (operation == ElasticsearchOperation.DELETE) {
DeleteRequest deleteRequest = message.getBody(DeleteRequest.class);
message.setBody(client.delete(deleteRequest).actionGet());
} else if (operation == ElasticsearchOperation.EXISTS) {
// ExistsRequest API is deprecated, using SearchRequest instead with size=0 and terminate_after=1
SearchRequest searchRequest = new SearchRequest(exchange.getIn().getHeader(ElasticsearchConstants.PARAM_INDEX_NAME, String.class));
try {
client.prepareSearch(searchRequest.indices()).setSize(0).setTerminateAfter(1).get();
message.setBody(true);
} catch (IndexNotFoundException e) {
message.setBody(false);
}
} else if (operation == ElasticsearchOperation.SEARCH) {
SearchRequest searchRequest = message.getBody(SearchRequest.class);
message.setBody(client.search(searchRequest).actionGet());
} else if (operation == ElasticsearchOperation.MULTISEARCH) {
MultiSearchRequest multiSearchRequest = message.getBody(MultiSearchRequest.class);
message.setBody(client.multiSearch(multiSearchRequest));
} else if (operation == ElasticsearchOperation.DELETE_INDEX) {
DeleteIndexRequest deleteIndexRequest = message.getBody(DeleteIndexRequest.class);
message.setBody(client.admin().indices().delete(deleteIndexRequest).actionGet());
} else {
throw new IllegalArgumentException(ElasticsearchConstants.PARAM_OPERATION + " value '" + operation + "' is not supported");
}
// subsequent endpoint index/type with the first endpoint index/type.
if (configIndexName) {
message.removeHeader(ElasticsearchConstants.PARAM_INDEX_NAME);
}
if (configIndexType) {
message.removeHeader(ElasticsearchConstants.PARAM_INDEX_TYPE);
}
if (configWaitForActiveShards) {
message.removeHeader(ElasticsearchConstants.PARAM_WAIT_FOR_ACTIVE_SHARDS);
}
}
use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkItemResponse in project camel by apache.
the class ElasticsearchProducer method process.
public void process(Exchange exchange) throws Exception {
// 2. Index and type will be set by:
// a. If the incoming body is already an action request
// b. If the body is not an action request we will use headers if they
// are set.
// c. If the body is not an action request and the headers aren't set we
// will use the configuration.
// No error is thrown by the component in the event none of the above
// conditions are met. The java es client
// will throw.
Message message = exchange.getIn();
final String operation = resolveOperation(exchange);
// Set the index/type headers on the exchange if necessary. This is used
// for type conversion.
boolean configIndexName = false;
String indexName = message.getHeader(ElasticsearchConstants.PARAM_INDEX_NAME, String.class);
if (indexName == null) {
message.setHeader(ElasticsearchConstants.PARAM_INDEX_NAME, getEndpoint().getConfig().getIndexName());
configIndexName = true;
}
boolean configIndexType = false;
String indexType = message.getHeader(ElasticsearchConstants.PARAM_INDEX_TYPE, String.class);
if (indexType == null) {
message.setHeader(ElasticsearchConstants.PARAM_INDEX_TYPE, getEndpoint().getConfig().getIndexType());
configIndexType = true;
}
boolean configConsistencyLevel = false;
String consistencyLevel = message.getHeader(ElasticsearchConstants.PARAM_CONSISTENCY_LEVEL, String.class);
if (consistencyLevel == null) {
message.setHeader(ElasticsearchConstants.PARAM_CONSISTENCY_LEVEL, getEndpoint().getConfig().getConsistencyLevel());
configConsistencyLevel = true;
}
Client client = getEndpoint().getClient();
if (ElasticsearchConstants.OPERATION_INDEX.equals(operation)) {
IndexRequest indexRequest = message.getBody(IndexRequest.class);
message.setBody(client.index(indexRequest).actionGet().getId());
} else if (ElasticsearchConstants.OPERATION_UPDATE.equals(operation)) {
UpdateRequest updateRequest = message.getBody(UpdateRequest.class);
message.setBody(client.update(updateRequest).actionGet().getId());
} else if (ElasticsearchConstants.OPERATION_GET_BY_ID.equals(operation)) {
GetRequest getRequest = message.getBody(GetRequest.class);
message.setBody(client.get(getRequest));
} else if (ElasticsearchConstants.OPERATION_MULTIGET.equals(operation)) {
MultiGetRequest multiGetRequest = message.getBody(MultiGetRequest.class);
message.setBody(client.multiGet(multiGetRequest));
} else if (ElasticsearchConstants.OPERATION_BULK.equals(operation)) {
BulkRequest bulkRequest = message.getBody(BulkRequest.class);
message.setBody(client.bulk(bulkRequest).actionGet());
} else if (ElasticsearchConstants.OPERATION_BULK_INDEX.equals(operation)) {
BulkRequest bulkRequest = message.getBody(BulkRequest.class);
List<String> indexedIds = new ArrayList<String>();
for (BulkItemResponse response : client.bulk(bulkRequest).actionGet().getItems()) {
indexedIds.add(response.getId());
}
message.setBody(indexedIds);
} else if (ElasticsearchConstants.OPERATION_DELETE.equals(operation)) {
DeleteRequest deleteRequest = message.getBody(DeleteRequest.class);
message.setBody(client.delete(deleteRequest).actionGet());
} else if (ElasticsearchConstants.OPERATION_EXISTS.equals(operation)) {
ExistsRequest existsRequest = message.getBody(ExistsRequest.class);
message.setBody(client.admin().indices().prepareExists(existsRequest.indices()).get().isExists());
} else if (ElasticsearchConstants.OPERATION_SEARCH.equals(operation)) {
SearchRequest searchRequest = message.getBody(SearchRequest.class);
message.setBody(client.search(searchRequest).actionGet());
} else if (ElasticsearchConstants.OPERATION_MULTISEARCH.equals(operation)) {
MultiSearchRequest multiSearchRequest = message.getBody(MultiSearchRequest.class);
message.setBody(client.multiSearch(multiSearchRequest));
} else if (ElasticsearchConstants.OPERATION_DELETE_INDEX.equals(operation)) {
DeleteIndexRequest deleteIndexRequest = message.getBody(DeleteIndexRequest.class);
message.setBody(client.admin().indices().delete(deleteIndexRequest).actionGet());
} else {
throw new IllegalArgumentException(ElasticsearchConstants.PARAM_OPERATION + " value '" + operation + "' is not supported");
}
// subsequent endpoint index/type with the first endpoint index/type.
if (configIndexName) {
message.removeHeader(ElasticsearchConstants.PARAM_INDEX_NAME);
}
if (configIndexType) {
message.removeHeader(ElasticsearchConstants.PARAM_INDEX_TYPE);
}
if (configConsistencyLevel) {
message.removeHeader(ElasticsearchConstants.PARAM_CONSISTENCY_LEVEL);
}
}
Aggregations