use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project crate by crate.
the class BulkShardProcessorTest method testThatAddAfterFailureBlocksDueToRetry.
@Test
public void testThatAddAfterFailureBlocksDueToRetry() throws Exception {
ClusterService clusterService = mock(ClusterService.class);
OperationRouting operationRouting = mock(OperationRouting.class);
mockShard(operationRouting, 1);
mockShard(operationRouting, 2);
mockShard(operationRouting, 3);
when(clusterService.operationRouting()).thenReturn(operationRouting);
// listener will be executed 2 times, once for the successfully added row and once for the failure
final CountDownLatch listenerLatch = new CountDownLatch(2);
final AtomicReference<ActionListener<ShardResponse>> ref = new AtomicReference<>();
BulkRequestExecutor<ShardUpsertRequest> transportShardBulkAction = (request, listener) -> {
ref.set(listener);
listenerLatch.countDown();
};
BulkRetryCoordinator bulkRetryCoordinator = new BulkRetryCoordinator(threadPool);
BulkRetryCoordinatorPool coordinatorPool = mock(BulkRetryCoordinatorPool.class);
when(coordinatorPool.coordinator(any(ShardId.class))).thenReturn(bulkRetryCoordinator);
ShardUpsertRequest.Builder builder = new ShardUpsertRequest.Builder(TimeValue.timeValueMillis(10), false, false, null, new Reference[] { fooRef }, UUID.randomUUID());
final BulkShardProcessor<ShardUpsertRequest> bulkShardProcessor = new BulkShardProcessor<>(clusterService, mock(TransportBulkCreateIndicesAction.class), new IndexNameExpressionResolver(Settings.EMPTY), Settings.EMPTY, coordinatorPool, false, 1, builder, transportShardBulkAction, UUID.randomUUID());
bulkShardProcessor.add("foo", new ShardUpsertRequest.Item("1", null, new Object[] { "bar1" }, null), null);
final ActionListener<ShardResponse> listener = ref.get();
listener.onFailure(new EsRejectedExecutionException());
// wait, failure retry lock is done in decoupled thread
listenerLatch.await(10, TimeUnit.SECONDS);
final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(2);
try {
final AtomicBoolean hadBlocked = new AtomicBoolean(false);
final AtomicBoolean hasBlocked = new AtomicBoolean(true);
final CountDownLatch latch = new CountDownLatch(1);
scheduledExecutorService.execute(new Runnable() {
@Override
public void run() {
scheduledExecutorService.schedule(new Runnable() {
@Override
public void run() {
hadBlocked.set(hasBlocked.get());
latch.countDown();
}
}, 10, TimeUnit.MILLISECONDS);
bulkShardProcessor.add("foo", new ShardUpsertRequest.Item("2", null, new Object[] { "bar2" }, null), null);
hasBlocked.set(false);
}
});
latch.await();
assertTrue(hadBlocked.get());
} finally {
scheduledExecutorService.shutdownNow();
}
}
use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project crate by crate.
the class BulkShardProcessor method processFailure.
private void processFailure(Throwable e, final ShardId shardId, final Request request, Optional<BulkRetryCoordinator> retryCoordinator) {
trace("execute failure");
e = SQLExceptions.unwrap(e);
// index missing exception on a partition should never bubble, mark all items as failed instead
if (e instanceof IndexNotFoundException && PartitionName.isPartition(request.index())) {
indicesDeleted.add(request.index());
markItemsAsFailedAndReleaseRetryLock(request, retryCoordinator);
return;
}
final BulkRetryCoordinator coordinator;
if (retryCoordinator.isPresent()) {
coordinator = retryCoordinator.get();
} else {
try {
coordinator = bulkRetryCoordinatorPool.coordinator(shardId);
} catch (Throwable coordinatorException) {
setFailure(coordinatorException);
return;
}
}
if (e instanceof EsRejectedExecutionException) {
trace("rejected execution: [%s] - retrying", e.getMessage());
coordinator.retry(request, requestExecutor, new ActionListener<ShardResponse>() {
@Override
public void onResponse(ShardResponse response) {
processResponse(response);
}
@Override
public void onFailure(Throwable e) {
processFailure(e, shardId, request, Optional.of(coordinator));
}
});
} else {
if (retryCoordinator.isPresent()) {
// release failed retry
coordinator.releaseWriteLock();
}
for (IntCursor intCursor : request.itemIndices()) {
synchronized (responsesLock) {
responses.set(intCursor.value, false);
}
}
setFailure(e);
}
}
use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project elasticsearch by elastic.
the class FixedThreadPoolTests method testRejectedExecutionCounter.
public void testRejectedExecutionCounter() throws InterruptedException {
final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.FIXED);
// some of the fixed thread pool are bound by the number of
// cores so we can not exceed that
final int size = randomIntBetween(1, EsExecutors.numberOfProcessors(Settings.EMPTY));
final int queueSize = randomIntBetween(1, 16);
final long rejections = randomIntBetween(1, 16);
ThreadPool threadPool = null;
final Settings nodeSettings = Settings.builder().put("node.name", "testRejectedExecutionCounter").put("thread_pool." + threadPoolName + ".size", size).put("thread_pool." + threadPoolName + ".queue_size", queueSize).build();
try {
threadPool = new ThreadPool(nodeSettings);
// these tasks will consume the thread pool causing further
// submissions to queue
final CountDownLatch latch = new CountDownLatch(size);
final CountDownLatch block = new CountDownLatch(1);
for (int i = 0; i < size; i++) {
threadPool.executor(threadPoolName).execute(() -> {
try {
latch.countDown();
block.await();
} catch (InterruptedException e) {
fail(e.toString());
}
});
}
// wait for the submitted tasks to be consumed by the thread
// pool
latch.await();
// these tasks will fill the thread pool queue
for (int i = 0; i < queueSize; i++) {
threadPool.executor(threadPoolName).execute(() -> {
});
}
// these tasks will be rejected
long counter = 0;
for (int i = 0; i < rejections; i++) {
try {
threadPool.executor(threadPoolName).execute(() -> {
});
} catch (EsRejectedExecutionException e) {
counter++;
}
}
block.countDown();
assertThat(counter, equalTo(rejections));
assertThat(stats(threadPool, threadPoolName).getRejected(), equalTo(rejections));
} finally {
terminateThreadPoolIfNeeded(threadPool);
}
}
use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project elasticsearch by elastic.
the class RetryTests method blockExecutor.
/**
* Blocks the named executor by getting its only thread running a task blocked on a CyclicBarrier and fills the queue with a noop task.
* So requests to use this queue should get {@link EsRejectedExecutionException}s.
*/
private CyclicBarrier blockExecutor(String name) throws Exception {
ThreadPool threadPool = getInstanceFromNode(ThreadPool.class);
CyclicBarrier barrier = new CyclicBarrier(2);
logger.info("Blocking the [{}] executor", name);
threadPool.executor(name).execute(() -> {
try {
threadPool.executor(name).execute(() -> {
});
barrier.await();
logger.info("Blocked the [{}] executor", name);
barrier.await();
logger.info("Unblocking the [{}] executor", name);
} catch (Exception e) {
throw new RuntimeException(e);
}
});
barrier.await();
blockedExecutors.add(barrier);
return barrier;
}
use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project elasticsearch by elastic.
the class ESIntegTestCase method indexRandom.
/**
* Indexes the given {@link IndexRequestBuilder} instances randomly. It shuffles the given builders and either
* indexes them in a blocking or async fashion. This is very useful to catch problems that relate to internal document
* ids or index segment creations. Some features might have bug when a given document is the first or the last in a
* segment or if only one document is in a segment etc. This method prevents issues like this by randomizing the index
* layout.
*
* @param forceRefresh if <tt>true</tt> all involved indices are refreshed once the documents are indexed.
* @param dummyDocuments if <tt>true</tt> some empty dummy documents may be randomly inserted into the document list and deleted once
* all documents are indexed. This is useful to produce deleted documents on the server side.
* @param maybeFlush if <tt>true</tt> this method may randomly execute full flushes after index operations.
* @param builders the documents to index.
*/
public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean maybeFlush, List<IndexRequestBuilder> builders) throws InterruptedException, ExecutionException {
Random random = random();
Set<String> indicesSet = new HashSet<>();
for (IndexRequestBuilder builder : builders) {
indicesSet.add(builder.request().index());
}
Set<Tuple<String, String>> bogusIds = new HashSet<>();
if (random.nextBoolean() && !builders.isEmpty() && dummyDocuments) {
builders = new ArrayList<>(builders);
final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
// inject some bogus docs
final int numBogusDocs = scaledRandomIntBetween(1, builders.size() * 2);
final int unicodeLen = between(1, 10);
for (int i = 0; i < numBogusDocs; i++) {
String id = randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet());
String index = RandomPicks.randomFrom(random, indices);
bogusIds.add(new Tuple<>(index, id));
builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}", XContentType.JSON));
}
}
final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
Collections.shuffle(builders, random());
final CopyOnWriteArrayList<Tuple<IndexRequestBuilder, Exception>> errors = new CopyOnWriteArrayList<>();
List<CountDownLatch> inFlightAsyncOperations = new ArrayList<>();
// If you are indexing just a few documents then frequently do it one at a time. If many then frequently in bulk.
if (builders.size() < FREQUENT_BULK_THRESHOLD ? frequently() : builders.size() < ALWAYS_BULK_THRESHOLD ? rarely() : false) {
if (frequently()) {
logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), true, false);
for (IndexRequestBuilder indexRequestBuilder : builders) {
indexRequestBuilder.execute(new PayloadLatchedActionListener<IndexResponse, IndexRequestBuilder>(indexRequestBuilder, newLatch(inFlightAsyncOperations), errors));
postIndexAsyncActions(indices, inFlightAsyncOperations, maybeFlush);
}
} else {
logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), false, false);
for (IndexRequestBuilder indexRequestBuilder : builders) {
indexRequestBuilder.execute().actionGet();
postIndexAsyncActions(indices, inFlightAsyncOperations, maybeFlush);
}
}
} else {
List<List<IndexRequestBuilder>> partition = eagerPartition(builders, Math.min(MAX_BULK_INDEX_REQUEST_SIZE, Math.max(1, (int) (builders.size() * randomDouble()))));
logger.info("Index [{}] docs async: [{}] bulk: [{}] partitions [{}]", builders.size(), false, true, partition.size());
for (List<IndexRequestBuilder> segmented : partition) {
BulkRequestBuilder bulkBuilder = client().prepareBulk();
for (IndexRequestBuilder indexRequestBuilder : segmented) {
bulkBuilder.add(indexRequestBuilder);
}
BulkResponse actionGet = bulkBuilder.execute().actionGet();
assertThat(actionGet.hasFailures() ? actionGet.buildFailureMessage() : "", actionGet.hasFailures(), equalTo(false));
}
}
for (CountDownLatch operation : inFlightAsyncOperations) {
operation.await();
}
final List<Exception> actualErrors = new ArrayList<>();
for (Tuple<IndexRequestBuilder, Exception> tuple : errors) {
if (ExceptionsHelper.unwrapCause(tuple.v2()) instanceof EsRejectedExecutionException) {
// re-index if rejected
tuple.v1().execute().actionGet();
} else {
actualErrors.add(tuple.v2());
}
}
assertThat(actualErrors, emptyIterable());
if (!bogusIds.isEmpty()) {
// delete the bogus types again - it might trigger merges or at least holes in the segments and enforces deleted docs!
for (Tuple<String, String> doc : bogusIds) {
assertEquals("failed to delete a dummy doc [" + doc.v1() + "][" + doc.v2() + "]", DocWriteResponse.Result.DELETED, client().prepareDelete(doc.v1(), RANDOM_BOGUS_TYPE, doc.v2()).get().getResult());
}
}
if (forceRefresh) {
assertNoFailures(client().admin().indices().prepareRefresh(indices).setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute().get());
}
}
Aggregations