Examples with EsRejectedExecutionException org.elasticsearch.common.util.concurrent.EsRejectedExecutionException used on opensource projects

Example 1 with EsRejectedExecutionException

use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project crate by crate.

the class BulkShardProcessorTest method testThatAddAfterFailureBlocksDueToRetry.

@Test
public void testThatAddAfterFailureBlocksDueToRetry() throws Exception {
    ClusterService clusterService = mock(ClusterService.class);
    OperationRouting operationRouting = mock(OperationRouting.class);
    mockShard(operationRouting, 1);
    mockShard(operationRouting, 2);
    mockShard(operationRouting, 3);
    when(clusterService.operationRouting()).thenReturn(operationRouting);
    // listener will be executed 2 times, once for the successfully added row and once for the failure
    final CountDownLatch listenerLatch = new CountDownLatch(2);
    final AtomicReference<ActionListener<ShardResponse>> ref = new AtomicReference<>();
    BulkRequestExecutor<ShardUpsertRequest> transportShardBulkAction = (request, listener) -> {
        ref.set(listener);
        listenerLatch.countDown();
    };
    BulkRetryCoordinator bulkRetryCoordinator = new BulkRetryCoordinator(threadPool);
    BulkRetryCoordinatorPool coordinatorPool = mock(BulkRetryCoordinatorPool.class);
    when(coordinatorPool.coordinator(any(ShardId.class))).thenReturn(bulkRetryCoordinator);
    ShardUpsertRequest.Builder builder = new ShardUpsertRequest.Builder(TimeValue.timeValueMillis(10), false, false, null, new Reference[] { fooRef }, UUID.randomUUID());
    final BulkShardProcessor<ShardUpsertRequest> bulkShardProcessor = new BulkShardProcessor<>(clusterService, mock(TransportBulkCreateIndicesAction.class), new IndexNameExpressionResolver(Settings.EMPTY), Settings.EMPTY, coordinatorPool, false, 1, builder, transportShardBulkAction, UUID.randomUUID());
    bulkShardProcessor.add("foo", new ShardUpsertRequest.Item("1", null, new Object[] { "bar1" }, null), null);
    final ActionListener<ShardResponse> listener = ref.get();
    listener.onFailure(new EsRejectedExecutionException());
    // wait, failure retry lock is done in decoupled thread
    listenerLatch.await(10, TimeUnit.SECONDS);
    final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(2);
    try {
        final AtomicBoolean hadBlocked = new AtomicBoolean(false);
        final AtomicBoolean hasBlocked = new AtomicBoolean(true);
        final CountDownLatch latch = new CountDownLatch(1);
        scheduledExecutorService.execute(new Runnable() {

            @Override
            public void run() {
                scheduledExecutorService.schedule(new Runnable() {

                    @Override
                    public void run() {
                        hadBlocked.set(hasBlocked.get());
                        latch.countDown();
                    }
                }, 10, TimeUnit.MILLISECONDS);
                bulkShardProcessor.add("foo", new ShardUpsertRequest.Item("2", null, new Object[] { "bar2" }, null), null);
                hasBlocked.set(false);
            }
        });
        latch.await();
        assertTrue(hadBlocked.get());
    } finally {
        scheduledExecutorService.shutdownNow();
    }
}

Example 2 with EsRejectedExecutionException

use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project crate by crate.

the class BulkShardProcessor method processFailure.

private void processFailure(Throwable e, final ShardId shardId, final Request request, Optional<BulkRetryCoordinator> retryCoordinator) {
    trace("execute failure");
    e = SQLExceptions.unwrap(e);
    // index missing exception on a partition should never bubble, mark all items as failed instead
    if (e instanceof IndexNotFoundException && PartitionName.isPartition(request.index())) {
        indicesDeleted.add(request.index());
        markItemsAsFailedAndReleaseRetryLock(request, retryCoordinator);
        return;
    }
    final BulkRetryCoordinator coordinator;
    if (retryCoordinator.isPresent()) {
        coordinator = retryCoordinator.get();
    } else {
        try {
            coordinator = bulkRetryCoordinatorPool.coordinator(shardId);
        } catch (Throwable coordinatorException) {
            setFailure(coordinatorException);
            return;
        }
    }
    if (e instanceof EsRejectedExecutionException) {
        trace("rejected execution: [%s] - retrying", e.getMessage());
        coordinator.retry(request, requestExecutor, new ActionListener<ShardResponse>() {

            @Override
            public void onResponse(ShardResponse response) {
                processResponse(response);
            }

            @Override
            public void onFailure(Throwable e) {
                processFailure(e, shardId, request, Optional.of(coordinator));
            }
        });
    } else {
        if (retryCoordinator.isPresent()) {
            // release failed retry
            coordinator.releaseWriteLock();
        }
        for (IntCursor intCursor : request.itemIndices()) {
            synchronized (responsesLock) {
                responses.set(intCursor.value, false);
            }
        }
        setFailure(e);
    }
}

Example 3 with EsRejectedExecutionException

use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project elasticsearch by elastic.

the class FixedThreadPoolTests method testRejectedExecutionCounter.

public void testRejectedExecutionCounter() throws InterruptedException {
    final String threadPoolName = randomThreadPool(ThreadPool.ThreadPoolType.FIXED);
    // some of the fixed thread pool are bound by the number of
    // cores so we can not exceed that
    final int size = randomIntBetween(1, EsExecutors.numberOfProcessors(Settings.EMPTY));
    final int queueSize = randomIntBetween(1, 16);
    final long rejections = randomIntBetween(1, 16);
    ThreadPool threadPool = null;
    final Settings nodeSettings = Settings.builder().put("node.name", "testRejectedExecutionCounter").put("thread_pool." + threadPoolName + ".size", size).put("thread_pool." + threadPoolName + ".queue_size", queueSize).build();
    try {
        threadPool = new ThreadPool(nodeSettings);
        // these tasks will consume the thread pool causing further
        // submissions to queue
        final CountDownLatch latch = new CountDownLatch(size);
        final CountDownLatch block = new CountDownLatch(1);
        for (int i = 0; i < size; i++) {
            threadPool.executor(threadPoolName).execute(() -> {
                try {
                    latch.countDown();
                    block.await();
                } catch (InterruptedException e) {
                    fail(e.toString());
                }
            });
        }
        // wait for the submitted tasks to be consumed by the thread
        // pool
        latch.await();
        // these tasks will fill the thread pool queue
        for (int i = 0; i < queueSize; i++) {
            threadPool.executor(threadPoolName).execute(() -> {
            });
        }
        // these tasks will be rejected
        long counter = 0;
        for (int i = 0; i < rejections; i++) {
            try {
                threadPool.executor(threadPoolName).execute(() -> {
                });
            } catch (EsRejectedExecutionException e) {
                counter++;
            }
        }
        block.countDown();
        assertThat(counter, equalTo(rejections));
        assertThat(stats(threadPool, threadPoolName).getRejected(), equalTo(rejections));
    } finally {
        terminateThreadPoolIfNeeded(threadPool);
    }
}

Example 4 with EsRejectedExecutionException

use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project elasticsearch by elastic.

the class RetryTests method blockExecutor.

/**
     * Blocks the named executor by getting its only thread running a task blocked on a CyclicBarrier and fills the queue with a noop task.
     * So requests to use this queue should get {@link EsRejectedExecutionException}s.
     */
private CyclicBarrier blockExecutor(String name) throws Exception {
    ThreadPool threadPool = getInstanceFromNode(ThreadPool.class);
    CyclicBarrier barrier = new CyclicBarrier(2);
    logger.info("Blocking the [{}] executor", name);
    threadPool.executor(name).execute(() -> {
        try {
            threadPool.executor(name).execute(() -> {
            });
            barrier.await();
            logger.info("Blocked the [{}] executor", name);
            barrier.await();
            logger.info("Unblocking the [{}] executor", name);
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    });
    barrier.await();
    blockedExecutors.add(barrier);
    return barrier;
}

Example 5 with EsRejectedExecutionException

use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project elasticsearch by elastic.

the class ESIntegTestCase method indexRandom.

/**
     * Indexes the given {@link IndexRequestBuilder} instances randomly. It shuffles the given builders and either
     * indexes them in a blocking or async fashion. This is very useful to catch problems that relate to internal document
     * ids or index segment creations. Some features might have bug when a given document is the first or the last in a
     * segment or if only one document is in a segment etc. This method prevents issues like this by randomizing the index
     * layout.
     *
     * @param forceRefresh   if <tt>true</tt> all involved indices are refreshed once the documents are indexed.
     * @param dummyDocuments if <tt>true</tt> some empty dummy documents may be randomly inserted into the document list and deleted once
     *                       all documents are indexed. This is useful to produce deleted documents on the server side.
     * @param maybeFlush     if <tt>true</tt> this method may randomly execute full flushes after index operations.
     * @param builders       the documents to index.
     */
public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean maybeFlush, List<IndexRequestBuilder> builders) throws InterruptedException, ExecutionException {
    Random random = random();
    Set<String> indicesSet = new HashSet<>();
    for (IndexRequestBuilder builder : builders) {
        indicesSet.add(builder.request().index());
    }
    Set<Tuple<String, String>> bogusIds = new HashSet<>();
    if (random.nextBoolean() && !builders.isEmpty() && dummyDocuments) {
        builders = new ArrayList<>(builders);
        final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
        // inject some bogus docs
        final int numBogusDocs = scaledRandomIntBetween(1, builders.size() * 2);
        final int unicodeLen = between(1, 10);
        for (int i = 0; i < numBogusDocs; i++) {
            String id = randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet());
            String index = RandomPicks.randomFrom(random, indices);
            bogusIds.add(new Tuple<>(index, id));
            builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}", XContentType.JSON));
        }
    }
    final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
    Collections.shuffle(builders, random());
    final CopyOnWriteArrayList<Tuple<IndexRequestBuilder, Exception>> errors = new CopyOnWriteArrayList<>();
    List<CountDownLatch> inFlightAsyncOperations = new ArrayList<>();
    // If you are indexing just a few documents then frequently do it one at a time.  If many then frequently in bulk.
    if (builders.size() < FREQUENT_BULK_THRESHOLD ? frequently() : builders.size() < ALWAYS_BULK_THRESHOLD ? rarely() : false) {
        if (frequently()) {
            logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), true, false);
            for (IndexRequestBuilder indexRequestBuilder : builders) {
                indexRequestBuilder.execute(new PayloadLatchedActionListener<IndexResponse, IndexRequestBuilder>(indexRequestBuilder, newLatch(inFlightAsyncOperations), errors));
                postIndexAsyncActions(indices, inFlightAsyncOperations, maybeFlush);
            }
        } else {
            logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), false, false);
            for (IndexRequestBuilder indexRequestBuilder : builders) {
                indexRequestBuilder.execute().actionGet();
                postIndexAsyncActions(indices, inFlightAsyncOperations, maybeFlush);
            }
        }
    } else {
        List<List<IndexRequestBuilder>> partition = eagerPartition(builders, Math.min(MAX_BULK_INDEX_REQUEST_SIZE, Math.max(1, (int) (builders.size() * randomDouble()))));
        logger.info("Index [{}] docs async: [{}] bulk: [{}] partitions [{}]", builders.size(), false, true, partition.size());
        for (List<IndexRequestBuilder> segmented : partition) {
            BulkRequestBuilder bulkBuilder = client().prepareBulk();
            for (IndexRequestBuilder indexRequestBuilder : segmented) {
                bulkBuilder.add(indexRequestBuilder);
            }
            BulkResponse actionGet = bulkBuilder.execute().actionGet();
            assertThat(actionGet.hasFailures() ? actionGet.buildFailureMessage() : "", actionGet.hasFailures(), equalTo(false));
        }
    }
    for (CountDownLatch operation : inFlightAsyncOperations) {
        operation.await();
    }
    final List<Exception> actualErrors = new ArrayList<>();
    for (Tuple<IndexRequestBuilder, Exception> tuple : errors) {
        if (ExceptionsHelper.unwrapCause(tuple.v2()) instanceof EsRejectedExecutionException) {
            // re-index if rejected
            tuple.v1().execute().actionGet();
        } else {
            actualErrors.add(tuple.v2());
        }
    }
    assertThat(actualErrors, emptyIterable());
    if (!bogusIds.isEmpty()) {
        // delete the bogus types again - it might trigger merges or at least holes in the segments and enforces deleted docs!
        for (Tuple<String, String> doc : bogusIds) {
            assertEquals("failed to delete a dummy doc [" + doc.v1() + "][" + doc.v2() + "]", DocWriteResponse.Result.DELETED, client().prepareDelete(doc.v1(), RANDOM_BOGUS_TYPE, doc.v2()).get().getResult());
        }
    }
    if (forceRefresh) {
        assertNoFailures(client().admin().indices().prepareRefresh(indices).setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute().get());
    }
}