Examples with EsRejectedExecutionException org.elasticsearch.common.util.concurrent.EsRejectedExecutionException used on opensource projects

Example 11 with EsRejectedExecutionException

use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project elasticsearch by elastic.

the class AsyncShardFetch method processAsyncFetch.

/**
     * Called by the response handler of the async action to fetch data. Verifies that its still working
     * on the same cache generation, otherwise the results are discarded. It then goes and fills the relevant data for
     * the shard (response + failures), issuing a reroute at the end of it to make sure there will be another round
     * of allocations taking this new data into account.
     */
protected synchronized void processAsyncFetch(ShardId shardId, List<T> responses, List<FailedNodeException> failures) {
    if (closed) {
        // we are closed, no need to process this async fetch at all
        logger.trace("{} ignoring fetched [{}] results, already closed", shardId, type);
        return;
    }
    logger.trace("{} processing fetched [{}] results", shardId, type);
    if (responses != null) {
        for (T response : responses) {
            NodeEntry<T> nodeEntry = cache.get(response.getNode().getId());
            // if the entry is there, and not marked as failed already, process it
            if (nodeEntry == null) {
                continue;
            }
            if (nodeEntry.isFailed()) {
                logger.trace("{} node {} has failed for [{}] (failure [{}])", shardId, nodeEntry.getNodeId(), type, nodeEntry.getFailure());
            } else {
                logger.trace("{} marking {} as done for [{}], result is [{}]", shardId, nodeEntry.getNodeId(), type, response);
                nodeEntry.doneFetching(response);
            }
        }
    }
    if (failures != null) {
        for (FailedNodeException failure : failures) {
            logger.trace("{} processing failure {} for [{}]", shardId, failure, type);
            NodeEntry<T> nodeEntry = cache.get(failure.nodeId());
            // if the entry is there, and not marked as failed already, process it
            if (nodeEntry != null && nodeEntry.isFailed() == false) {
                Throwable unwrappedCause = ExceptionsHelper.unwrapCause(failure.getCause());
                // if the request got rejected or timed out, we need to try it again next time...
                if (unwrappedCause instanceof EsRejectedExecutionException || unwrappedCause instanceof ReceiveTimeoutTransportException || unwrappedCause instanceof ElasticsearchTimeoutException) {
                    nodeEntry.restartFetching();
                } else {
                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("{}: failed to list shard for {} on node [{}]", shardId, type, failure.nodeId()), failure);
                    nodeEntry.doneFetching(failure.getCause());
                }
            }
        }
    }
    reroute(shardId, "post_response");
}

Example 12 with EsRejectedExecutionException

use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project elasticsearch by elastic.

the class BulkProcessorRetryIT method executeBulkRejectionLoad.

private void executeBulkRejectionLoad(BackoffPolicy backoffPolicy, boolean rejectedExecutionExpected) throws Throwable {
    final CorrelatingBackoffPolicy internalPolicy = new CorrelatingBackoffPolicy(backoffPolicy);
    int numberOfAsyncOps = randomIntBetween(600, 700);
    final CountDownLatch latch = new CountDownLatch(numberOfAsyncOps);
    final Set<Object> responses = Collections.newSetFromMap(new ConcurrentHashMap<>());
    assertAcked(prepareCreate(INDEX_NAME));
    ensureGreen();
    BulkProcessor bulkProcessor = BulkProcessor.builder(client(), new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
        // no op
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            internalPolicy.logResponse(response);
            responses.add(response);
            latch.countDown();
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            responses.add(failure);
            latch.countDown();
        }
    }).setBulkActions(1).setConcurrentRequests(randomIntBetween(0, 100)).setBackoffPolicy(internalPolicy).build();
    indexDocs(bulkProcessor, numberOfAsyncOps);
    latch.await(10, TimeUnit.SECONDS);
    bulkProcessor.close();
    assertThat(responses.size(), equalTo(numberOfAsyncOps));
    // validate all responses
    for (Object response : responses) {
        if (response instanceof BulkResponse) {
            BulkResponse bulkResponse = (BulkResponse) response;
            for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
                if (bulkItemResponse.isFailed()) {
                    BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
                    Throwable rootCause = ExceptionsHelper.unwrapCause(failure.getCause());
                    if (rootCause instanceof EsRejectedExecutionException) {
                        if (rejectedExecutionExpected == false) {
                            Iterator<TimeValue> backoffState = internalPolicy.backoffStateFor(bulkResponse);
                            assertNotNull("backoffState is null (indicates a bulk request got rejected without retry)", backoffState);
                            if (backoffState.hasNext()) {
                                // we're not expecting that we overwhelmed it even once when we maxed out the number of retries
                                throw new AssertionError("Got rejected although backoff policy would allow more retries", rootCause);
                            } else {
                                logger.debug("We maxed out the number of bulk retries and got rejected (this is ok).");
                            }
                        }
                    } else {
                        throw new AssertionError("Unexpected failure", rootCause);
                    }
                }
            }
        } else {
            Throwable t = (Throwable) response;
            // we're not expecting any other errors
            throw new AssertionError("Unexpected failure", t);
        }
    }
    client().admin().indices().refresh(new RefreshRequest()).get();
    // validate we did not create any duplicates due to retries
    Matcher<Long> searchResultCount;
    // it is ok if we lost some index operations to rejected executions (which is possible even when backing off (although less likely)
    searchResultCount = lessThanOrEqualTo((long) numberOfAsyncOps);
    SearchResponse results = client().prepareSearch(INDEX_NAME).setTypes(TYPE_NAME).setQuery(QueryBuilders.matchAllQuery()).setSize(0).get();
    assertThat(results.getHits().getTotalHits(), searchResultCount);
}

Example 13 with EsRejectedExecutionException

use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project elasticsearch by elastic.

the class ScheduleWithFixedDelayTests method testOnRejectionCausesCancellation.

public void testOnRejectionCausesCancellation() throws Exception {
    final TimeValue delay = TimeValue.timeValueMillis(10L);
    terminate(threadPool);
    threadPool = new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "fixed delay tests").build()) {

        @Override
        public ScheduledFuture<?> schedule(TimeValue delay, String executor, Runnable command) {
            if (command instanceof ReschedulingRunnable) {
                ((ReschedulingRunnable) command).onRejection(new EsRejectedExecutionException());
            } else {
                fail("this should only be called with a rescheduling runnable in this test");
            }
            return null;
        }
    };
    Runnable runnable = () -> {
    };
    ReschedulingRunnable reschedulingRunnable = new ReschedulingRunnable(runnable, delay, Names.GENERIC, threadPool);
    assertTrue(reschedulingRunnable.isCancelled());
}

Example 14 with EsRejectedExecutionException

use of org.elasticsearch.common.util.concurrent.EsRejectedExecutionException in project elasticsearch by elastic.

the class ClusterService method submitStateUpdateTasks.

/**
     * Submits a batch of cluster state update tasks; submitted updates are guaranteed to be processed together,
     * potentially with more tasks of the same executor.
     *
     * @param source   the source of the cluster state update task
     * @param tasks    a map of update tasks and their corresponding listeners
     * @param config   the cluster state update task configuration
     * @param executor the cluster state update task executor; tasks
     *                 that share the same executor will be executed
     *                 batches on this executor
     * @param <T>      the type of the cluster state update task state
     *
     */
public <T> void submitStateUpdateTasks(final String source, final Map<T, ClusterStateTaskListener> tasks, final ClusterStateTaskConfig config, final ClusterStateTaskExecutor<T> executor) {
    if (!lifecycle.started()) {
        return;
    }
    if (tasks.isEmpty()) {
        return;
    }
    try {
        @SuppressWarnings("unchecked") ClusterStateTaskExecutor<Object> taskExecutor = (ClusterStateTaskExecutor<Object>) executor;
        // convert to an identity map to check for dups based on update tasks semantics of using identity instead of equal
        final IdentityHashMap<Object, ClusterStateTaskListener> tasksIdentity = new IdentityHashMap<>(tasks);
        final List<UpdateTask> updateTasks = tasksIdentity.entrySet().stream().map(entry -> new UpdateTask(source, entry.getKey(), config.priority(), taskExecutor, safe(entry.getValue(), logger))).collect(Collectors.toList());
        synchronized (updateTasksPerExecutor) {
            LinkedHashSet<UpdateTask> existingTasks = updateTasksPerExecutor.computeIfAbsent(executor, k -> new LinkedHashSet<>(updateTasks.size()));
            for (UpdateTask existing : existingTasks) {
                if (tasksIdentity.containsKey(existing.task)) {
                    throw new IllegalStateException("task [" + taskExecutor.describeTasks(Collections.singletonList(existing.task)) + "] with source [" + source + "] is already queued");
                }
            }
            existingTasks.addAll(updateTasks);
        }
        final UpdateTask firstTask = updateTasks.get(0);
        final TimeValue timeout = config.timeout();
        if (timeout != null) {
            threadPoolExecutor.execute(firstTask, threadPool.scheduler(), timeout, () -> onTimeout(updateTasks, source, timeout));
        } else {
            threadPoolExecutor.execute(firstTask);
        }
    } catch (EsRejectedExecutionException e) {
        // to be done here...
        if (!lifecycle.stoppedOrClosed()) {
            throw e;
        }
    }
}