Search in sources :

Example 1 with ClusterStateTaskConfig

use of org.opensearch.cluster.ClusterStateTaskConfig in project OpenSearch by opensearch-project.

the class MasterService method submitStateUpdateTasks.

/**
 * Submits a batch of cluster state update tasks; submitted updates are guaranteed to be processed together,
 * potentially with more tasks of the same executor.
 *
 * @param source   the source of the cluster state update task
 * @param tasks    a map of update tasks and their corresponding listeners
 * @param config   the cluster state update task configuration
 * @param executor the cluster state update task executor; tasks
 *                 that share the same executor will be executed
 *                 batches on this executor
 * @param <T>      the type of the cluster state update task state
 */
public <T> void submitStateUpdateTasks(final String source, final Map<T, ClusterStateTaskListener> tasks, final ClusterStateTaskConfig config, final ClusterStateTaskExecutor<T> executor) {
    if (!lifecycle.started()) {
        return;
    }
    final ThreadContext threadContext = threadPool.getThreadContext();
    final Supplier<ThreadContext.StoredContext> supplier = threadContext.newRestorableContext(true);
    try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
        threadContext.markAsSystemContext();
        List<Batcher.UpdateTask> safeTasks = tasks.entrySet().stream().map(e -> taskBatcher.new UpdateTask(config.priority(), source, e.getKey(), safe(e.getValue(), supplier), executor)).collect(Collectors.toList());
        taskBatcher.submitTasks(safeTasks, config.timeout());
    } catch (OpenSearchRejectedExecutionException e) {
        // to be done here...
        if (!lifecycle.stoppedOrClosed()) {
            throw e;
        }
    }
}
Also used : ClusterStateTaskListener(org.opensearch.cluster.ClusterStateTaskListener) OpenSearchRejectedExecutionException(org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException) AckedClusterStateTaskListener(org.opensearch.cluster.AckedClusterStateTaskListener) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) Arrays(java.util.Arrays) Metadata(org.opensearch.cluster.metadata.Metadata) CountDown(org.opensearch.common.util.concurrent.CountDown) PrioritizedOpenSearchThreadPoolExecutor(org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor) ThreadPool(org.opensearch.threadpool.ThreadPool) Priority(org.opensearch.common.Priority) Node(org.opensearch.node.Node) FutureUtils(org.opensearch.common.util.concurrent.FutureUtils) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) OpenSearchExecutors(org.opensearch.common.util.concurrent.OpenSearchExecutors) Supplier(java.util.function.Supplier) ClusterState(org.opensearch.cluster.ClusterState) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) ClusterTasksResult(org.opensearch.cluster.ClusterStateTaskExecutor.ClusterTasksResult) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) Locale(java.util.Locale) OpenSearchExecutors.daemonThreadFactory(org.opensearch.common.util.concurrent.OpenSearchExecutors.daemonThreadFactory) Assertions(org.opensearch.Assertions) Map(java.util.Map) ClusterStatePublisher(org.opensearch.cluster.coordination.ClusterStatePublisher) ClusterStateTaskConfig(org.opensearch.cluster.ClusterStateTaskConfig) ClusterSettings(org.opensearch.common.settings.ClusterSettings) ProcessClusterEventTimeoutException(org.opensearch.cluster.metadata.ProcessClusterEventTimeoutException) Setting(org.opensearch.common.settings.Setting) TimeValue(org.opensearch.common.unit.TimeValue) ClusterStateTaskExecutor(org.opensearch.cluster.ClusterStateTaskExecutor) Settings(org.opensearch.common.settings.Settings) Discovery(org.opensearch.discovery.Discovery) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) FailedToCommitClusterStateException(org.opensearch.cluster.coordination.FailedToCommitClusterStateException) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) AbstractLifecycleComponent(org.opensearch.common.component.AbstractLifecycleComponent) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Builder(org.opensearch.cluster.ClusterState.Builder) RoutingTable(org.opensearch.cluster.routing.RoutingTable) LogManager(org.apache.logging.log4j.LogManager) Collections(java.util.Collections) Text(org.opensearch.common.text.Text) Scheduler(org.opensearch.threadpool.Scheduler) ClusterChangedEvent(org.opensearch.cluster.ClusterChangedEvent) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) OpenSearchRejectedExecutionException(org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException)

Example 2 with ClusterStateTaskConfig

use of org.opensearch.cluster.ClusterStateTaskConfig in project OpenSearch by opensearch-project.

the class TaskBatcherTests method testOneExecutorDoesntStarveAnother.

public void testOneExecutorDoesntStarveAnother() throws InterruptedException {
    final List<String> executionOrder = Collections.synchronizedList(new ArrayList<>());
    final Semaphore allowProcessing = new Semaphore(0);
    final Semaphore startedProcessing = new Semaphore(0);
    class TaskExecutor implements TestExecutor<String> {

        @Override
        public void execute(List<String> tasks) {
            // do this first, so startedProcessing can be used as a notification that this is done.
            executionOrder.addAll(tasks);
            startedProcessing.release(tasks.size());
            try {
                allowProcessing.acquire(tasks.size());
            } catch (InterruptedException e) {
                throw new RuntimeException(e);
            }
        }
    }
    TaskExecutor executorA = new TaskExecutor();
    TaskExecutor executorB = new TaskExecutor();
    final ClusterStateTaskConfig config = ClusterStateTaskConfig.build(Priority.NORMAL);
    final TestListener noopListener = (source, e) -> {
        throw new AssertionError(e);
    };
    // this blocks the cluster state queue, so we can set it up right
    submitTask("0", "A0", config, executorA, noopListener);
    // wait to be processed
    startedProcessing.acquire(1);
    assertThat(executionOrder, equalTo(Arrays.asList("A0")));
    // these will be the first batch
    submitTask("1", "A1", config, executorA, noopListener);
    submitTask("2", "A2", config, executorA, noopListener);
    // release the first 0 task, but not the second
    allowProcessing.release(1);
    startedProcessing.acquire(2);
    assertThat(executionOrder, equalTo(Arrays.asList("A0", "A1", "A2")));
    // setup the queue with pending tasks for another executor same priority
    submitTask("3", "B3", config, executorB, noopListener);
    submitTask("4", "B4", config, executorB, noopListener);
    submitTask("5", "A5", config, executorA, noopListener);
    submitTask("6", "A6", config, executorA, noopListener);
    // now release the processing
    allowProcessing.release(6);
    // wait for last task to be processed
    startedProcessing.acquire(4);
    assertThat(executionOrder, equalTo(Arrays.asList("A0", "A1", "A2", "B3", "B4", "A5", "A6")));
}
Also used : Matchers.hasToString(org.hamcrest.Matchers.hasToString) Arrays(java.util.Arrays) PrioritizedOpenSearchThreadPoolExecutor(org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor) Priority(org.opensearch.common.Priority) HashMap(java.util.HashMap) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Map(java.util.Map) ClusterStateTaskConfig(org.opensearch.cluster.ClusterStateTaskConfig) ProcessClusterEventTimeoutException(org.opensearch.cluster.metadata.ProcessClusterEventTimeoutException) Before(org.junit.Before) CyclicBarrier(java.util.concurrent.CyclicBarrier) Matchers.empty(org.hamcrest.Matchers.empty) TimeValue(org.opensearch.common.unit.TimeValue) Semaphore(java.util.concurrent.Semaphore) Set(java.util.Set) BrokenBarrierException(java.util.concurrent.BrokenBarrierException) Collectors(java.util.stream.Collectors) Tuple(org.opensearch.common.collect.Tuple) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Collections(java.util.Collections) Matchers.containsString(org.hamcrest.Matchers.containsString) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) Matchers.hasToString(org.hamcrest.Matchers.hasToString) Matchers.containsString(org.hamcrest.Matchers.containsString) Semaphore(java.util.concurrent.Semaphore) ClusterStateTaskConfig(org.opensearch.cluster.ClusterStateTaskConfig) ArrayList(java.util.ArrayList) List(java.util.List) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList)

Aggregations

Arrays (java.util.Arrays)2 Collections (java.util.Collections)2 List (java.util.List)2 Map (java.util.Map)2 Collectors (java.util.stream.Collectors)2 Logger (org.apache.logging.log4j.Logger)2 ParameterizedMessage (org.apache.logging.log4j.message.ParameterizedMessage)2 ClusterStateTaskConfig (org.opensearch.cluster.ClusterStateTaskConfig)2 ProcessClusterEventTimeoutException (org.opensearch.cluster.metadata.ProcessClusterEventTimeoutException)2 Priority (org.opensearch.common.Priority)2 TimeValue (org.opensearch.common.unit.TimeValue)2 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 Locale (java.util.Locale)1 Objects (java.util.Objects)1 Set (java.util.Set)1 BrokenBarrierException (java.util.concurrent.BrokenBarrierException)1 CopyOnWriteArrayList (java.util.concurrent.CopyOnWriteArrayList)1 CountDownLatch (java.util.concurrent.CountDownLatch)1