Examples with ExecutorBuilder org.opensearch.threadpool.ExecutorBuilder used on opensource projects

Example 1 with ExecutorBuilder

use of org.opensearch.threadpool.ExecutorBuilder in project job-scheduler by opensearch-project.

the class JobSchedulerPlugin method getExecutorBuilders.

@Override
public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
    final int processorCount = OpenSearchExecutors.allocatedProcessors(settings);
    List<ExecutorBuilder<?>> executorBuilders = new ArrayList<>();
    executorBuilders.add(new FixedExecutorBuilder(settings, OPEN_DISTRO_JOB_SCHEDULER_THREAD_POOL_NAME, processorCount, 200, "opendistro.jobscheduler.threadpool"));
    return executorBuilders;
}

Example 2 with ExecutorBuilder

use of org.opensearch.threadpool.ExecutorBuilder in project asynchronous-search by opensearch-project.

the class AsynchronousSearchPlugin method getExecutorBuilders.

// TODO Revisit these once we performance test the feature
@Override
public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
    final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings);
    List<ExecutorBuilder<?>> executorBuilders = new ArrayList<>();
    executorBuilders.add(new ScalingExecutorBuilder(OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30)));
    return executorBuilders;
}

Example 3 with ExecutorBuilder

use of org.opensearch.threadpool.ExecutorBuilder in project asynchronous-search by opensearch-project.

the class AsynchronousSearchServiceTests method testFindContext.

public void testFindContext() throws InterruptedException {
    DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
    ThreadPool testThreadPool = null;
    try {
        testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder);
        ClusterService mockClusterService = TestUtils.createClusterService(settings, testThreadPool, discoveryNode, clusterSettings);
        FakeClient fakeClient = new FakeClient(testThreadPool);
        AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService);
        AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, mockClusterService, testThreadPool);
        AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList()));
        TimeValue keepAlive = timeValueHours(9);
        boolean keepOnCompletion = randomBoolean();
        User user1 = TestClientUtils.randomUser();
        User user2 = TestClientUtils.randomUser();
        SearchRequest searchRequest = new SearchRequest();
        SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest);
        submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion);
        submitAsynchronousSearchRequest.keepAlive(keepAlive);
        AsynchronousSearchContext context = asService.createAndStoreContext(submitAsynchronousSearchRequest, System.currentTimeMillis(), () -> null, user1);
        assertTrue(context instanceof AsynchronousSearchActiveContext);
        AsynchronousSearchActiveContext asActiveContext = (AsynchronousSearchActiveContext) context;
        assertNull(asActiveContext.getTask());
        assertNull(asActiveContext.getAsynchronousSearchId());
        assertEquals(asActiveContext.getAsynchronousSearchState(), AsynchronousSearchState.INIT);
        assertEquals(asActiveContext.getUser(), user1);
        // bootstrap search
        AsynchronousSearchTask task = new AsynchronousSearchTask(randomNonNegativeLong(), "transport", SearchAction.NAME, TaskId.EMPTY_TASK_ID, emptyMap(), (AsynchronousSearchActiveContext) context, null, (c) -> {
        });
        asService.bootstrapSearch(task, context.getContextId());
        assertEquals(asActiveContext.getTask(), task);
        assertEquals(asActiveContext.getStartTimeMillis(), task.getStartTime());
        assertEquals(asActiveContext.getExpirationTimeMillis(), task.getStartTime() + keepAlive.millis());
        assertEquals(asActiveContext.getAsynchronousSearchState(), AsynchronousSearchState.RUNNING);
        CountDownLatch findContextLatch = new CountDownLatch(3);
        ActionListener<AsynchronousSearchContext> expectedSuccessfulActive = new LatchedActionListener<>(wrap(r -> {
            assertTrue(r instanceof AsynchronousSearchActiveContext);
            assertEquals(r, context);
        }, e -> fail("Find context shouldn't have failed. " + e.getMessage())), findContextLatch);
        ActionListener<AsynchronousSearchContext> expectedSecurityException = new LatchedActionListener<>(wrap(r -> fail("Expecting security exception"), e -> assertTrue(e instanceof ResourceNotFoundException)), findContextLatch);
        asService.findContext(asActiveContext.getAsynchronousSearchId(), asActiveContext.getContextId(), user1, expectedSuccessfulActive);
        asService.findContext(asActiveContext.getAsynchronousSearchId(), asActiveContext.getContextId(), user2, expectedSecurityException);
        asService.findContext(asActiveContext.getAsynchronousSearchId(), asActiveContext.getContextId(), null, expectedSuccessfulActive);
        findContextLatch.await();
        AsynchronousSearchProgressListener asProgressListener = asActiveContext.getAsynchronousSearchProgressListener();
        boolean success = randomBoolean();
        if (success) {
            // successful search response
            asProgressListener.onResponse(getMockSearchResponse());
        } else {
            // exception occurred in search
            asProgressListener.onFailure(new RuntimeException("test"));
        }
        waitUntil(() -> asService.getAllActiveContexts().isEmpty());
        if (keepOnCompletion) {
            // persist to disk
            assertEquals(1, fakeClient.persistenceCount.intValue());
        } else {
            assertEquals(fakeClient.persistenceCount, Integer.valueOf(0));
            CountDownLatch freeContextLatch = new CountDownLatch(1);
            asService.findContext(context.getAsynchronousSearchId(), context.getContextId(), null, new LatchedActionListener<>(wrap(r -> fail("No context should have been found but found " + asService.getAllActiveContexts().size()), e -> assertTrue(e instanceof ResourceNotFoundException)), freeContextLatch));
            freeContextLatch.await();
        }
    } finally {
        ThreadPool.terminate(testThreadPool, 200, TimeUnit.MILLISECONDS);
    }
}

Example 4 with ExecutorBuilder

use of org.opensearch.threadpool.ExecutorBuilder in project asynchronous-search by opensearch-project.

the class AsynchronousSearchServiceTests method createObjects.

@Before
public void createObjects() {
    settings = Settings.builder().put("node.name", "test").put("cluster.name", "ClusterServiceTests").put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10).put(AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING.getKey(), true).build();
    final Set<Setting<?>> settingsSet = Stream.concat(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), Stream.of(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING)).collect(Collectors.toSet());
    final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings);
    List<ExecutorBuilder<?>> executorBuilders = new ArrayList<>();
    executorBuilders.add(new ScalingExecutorBuilder(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30)));
    executorBuilder = executorBuilders.get(0);
    clusterSettings = new ClusterSettings(settings, settingsSet);
    blockPersistence = false;
}

Example 5 with ExecutorBuilder

use of org.opensearch.threadpool.ExecutorBuilder in project asynchronous-search by opensearch-project.

the class AsynchronousSearchPostProcessorTests method createObjects.

@Before
public void createObjects() {
    Settings settings = Settings.builder().put("node.name", "test").put("cluster.name", "ClusterServiceTests").put(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING.getKey(), 10).build();
    final Set<Setting<?>> settingsSet = Stream.concat(ClusterSettings.BUILT_IN_CLUSTER_SETTINGS.stream(), Stream.of(AsynchronousSearchActiveStore.NODE_CONCURRENT_RUNNING_SEARCHES_SETTING, AsynchronousSearchService.PERSIST_SEARCH_FAILURES_SETTING, AsynchronousSearchService.MAX_KEEP_ALIVE_SETTING, AsynchronousSearchService.MAX_SEARCH_RUNNING_TIME_SETTING, AsynchronousSearchService.MAX_WAIT_FOR_COMPLETION_TIMEOUT_SETTING)).collect(Collectors.toSet());
    final int availableProcessors = OpenSearchExecutors.allocatedProcessors(settings);
    List<ExecutorBuilder<?>> executorBuilders = new ArrayList<>();
    executorBuilders.add(new ScalingExecutorBuilder(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, 1, Math.min(2 * availableProcessors, Math.max(128, 512)), TimeValue.timeValueMinutes(30)));
    executorBuilder = executorBuilders.get(0);
    clusterSettings = new ClusterSettings(settings, settingsSet);
}