Examples with LatchedActionListener org.opensearch.action.LatchedActionListener used on opensource projects

Example 31 with LatchedActionListener

use of org.opensearch.action.LatchedActionListener in project asynchronous-search by opensearch-project.

the class MixedOperationSingleNodeIT method assertConcurrentGetOrUpdatesWithDeletes.

private void assertConcurrentGetOrUpdatesWithDeletes(AsynchronousSearchResponse submitResponse, QuadConsumer<AtomicInteger, AtomicInteger, AtomicInteger, AtomicInteger> assertionConsumer, boolean update, int concurrentRuns, boolean retainResponse) throws InterruptedException {
    AtomicInteger numSuccess = new AtomicInteger();
    AtomicInteger numGetFailures = new AtomicInteger();
    AtomicInteger numVersionConflictFailures = new AtomicInteger();
    AtomicInteger numResourceNotFoundFailures = new AtomicInteger();
    AtomicInteger numTimeouts = new AtomicInteger();
    TestThreadPool testThreadPool = null;
    try {
        testThreadPool = new TestThreadPool(GetAsynchronousSearchSingleNodeIT.class.getName());
        int numThreads = concurrentRuns;
        // 5 hours in millis
        long lowerKeepAliveMillis = 5 * 1000 * 60 * 60;
        // 10 hours in millis
        long higherKeepAliveMillis = 10 * 1000 * 60 * 60;
        List<Runnable> operationThreads = new ArrayList<>();
        CountDownLatch countDownLatch = new CountDownLatch(numThreads);
        long randomDeleteThread = randomLongBetween(0, numThreads - 1);
        for (int i = 0; i < numThreads; i++) {
            long keepAlive = randomLongBetween(lowerKeepAliveMillis, higherKeepAliveMillis);
            int currentThreadIteration = i;
            Runnable thread = () -> {
                if (currentThreadIteration == randomDeleteThread) {
                    DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest(submitResponse.getId());
                    executeDeleteAsynchronousSearch(client(), deleteAsynchronousSearchRequest, new LatchedActionListener<>(new ActionListener<AcknowledgedResponse>() {

                        @Override
                        public void onResponse(AcknowledgedResponse acknowledgedResponse) {
                            assertTrue(acknowledgedResponse.isAcknowledged());
                            numSuccess.incrementAndGet();
                        }

                        @Override
                        public void onFailure(Exception e) {
                            if (e instanceof OpenSearchTimeoutException) {
                                numTimeouts.incrementAndGet();
                            } else {
                                fail("Unexpected exception " + e.getMessage());
                            }
                        }
                    }, countDownLatch));
                } else {
                    GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest(submitResponse.getId());
                    long requestedTime = System.currentTimeMillis() + keepAlive;
                    if (update) {
                        logger.info("Triggering asynchronous search gets with keep alive [{}] --->", requestedTime);
                        getAsynchronousSearchRequest.setKeepAlive(TimeValue.timeValueMillis(keepAlive));
                    }
                    getAsynchronousSearchRequest.setWaitForCompletionTimeout(TimeValue.timeValueMillis(randomLongBetween(1, 5000)));
                    executeGetAsynchronousSearch(client(), getAsynchronousSearchRequest, new LatchedActionListener<>(new ActionListener<AsynchronousSearchResponse>() {

                        @Override
                        public void onResponse(AsynchronousSearchResponse asResponse) {
                            if (update) {
                                assertThat(asResponse.getExpirationTimeMillis(), greaterThanOrEqualTo(System.currentTimeMillis() + lowerKeepAliveMillis));
                                assertThat(asResponse.getExpirationTimeMillis(), lessThanOrEqualTo(System.currentTimeMillis() + higherKeepAliveMillis));
                            }
                            numSuccess.incrementAndGet();
                        }

                        @Override
                        public void onFailure(Exception e) {
                            if (e instanceof VersionConflictEngineException) {
                                numVersionConflictFailures.incrementAndGet();
                            } else if (e instanceof ResourceNotFoundException) {
                                numResourceNotFoundFailures.incrementAndGet();
                            } else if (e instanceof OpenSearchTimeoutException) {
                                numTimeouts.incrementAndGet();
                            } else {
                                numGetFailures.incrementAndGet();
                            }
                        }
                    }, countDownLatch));
                }
            };
            operationThreads.add(thread);
        }
        TestThreadPool finalTestThreadPool = testThreadPool;
        operationThreads.forEach(runnable -> finalTestThreadPool.executor("generic").execute(runnable));
        countDownLatch.await();
        if (retainResponse && update) {
            assertEquals(numTimeouts.get(), 0);
            assertionConsumer.apply(numSuccess, numGetFailures, numVersionConflictFailures, numTimeouts);
        } else {
            assertionConsumer.apply(numSuccess, numGetFailures, numVersionConflictFailures, numResourceNotFoundFailures);
        }
    } finally {
        ThreadPool.terminate(testThreadPool, 500, TimeUnit.MILLISECONDS);
    }
}

Example 32 with LatchedActionListener

use of org.opensearch.action.LatchedActionListener in project asynchronous-search by opensearch-project.

the class SubmitAsynchronousSearchSingleNodeIT method assertConcurrentSubmits.

private void assertConcurrentSubmits(SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest, SearchResponse searchResponse, TriConsumer<AtomicInteger, AtomicInteger, AtomicInteger> assertionConsumer, int concurrentRuns) throws InterruptedException {
    AtomicInteger numStartedAsynchronousSearch = new AtomicInteger();
    AtomicInteger numFailedAsynchronousSearch = new AtomicInteger();
    AtomicInteger numErrorResponseAsynchronousSearch = new AtomicInteger();
    final ClusterState state = getInstanceFromNode(ClusterService.class).state();
    TestThreadPool testThreadPool = null;
    CountDownLatch countDownLatch;
    try {
        testThreadPool = new TestThreadPool(SubmitAsynchronousSearchSingleNodeIT.class.getName());
        int numThreads = concurrentRuns;
        List<Runnable> operationThreads = new ArrayList<>();
        if (submitAsynchronousSearchRequest.getKeepOnCompletion()) {
            // we also need to delete asynchronous search response to ensure test completes gracefully with no background tasks
            // running
            countDownLatch = new CountDownLatch(2 * numThreads);
        } else {
            countDownLatch = new CountDownLatch(numThreads);
        }
        for (int i = 0; i < numThreads; i++) {
            CountDownLatch finalCountDownLatch = countDownLatch;
            Runnable thread = () -> {
                logger.info("Triggering asynchronous search submit --->");
                executeSubmitAsynchronousSearch(client(), submitAsynchronousSearchRequest, new ActionListener<AsynchronousSearchResponse>() {

                    @Override
                    public void onResponse(AsynchronousSearchResponse asResponse) {
                        if (asResponse.getId() != null) {
                            AsynchronousSearchId asId = AsynchronousSearchIdConverter.parseAsyncId(asResponse.getId());
                            assertEquals(state.nodes().getLocalNodeId(), asId.getNode());
                            AsynchronousSearchAssertions.assertSearchResponses(searchResponse, asResponse.getSearchResponse());
                            numStartedAsynchronousSearch.incrementAndGet();
                        }
                        if (asResponse.getError() != null) {
                            numErrorResponseAsynchronousSearch.incrementAndGet();
                        }
                        finalCountDownLatch.countDown();
                        if (submitAsynchronousSearchRequest.getKeepOnCompletion()) {
                            DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest(asResponse.getId());
                            executeDeleteAsynchronousSearch(client(), deleteAsynchronousSearchRequest, new LatchedActionListener<>(new ActionListener<AcknowledgedResponse>() {

                                @Override
                                public void onResponse(AcknowledgedResponse acknowledgedResponse) {
                                    assertTrue(acknowledgedResponse.isAcknowledged());
                                }

                                @Override
                                public void onFailure(Exception e) {
                                    fail("Search deletion failed for asynchronous search id " + e.getMessage());
                                }
                            }, finalCountDownLatch));
                        }
                        ;
                    }

                    @Override
                    public void onFailure(Exception e) {
                        numFailedAsynchronousSearch.incrementAndGet();
                        finalCountDownLatch.countDown();
                    }
                });
            };
            operationThreads.add(thread);
        }
        TestThreadPool finalTestThreadPool = testThreadPool;
        operationThreads.forEach(runnable -> finalTestThreadPool.executor("generic").execute(runnable));
        countDownLatch.await();
        assertionConsumer.apply(numStartedAsynchronousSearch, numFailedAsynchronousSearch, numErrorResponseAsynchronousSearch);
    } finally {
        ThreadPool.terminate(testThreadPool, 500, TimeUnit.MILLISECONDS);
    }
}

Example 33 with LatchedActionListener

use of org.opensearch.action.LatchedActionListener in project asynchronous-search by opensearch-project.

the class AsynchronousSearchRequestRoutingIT method testRequestForwardingToCoordinatorNodeForRunningAsynchronousSearch.

public void testRequestForwardingToCoordinatorNodeForRunningAsynchronousSearch() throws Exception {
    List<ScriptedBlockPlugin> plugins = initBlockFactory();
    String index = "idx";
    assertAcked(prepareCreate(index).addMapping("type", "ip", "type=ip", "ips", "type=ip"));
    waitForRelocation(ClusterHealthStatus.GREEN);
    indexRandom(true, client().prepareIndex(index).setId("1").setSource("ip", "192.168.1.7", "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), client().prepareIndex(index).setId("2").setSource("ip", "192.168.1.10", "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), client().prepareIndex(index).setId("3").setSource("ip", "2001:db8::ff00:42:8329", "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380")));
    assertAcked(prepareCreate("idx_unmapped"));
    waitForRelocation(ClusterHealthStatus.GREEN);
    refresh();
    SearchRequest searchRequest = client().prepareSearch(index).setQuery(scriptQuery(new Script(ScriptType.INLINE, "mockscript", SCRIPT_NAME, Collections.emptyMap()))).request();
    SubmitAsynchronousSearchRequest request = new SubmitAsynchronousSearchRequest(searchRequest);
    request.keepOnCompletion(false);
    request.waitForCompletionTimeout(TimeValue.timeValueMillis(0));
    CountDownLatch latch = new CountDownLatch(1);
    client().execute(SubmitAsynchronousSearchAction.INSTANCE, request, new LatchedActionListener<>(new ActionListener<AsynchronousSearchResponse>() {

        @Override
        public void onResponse(AsynchronousSearchResponse submitResponse) {
            String id = submitResponse.getId();
            assertNotNull(id);
            assertEquals(AsynchronousSearchState.RUNNING, submitResponse.getState());
            AsynchronousSearchId asId = AsynchronousSearchIdConverter.parseAsyncId(id);
            ClusterService clusterService = internalCluster().getInstance(ClusterService.class);
            assertEquals(clusterService.state().nodes().getDataNodes().size(), 5);
            List<String> nonCoordinatorNodeNames = new LinkedList<>();
            clusterService.state().nodes().iterator().forEachRemaining(node -> {
                if (asId.getNode().equals(node.getId()) == false)
                    nonCoordinatorNodeNames.add(node.getName());
            });
            nonCoordinatorNodeNames.forEach(n -> {
                try {
                    AsynchronousSearchResponse getResponse = client(n).execute(GetAsynchronousSearchAction.INSTANCE, new GetAsynchronousSearchRequest(id)).get();
                    assertEquals(getResponse.getState(), AsynchronousSearchState.RUNNING);
                } catch (InterruptedException | ExecutionException e) {
                    fail("Get asynchronous search request should not have failed");
                }
            });
            String randomNonCoordinatorNode = nonCoordinatorNodeNames.get(randomInt(nonCoordinatorNodeNames.size() - 1));
            try {
                AcknowledgedResponse acknowledgedResponse = client(randomNonCoordinatorNode).execute(DeleteAsynchronousSearchAction.INSTANCE, new DeleteAsynchronousSearchRequest(id)).get();
                assertTrue(acknowledgedResponse.isAcknowledged());
                ExecutionException executionException = expectThrows(ExecutionException.class, () -> client().execute(GetAsynchronousSearchAction.INSTANCE, new GetAsynchronousSearchRequest(id)).get());
                assertThat(executionException.getMessage(), containsString("ResourceNotFoundException"));
            } catch (InterruptedException | ExecutionException e) {
                fail("Delete asynchronous search request from random non-coordinator node should have succeeded.");
            }
        }

        @Override
        public void onFailure(Exception e) {
            fail(e.getMessage());
        }
    }, latch));
    latch.await();
    disableBlocks(plugins);
}

Example 34 with LatchedActionListener

use of org.opensearch.action.LatchedActionListener in project asynchronous-search by opensearch-project.

the class AsynchronousSearchServiceTests method testFindContextOnNonExistentSearch.

public void testFindContextOnNonExistentSearch() throws InterruptedException {
    DiscoveryNode discoveryNode = new DiscoveryNode("node", OpenSearchTestCase.buildNewFakeTransportAddress(), emptyMap(), DiscoveryNodeRole.BUILT_IN_ROLES, Version.CURRENT);
    ThreadPool testThreadPool = null;
    try {
        testThreadPool = new TestThreadPool(AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME, executorBuilder);
        ClusterService mockClusterService = ClusterServiceUtils.createClusterService(testThreadPool, discoveryNode, clusterSettings);
        FakeClient fakeClient = new FakeClient(testThreadPool);
        AsynchronousSearchActiveStore asActiveStore = new AsynchronousSearchActiveStore(mockClusterService);
        AsynchronousSearchPersistenceService persistenceService = new AsynchronousSearchPersistenceService(fakeClient, mockClusterService, testThreadPool);
        AsynchronousSearchService asService = new AsynchronousSearchService(persistenceService, asActiveStore, fakeClient, mockClusterService, testThreadPool, new InternalAsynchronousSearchStats(), new NamedWriteableRegistry(emptyList()));
        TimeValue keepAlive = timeValueHours(9);
        User user1 = TestClientUtils.randomUser();
        SearchRequest searchRequest = new SearchRequest();
        SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest);
        submitAsynchronousSearchRequest.keepOnCompletion(false);
        submitAsynchronousSearchRequest.keepAlive(keepAlive);
        CountDownLatch findContextLatch = new CountDownLatch(2);
        ActionListener<AsynchronousSearchContext> failureExpectingListener = new LatchedActionListener<>(wrap(r -> fail(), e -> assertTrue(e instanceof ResourceNotFoundException)), findContextLatch);
        asService.findContext("nonExistentId", new AsynchronousSearchContextId(randomAlphaOfLength(10), randomNonNegativeLong()), null, failureExpectingListener);
        asService.findContext("nonExistentId", new AsynchronousSearchContextId(randomAlphaOfLength(10), randomNonNegativeLong()), user1, failureExpectingListener);
        findContextLatch.await();
    } finally {
        ThreadPool.terminate(testThreadPool, 200, TimeUnit.MILLISECONDS);
    }
}

Example 35 with LatchedActionListener

use of org.opensearch.action.LatchedActionListener in project asynchronous-search by opensearch-project.

the class AsynchronousSearchPersistenceServiceIT method testCreateConcurrentDocsWhenIndexNotExists.

public void testCreateConcurrentDocsWhenIndexNotExists() throws InterruptedException, IOException, ExecutionException {
    AsynchronousSearchPersistenceService persistenceService = getInstanceFromNode(AsynchronousSearchPersistenceService.class);
    TransportService transportService = getInstanceFromNode(TransportService.class);
    SearchResponse searchResponse = client().search(new SearchRequest(TEST_INDEX)).get();
    AsynchronousSearchId asId1 = generateNewAsynchronousSearchId(transportService);
    AsynchronousSearchId asId2 = generateNewAsynchronousSearchId(transportService);
    AsynchronousSearchPersistenceModel model1 = new AsynchronousSearchPersistenceModel(System.currentTimeMillis(), System.currentTimeMillis() + new TimeValue(10, TimeUnit.DAYS).getMillis(), searchResponse, null, null);
    String id1 = AsynchronousSearchIdConverter.buildAsyncId(asId1);
    AsynchronousSearchPersistenceModel model2 = new AsynchronousSearchPersistenceModel(System.currentTimeMillis(), System.currentTimeMillis() + new TimeValue(10, TimeUnit.DAYS).getMillis(), searchResponse, null, null);
    String id2 = AsynchronousSearchIdConverter.buildAsyncId(asId2);
    CountDownLatch createLatch = new CountDownLatch(2);
    threadPool.generic().execute(() -> persistenceService.storeResponse(id1, model1, new LatchedActionListener<>(ActionListener.wrap(r -> assertSuccessfulResponseCreation(id1, r), e -> {
        logger.debug("expect successful create, got", e);
        fail("Expected successful create, got " + e.getMessage());
    }), createLatch)));
    threadPool.generic().execute(() -> persistenceService.storeResponse(id2, model2, new LatchedActionListener<>(ActionListener.wrap(r -> assertSuccessfulResponseCreation(id2, r), e -> {
        logger.debug("expect successful create, got", e);
        fail("Expected successful create, got " + e.getMessage());
    }), createLatch)));
    createLatch.await();
    CountDownLatch getLatch1 = new CountDownLatch(1);
    persistenceService.getResponse(id1, null, new LatchedActionListener<>(ActionListener.wrap((AsynchronousSearchPersistenceModel r) -> assertEquals(model1, r), e -> {
        logger.debug("expect successful get result, got", e);
        fail("Expected successful get result, got " + e.getMessage());
    }), getLatch1));
    getLatch1.await();
    CountDownLatch getLatch2 = new CountDownLatch(1);
    persistenceService.getResponse(id2, null, new LatchedActionListener<>(ActionListener.wrap((AsynchronousSearchPersistenceModel r) -> assertEquals(model2, r), e -> {
        logger.debug("expect successful create, got", e);
        fail("Expected successful create, got " + e.getMessage());
    }), getLatch2));
    getLatch2.await();
}