Examples with AcknowledgedResponse org.opensearch.search.asynchronous.response.AcknowledgedResponse used on opensource projects

Example 1 with AcknowledgedResponse

use of org.opensearch.search.asynchronous.response.AcknowledgedResponse in project asynchronous-search by opensearch-project.

the class AsynchronousSearchPersistenceService method deleteExpiredResponses.

/**
 * Deletes all responses past a given expiration time
 *
 * @param listener               invoked once delete by query request completes
 * @param expirationTimeInMillis the expiration time
 */
public void deleteExpiredResponses(ActionListener<AcknowledgedResponse> listener, long expirationTimeInMillis) {
    if (indexExists() == false) {
        logger.debug("Async search index not yet created! Nothing to delete.");
        listener.onResponse(new AcknowledgedResponse(true));
    } else {
        DeleteByQueryRequest request = new DeleteByQueryRequest(ASYNC_SEARCH_RESPONSE_INDEX).setQuery(QueryBuilders.rangeQuery(EXPIRATION_TIME_MILLIS).lte(expirationTimeInMillis));
        client.execute(DeleteByQueryAction.INSTANCE, request, ActionListener.wrap(deleteResponse -> {
            if ((deleteResponse.getBulkFailures() != null && deleteResponse.getBulkFailures().size() > 0) || (deleteResponse.getSearchFailures() != null && deleteResponse.getSearchFailures().size() > 0)) {
                logger.error("Failed to delete expired asynchronous search responses with bulk failures[{}] / search " + "failures [{}]", deleteResponse.getBulkFailures(), deleteResponse.getSearchFailures());
                listener.onResponse(new AcknowledgedResponse(false));
            } else {
                logger.debug("Successfully deleted expired responses");
                listener.onResponse(new AcknowledgedResponse(true));
            }
        }, (e) -> {
            logger.error(() -> new ParameterizedMessage("Failed to delete expired response for expiration time {}", expirationTimeInMillis), e);
            final Throwable cause = ExceptionsHelper.unwrapCause(e);
            listener.onFailure(cause instanceof Exception ? (Exception) cause : new NotSerializableExceptionWrapper(cause));
        }));
    }
}

Example 2 with AcknowledgedResponse

use of org.opensearch.search.asynchronous.response.AcknowledgedResponse in project asynchronous-search by opensearch-project.

the class AsynchronousSearchRejectionIT method testSimulatedSearchRejectionLoad.

@TestLogging(value = "_root:DEBUG", reason = "flaky")
public void testSimulatedSearchRejectionLoad() throws Throwable {
    for (int i = 0; i < 10; i++) {
        client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "1").get();
    }
    AtomicInteger numRejections = new AtomicInteger();
    AtomicInteger numRnf = new AtomicInteger();
    AtomicInteger numTimeouts = new AtomicInteger();
    AtomicInteger numFailures = new AtomicInteger();
    int numberOfAsyncOps = randomIntBetween(100, 200);
    final CountDownLatch latch = new CountDownLatch(numberOfAsyncOps * 2);
    final CopyOnWriteArrayList<Object> responses = new CopyOnWriteArrayList<>();
    for (int i = 0; i < numberOfAsyncOps; i++) {
        SearchRequest request = client().prepareSearch("test").setSearchType(SearchType.QUERY_THEN_FETCH).setQuery(QueryBuilders.matchQuery("field", "1")).request();
        SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(request);
        submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMinutes(1));
        boolean keepOnCompletion = randomBoolean();
        submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion);
        client().execute(SubmitAsynchronousSearchAction.INSTANCE, submitAsynchronousSearchRequest, new LatchedActionListener<>(new ActionListener<AsynchronousSearchResponse>() {

            @Override
            public void onResponse(AsynchronousSearchResponse asynchronousSearchResponse) {
                if (asynchronousSearchResponse.getSearchResponse() != null) {
                    responses.add(asynchronousSearchResponse.getSearchResponse());
                } else if (asynchronousSearchResponse.getError() != null) {
                    responses.add(asynchronousSearchResponse.getError());
                }
                if (asynchronousSearchResponse.getId() == null) {
                    // task cancelled by the time we process final response/error due to during partial merge failure.
                    // no  delete required
                    latch.countDown();
                } else {
                    DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest(asynchronousSearchResponse.getId());
                    client().execute(DeleteAsynchronousSearchAction.INSTANCE, deleteAsynchronousSearchRequest, new LatchedActionListener<>(new ActionListener<AcknowledgedResponse>() {

                        @Override
                        public void onResponse(AcknowledgedResponse acknowledgedResponse) {
                            assertTrue(acknowledgedResponse.isAcknowledged());
                        }

                        @Override
                        public void onFailure(Exception e) {
                            Throwable cause = ExceptionsHelper.unwrapCause(e);
                            if (cause instanceof OpenSearchRejectedExecutionException) {
                                numRejections.incrementAndGet();
                            } else if (cause instanceof OpenSearchTimeoutException) {
                                numTimeouts.incrementAndGet();
                            } else if (cause instanceof ResourceNotFoundException) {
                                // deletion is in race with task cancellation due to partial merge failure
                                numRnf.getAndIncrement();
                            } else {
                                numFailures.incrementAndGet();
                            }
                        }
                    }, latch));
                }
            }

            @Override
            public void onFailure(Exception e) {
                responses.add(e);
                assertThat(e.getMessage(), startsWith("Trying to create too many concurrent searches"));
                latch.countDown();
            }
        }, latch));
    }
    latch.await();
    // validate all responses
    for (Object response : responses) {
        if (response instanceof SearchResponse) {
            SearchResponse searchResponse = (SearchResponse) response;
            for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
                assertTrue("got unexpected reason..." + failure.reason(), failure.reason().toLowerCase(Locale.ENGLISH).contains("rejected"));
            }
        } else {
            Exception t = (Exception) response;
            Throwable unwrap = ExceptionsHelper.unwrapCause(t);
            if (unwrap instanceof SearchPhaseExecutionException) {
                SearchPhaseExecutionException e = (SearchPhaseExecutionException) unwrap;
                for (ShardSearchFailure failure : e.shardFailures()) {
                    assertTrue("got unexpected reason..." + failure.reason(), // task cancellation can occur due to partial merge failures
                    failure.reason().toLowerCase(Locale.ENGLISH).contains("cancelled") || failure.reason().toLowerCase(Locale.ENGLISH).contains("rejected"));
                }
            // we have have null responses if submit completes before search starts
            } else if (unwrap instanceof OpenSearchRejectedExecutionException == false) {
                throw new AssertionError("unexpected failure + ", (Throwable) response);
            }
        }
    }
    assertThat(responses.size(), equalTo(numberOfAsyncOps));
    assertThat(numFailures.get(), equalTo(0));
}

Example 3 with AcknowledgedResponse

use of org.opensearch.search.asynchronous.response.AcknowledgedResponse in project asynchronous-search by opensearch-project.

the class DeleteAsynchronousSearchSingleNodeIT method assertConcurrentDeletesForBlockedSearch.

private void assertConcurrentDeletesForBlockedSearch(String id, TriConsumer<AtomicInteger, AtomicInteger, AtomicInteger> assertionConsumer, int concurrentRuns, List<SearchDelayPlugin> plugins) throws Exception {
    AtomicInteger numDeleteAcknowledged = new AtomicInteger();
    AtomicInteger numDeleteUnAcknowledged = new AtomicInteger();
    AtomicInteger numResourceNotFound = new AtomicInteger();
    TestThreadPool testThreadPool = null;
    try {
        testThreadPool = new TestThreadPool(DeleteAsynchronousSearchSingleNodeIT.class.getName());
        int numThreads = concurrentRuns;
        List<Runnable> operationThreads = new ArrayList<>();
        CountDownLatch countDownLatch = new CountDownLatch(numThreads);
        for (int i = 0; i < numThreads; i++) {
            Runnable thread = () -> {
                logger.info("Triggering asynchronous search delete --->");
                DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest(id);
                executeDeleteAsynchronousSearch(client(), deleteAsynchronousSearchRequest, new LatchedActionListener<>(new ActionListener<AcknowledgedResponse>() {

                    @Override
                    public void onResponse(AcknowledgedResponse acknowledgedResponse) {
                        if (acknowledgedResponse.isAcknowledged()) {
                            numDeleteAcknowledged.incrementAndGet();
                        } else {
                            numDeleteUnAcknowledged.incrementAndGet();
                        }
                    }

                    @Override
                    public void onFailure(Exception e) {
                        if (e instanceof ResourceNotFoundException) {
                            numResourceNotFound.incrementAndGet();
                        }
                    }
                }, countDownLatch));
            };
            operationThreads.add(thread);
        }
        TestThreadPool finalTestThreadPool = testThreadPool;
        operationThreads.forEach(runnable -> finalTestThreadPool.executor("generic").execute(runnable));
        countDownLatch.await();
        disableBlocks(plugins);
        assertionConsumer.apply(numDeleteAcknowledged, numDeleteUnAcknowledged, numResourceNotFound);
    } finally {
        ThreadPool.terminate(testThreadPool, 500, TimeUnit.MILLISECONDS);
    }
}

Example 4 with AcknowledgedResponse

use of org.opensearch.search.asynchronous.response.AcknowledgedResponse in project asynchronous-search by opensearch-project.

the class AsynchronousSearchManagementService method performCleanUp.

public final void performCleanUp() {
    final ThreadContext threadContext = threadPool.getThreadContext();
    try (ThreadContext.StoredContext ignore = threadContext.stashContext()) {
        // we have to execute under the system context so that if security is enabled the sync is authorized
        threadContext.markAsSystemContext();
        ImmutableOpenMap<String, DiscoveryNode> dataNodes = clusterService.state().nodes().getDataNodes();
        List<DiscoveryNode> nodes = Stream.of(dataNodes.values().toArray(DiscoveryNode.class)).collect(Collectors.toList());
        if (nodes == null || nodes.isEmpty()) {
            logger.debug("Found empty data nodes with asynchronous search enabled attribute [{}] for response clean up", dataNodes);
            return;
        }
        int pos = Randomness.get().nextInt(nodes.size());
        DiscoveryNode randomNode = nodes.get(pos);
        transportService.sendRequest(randomNode, PERSISTED_RESPONSE_CLEANUP_ACTION_NAME, new AsynchronousSearchCleanUpRequest(threadPool.absoluteTimeInMillis()), new TransportResponseHandler<AcknowledgedResponse>() {

            @Override
            public AcknowledgedResponse read(StreamInput in) throws IOException {
                return new AcknowledgedResponse(in);
            }

            @Override
            public void handleResponse(AcknowledgedResponse response) {
                logger.debug("Successfully executed clean up action on node [{}] with response [{}]", randomNode, response.isAcknowledged());
            }

            @Override
            public void handleException(TransportException e) {
                logger.error(() -> new ParameterizedMessage("Exception executing action [{}]", PERSISTED_RESPONSE_CLEANUP_ACTION_NAME), e);
            }

            @Override
            public String executor() {
                return AsynchronousSearchPlugin.OPEN_DISTRO_ASYNC_SEARCH_GENERIC_THREAD_POOL_NAME;
            }
        });
    } catch (Exception ex) {
        logger.error("Failed to schedule asynchronous search cleanup", ex);
    }
}

Example 5 with AcknowledgedResponse

use of org.opensearch.search.asynchronous.response.AcknowledgedResponse in project asynchronous-search by opensearch-project.

the class AsynchronousSearchManagementServiceIT method testDeletesExpiredAsynchronousSearchResponseFromPersistedStore.

public void testDeletesExpiredAsynchronousSearchResponseFromPersistedStore() throws Exception {
    String idx = "idx";
    assertAcked(prepareCreate(idx).addMapping("type", "ip", "type=ip", "ips", "type=ip"));
    waitForRelocation(ClusterHealthStatus.GREEN);
    indexRandom(true, client().prepareIndex(idx).setId("1").setSource("ip", "192.168.1.7", "ips", Arrays.asList("192.168.0.13", "192.168.1.2")), client().prepareIndex(idx).setId("2").setSource("ip", "192.168.1.10", "ips", Arrays.asList("192.168.1.25", "192.168.1.28")), client().prepareIndex(idx).setId("3").setSource("ip", "2001:db8::ff00:42:8329", "ips", Arrays.asList("2001:db8::ff00:42:8329", "2001:db8::ff00:42:8380")));
    assertAcked(prepareCreate("idx_unmapped"));
    waitForRelocation(ClusterHealthStatus.GREEN);
    refresh();
    final AtomicReference<AsynchronousSearchResponse> asResponseRef = new AtomicReference<>();
    final AtomicReference<AsynchronousSearchResponse> nonExpiredAsynchronousSearchResponseRef = new AtomicReference<>();
    final AtomicReference<Exception> exceptionRef = new AtomicReference<>();
    SearchRequest searchRequest = new SearchRequest(idx);
    SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(searchRequest);
    submitAsynchronousSearchRequest.keepOnCompletion(true);
    submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMillis(5000));
    CountDownLatch latch = new CountDownLatch(2);
    client().execute(SubmitAsynchronousSearchAction.INSTANCE, submitAsynchronousSearchRequest, new ActionListener<AsynchronousSearchResponse>() {

        @Override
        public void onResponse(AsynchronousSearchResponse asResponse) {
            asResponseRef.set(asResponse);
            exceptionRef.set(asResponse.getError());
            latch.countDown();
        }

        @Override
        public void onFailure(Exception e) {
            exceptionRef.set(e);
            latch.countDown();
        }
    });
    // submit another request to verify that the second request is not cancelled
    client().execute(SubmitAsynchronousSearchAction.INSTANCE, submitAsynchronousSearchRequest, new ActionListener<AsynchronousSearchResponse>() {

        @Override
        public void onResponse(AsynchronousSearchResponse asResponse) {
            nonExpiredAsynchronousSearchResponseRef.set(asResponse);
            latch.countDown();
        }

        @Override
        public void onFailure(Exception e) {
            latch.countDown();
        }
    });
    latch.await();
    waitUntil(() -> verifyResponsePersisted(asResponseRef.get().getId()));
    waitUntil(() -> verifyResponsePersisted(nonExpiredAsynchronousSearchResponseRef.get().getId()));
    CountDownLatch updateLatch = new CountDownLatch(1);
    GetAsynchronousSearchRequest getAsynchronousSearchRequest = new GetAsynchronousSearchRequest(asResponseRef.get().getId());
    getAsynchronousSearchRequest.setKeepAlive(TimeValue.timeValueMillis(1));
    client().execute(GetAsynchronousSearchAction.INSTANCE, getAsynchronousSearchRequest, new ActionListener<AsynchronousSearchResponse>() {

        @Override
        public void onResponse(AsynchronousSearchResponse asResponse) {
            asResponseRef.set(asResponse);
            exceptionRef.set(asResponse.getError());
            updateLatch.countDown();
        }

        @Override
        public void onFailure(Exception e) {
            exceptionRef.set(e);
            updateLatch.countDown();
        }
    });
    updateLatch.await();
    waitUntil(() -> verifyResponseRemoved(asResponseRef.get().getId()));
    assertBusy(() -> assertTrue(verifyResponsePersisted(nonExpiredAsynchronousSearchResponseRef.get().getId())));
    // delete the non expired response explicitly
    CountDownLatch deleteLatch = new CountDownLatch(1);
    DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest(nonExpiredAsynchronousSearchResponseRef.get().getId());
    client().execute(DeleteAsynchronousSearchAction.INSTANCE, deleteAsynchronousSearchRequest, new ActionListener<AcknowledgedResponse>() {

        @Override
        public void onResponse(AcknowledgedResponse response) {
            assertTrue(response.isAcknowledged());
            deleteLatch.countDown();
        }

        @Override
        public void onFailure(Exception e) {
            deleteLatch.countDown();
            fail("Cleanup failed");
        }
    });
    deleteLatch.await();
}