use of org.opensearch.search.asynchronous.response.AsynchronousSearchResponse in project asynchronous-search by opensearch-project.
the class AsynchronousSearchPersistenceContextTests method testSerializationRoundTripWithSearchResponse.
/**
* asynchronous search persistence context serializes search response into {@linkplain BytesReference}. We verify that de-serializing
* the{@linkplain BytesReference} yields the same object.@throws IOException when there is a serialization issue
*/
public void testSerializationRoundTripWithSearchResponse() throws IOException {
AsynchronousSearchContextId asContextId = new AsynchronousSearchContextId(UUID.randomUUID().toString(), randomNonNegativeLong());
String id = AsynchronousSearchIdConverter.buildAsyncId(new AsynchronousSearchId(UUID.randomUUID().toString(), randomNonNegativeLong(), asContextId));
long expirationTimeMillis = randomNonNegativeLong();
long startTimeMillis = randomNonNegativeLong();
SearchResponse searchResponse = getMockSearchResponse();
User user = TestClientUtils.randomUserOrNull();
SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList());
AsynchronousSearchPersistenceContext asPersistenceContext = new AsynchronousSearchPersistenceContext(id, asContextId, new AsynchronousSearchPersistenceModel(startTimeMillis, expirationTimeMillis, searchResponse, null, user), System::currentTimeMillis, new NamedWriteableRegistry(searchModule.getNamedWriteables()));
assertEquals(asPersistenceContext, new AsynchronousSearchPersistenceContext(id, asContextId, new AsynchronousSearchPersistenceModel(startTimeMillis, expirationTimeMillis, searchResponse, null, user), System::currentTimeMillis, new NamedWriteableRegistry(Collections.emptyList())));
assertEquals(asPersistenceContext.getAsynchronousSearchResponse(), new AsynchronousSearchResponse(id, asPersistenceContext.getAsynchronousSearchState(), startTimeMillis, expirationTimeMillis, searchResponse, null));
}
use of org.opensearch.search.asynchronous.response.AsynchronousSearchResponse in project asynchronous-search by opensearch-project.
the class AsynchronousSearchRejectionIT method testSearchFailures.
public void testSearchFailures() throws Exception {
for (int i = 0; i < 10; i++) {
client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "1").get();
}
int numberOfAsyncOps = randomIntBetween(100, 200);
final CountDownLatch latch = new CountDownLatch(numberOfAsyncOps);
final CopyOnWriteArrayList<Object> responses = new CopyOnWriteArrayList<>();
TestThreadPool threadPool = null;
try {
threadPool = new TestThreadPool(AsynchronousSearchProgressListenerIT.class.getName());
for (int i = 0; i < numberOfAsyncOps; i++) {
SearchRequest request = client().prepareSearch("test").setSearchType(SearchType.QUERY_THEN_FETCH).setQuery(QueryBuilders.matchQuery("field", "1")).request();
AtomicReference<SearchResponse> responseRef = new AtomicReference<>();
AtomicInteger reduceContextInvocation = new AtomicInteger();
AsynchronousSearchProgressListener listener;
SearchService service = internalCluster().getInstance(SearchService.class);
InternalAggregation.ReduceContextBuilder reduceContextBuilder = service.aggReduceContextBuilder(request);
AtomicReference<Exception> exceptionRef = new AtomicReference<>();
Function<SearchResponse, AsynchronousSearchResponse> responseFunction = (r) -> null;
Function<Exception, AsynchronousSearchResponse> failureFunction = (e) -> null;
listener = new AsynchronousSearchProgressListener(threadPool.relativeTimeInMillis(), responseFunction, failureFunction, threadPool.generic(), threadPool::relativeTimeInMillis, () -> reduceContextBuilder) {
@Override
public void onResponse(SearchResponse searchResponse) {
assertTrue(responseRef.compareAndSet(null, searchResponse));
AsynchronousSearchAssertions.assertSearchResponses(searchResponse, this.partialResponse());
latch.countDown();
}
@Override
public void onFailure(Exception exception) {
assertTrue(exceptionRef.compareAndSet(null, exception));
latch.countDown();
}
};
client().execute(SearchAction.INSTANCE, new SearchRequest(request) {
@Override
public SearchTask createTask(long id, String type, String action, TaskId parentTaskId, Map<String, String> headers) {
SearchTask task = super.createTask(id, type, action, parentTaskId, headers);
task.setProgressListener(listener);
return task;
}
}, listener);
}
latch.await();
} finally {
ThreadPool.terminate(threadPool, 100, TimeUnit.MILLISECONDS);
}
}
use of org.opensearch.search.asynchronous.response.AsynchronousSearchResponse in project asynchronous-search by opensearch-project.
the class AsynchronousSearchRejectionIT method testSimulatedSearchRejectionLoad.
@TestLogging(value = "_root:DEBUG", reason = "flaky")
public void testSimulatedSearchRejectionLoad() throws Throwable {
for (int i = 0; i < 10; i++) {
client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "1").get();
}
AtomicInteger numRejections = new AtomicInteger();
AtomicInteger numRnf = new AtomicInteger();
AtomicInteger numTimeouts = new AtomicInteger();
AtomicInteger numFailures = new AtomicInteger();
int numberOfAsyncOps = randomIntBetween(100, 200);
final CountDownLatch latch = new CountDownLatch(numberOfAsyncOps * 2);
final CopyOnWriteArrayList<Object> responses = new CopyOnWriteArrayList<>();
for (int i = 0; i < numberOfAsyncOps; i++) {
SearchRequest request = client().prepareSearch("test").setSearchType(SearchType.QUERY_THEN_FETCH).setQuery(QueryBuilders.matchQuery("field", "1")).request();
SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(request);
submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueMinutes(1));
boolean keepOnCompletion = randomBoolean();
submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion);
client().execute(SubmitAsynchronousSearchAction.INSTANCE, submitAsynchronousSearchRequest, new LatchedActionListener<>(new ActionListener<AsynchronousSearchResponse>() {
@Override
public void onResponse(AsynchronousSearchResponse asynchronousSearchResponse) {
if (asynchronousSearchResponse.getSearchResponse() != null) {
responses.add(asynchronousSearchResponse.getSearchResponse());
} else if (asynchronousSearchResponse.getError() != null) {
responses.add(asynchronousSearchResponse.getError());
}
if (asynchronousSearchResponse.getId() == null) {
// task cancelled by the time we process final response/error due to during partial merge failure.
// no delete required
latch.countDown();
} else {
DeleteAsynchronousSearchRequest deleteAsynchronousSearchRequest = new DeleteAsynchronousSearchRequest(asynchronousSearchResponse.getId());
client().execute(DeleteAsynchronousSearchAction.INSTANCE, deleteAsynchronousSearchRequest, new LatchedActionListener<>(new ActionListener<AcknowledgedResponse>() {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
assertTrue(acknowledgedResponse.isAcknowledged());
}
@Override
public void onFailure(Exception e) {
Throwable cause = ExceptionsHelper.unwrapCause(e);
if (cause instanceof OpenSearchRejectedExecutionException) {
numRejections.incrementAndGet();
} else if (cause instanceof OpenSearchTimeoutException) {
numTimeouts.incrementAndGet();
} else if (cause instanceof ResourceNotFoundException) {
// deletion is in race with task cancellation due to partial merge failure
numRnf.getAndIncrement();
} else {
numFailures.incrementAndGet();
}
}
}, latch));
}
}
@Override
public void onFailure(Exception e) {
responses.add(e);
assertThat(e.getMessage(), startsWith("Trying to create too many concurrent searches"));
latch.countDown();
}
}, latch));
}
latch.await();
// validate all responses
for (Object response : responses) {
if (response instanceof SearchResponse) {
SearchResponse searchResponse = (SearchResponse) response;
for (ShardSearchFailure failure : searchResponse.getShardFailures()) {
assertTrue("got unexpected reason..." + failure.reason(), failure.reason().toLowerCase(Locale.ENGLISH).contains("rejected"));
}
} else {
Exception t = (Exception) response;
Throwable unwrap = ExceptionsHelper.unwrapCause(t);
if (unwrap instanceof SearchPhaseExecutionException) {
SearchPhaseExecutionException e = (SearchPhaseExecutionException) unwrap;
for (ShardSearchFailure failure : e.shardFailures()) {
assertTrue("got unexpected reason..." + failure.reason(), // task cancellation can occur due to partial merge failures
failure.reason().toLowerCase(Locale.ENGLISH).contains("cancelled") || failure.reason().toLowerCase(Locale.ENGLISH).contains("rejected"));
}
// we have have null responses if submit completes before search starts
} else if (unwrap instanceof OpenSearchRejectedExecutionException == false) {
throw new AssertionError("unexpected failure + ", (Throwable) response);
}
}
}
assertThat(responses.size(), equalTo(numberOfAsyncOps));
assertThat(numFailures.get(), equalTo(0));
}
use of org.opensearch.search.asynchronous.response.AsynchronousSearchResponse in project asynchronous-search by opensearch-project.
the class AsynchronousSearchStatsIT method testStatsAcrossNodes.
@TestLogging(value = "_root:DEBUG", reason = "flaky")
public void testStatsAcrossNodes() throws InterruptedException, ExecutionException {
TestThreadPool threadPool = null;
try {
threadPool = new TestThreadPool(AsynchronousSearchStatsIT.class.getName());
String index = "idx";
createIndex(index);
indexRandom(super.ignoreExternalCluster(), client().prepareIndex(index).setId("1").setSource("field1", "the quick brown fox jumps"), client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), client().prepareIndex(index).setId("3").setSource("field1", "quick"));
List<DiscoveryNode> dataNodes = new LinkedList<>();
clusterService().state().nodes().getDataNodes().iterator().forEachRemaining(node -> {
dataNodes.add(node.value);
});
assertFalse(dataNodes.isEmpty());
int numThreads = 20;
List<Runnable> threads = new ArrayList<>();
AtomicLong expectedNumSuccesses = new AtomicLong();
AtomicLong expectedNumFailures = new AtomicLong();
AtomicLong expectedNumPersisted = new AtomicLong();
CountDownLatch latch = new CountDownLatch(numThreads);
for (int i = 0; i < numThreads; i++) {
threads.add(() -> {
try {
boolean success = randomBoolean();
boolean keepOnCompletion = randomBoolean();
if (keepOnCompletion) {
expectedNumPersisted.getAndIncrement();
}
SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest;
if (success) {
expectedNumSuccesses.getAndIncrement();
submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest(index));
submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueSeconds(2));
submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion);
} else {
expectedNumFailures.getAndIncrement();
submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest("non_existent_index"));
submitAsynchronousSearchRequest.keepOnCompletion(keepOnCompletion);
}
AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch(client(dataNodes.get(randomInt(1)).getName()), submitAsynchronousSearchRequest);
if (keepOnCompletion) {
TestClientUtils.assertResponsePersistence(client(), asResponse.getId());
}
} catch (Exception e) {
fail(e.getMessage());
} finally {
latch.countDown();
}
});
}
TestThreadPool finalThreadPool = threadPool;
threads.forEach(t -> finalThreadPool.generic().execute(t));
latch.await();
AsynchronousSearchStatsResponse statsResponse = client().execute(AsynchronousSearchStatsAction.INSTANCE, new AsynchronousSearchStatsRequest()).get();
AtomicLong actualNumSuccesses = new AtomicLong();
AtomicLong actualNumFailures = new AtomicLong();
AtomicLong actualNumPersisted = new AtomicLong();
for (AsynchronousSearchStats node : statsResponse.getNodes()) {
AsynchronousSearchCountStats asCountStats = node.getAsynchronousSearchCountStats();
assertEquals(asCountStats.getRunningCount(), 0);
assertThat(expectedNumSuccesses.get(), greaterThanOrEqualTo(asCountStats.getCompletedCount()));
actualNumSuccesses.getAndAdd(asCountStats.getCompletedCount());
assertThat(expectedNumFailures.get(), greaterThanOrEqualTo(asCountStats.getFailedCount()));
actualNumFailures.getAndAdd(asCountStats.getFailedCount());
assertThat(expectedNumPersisted.get(), greaterThanOrEqualTo(asCountStats.getPersistedCount()));
actualNumPersisted.getAndAdd(asCountStats.getPersistedCount());
}
assertEquals(expectedNumPersisted.get(), actualNumPersisted.get());
assertEquals(expectedNumFailures.get(), actualNumFailures.get());
assertEquals(expectedNumSuccesses.get(), actualNumSuccesses.get());
waitForAsyncSearchTasksToComplete();
} finally {
ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
}
}
use of org.opensearch.search.asynchronous.response.AsynchronousSearchResponse in project asynchronous-search by opensearch-project.
the class AsynchronousSearchStatsIT method testNodewiseStats.
public void testNodewiseStats() throws InterruptedException {
String index = "idx";
createIndex(index);
indexRandom(super.ignoreExternalCluster(), client().prepareIndex(index).setId("1").setSource("field1", "the quick brown fox jumps"), client().prepareIndex(index).setId("2").setSource("field1", "quick brown"), client().prepareIndex(index).setId("3").setSource("field1", "quick"));
SubmitAsynchronousSearchRequest submitAsynchronousSearchRequest = new SubmitAsynchronousSearchRequest(new SearchRequest(index));
submitAsynchronousSearchRequest.waitForCompletionTimeout(TimeValue.timeValueSeconds(2));
submitAsynchronousSearchRequest.keepOnCompletion(true);
List<DiscoveryNode> dataNodes = new LinkedList<>();
clusterService().state().nodes().getDataNodes().iterator().forEachRemaining(node -> {
dataNodes.add(node.value);
});
assertFalse(dataNodes.isEmpty());
DiscoveryNode randomDataNode = dataNodes.get(randomInt(dataNodes.size() - 1));
try {
AsynchronousSearchResponse asResponse = executeSubmitAsynchronousSearch(client(randomDataNode.getName()), submitAsynchronousSearchRequest);
assertNotNull(asResponse.getSearchResponse());
TestClientUtils.assertResponsePersistence(client(), asResponse.getId());
AsynchronousSearchStatsResponse statsResponse = client().execute(AsynchronousSearchStatsAction.INSTANCE, new AsynchronousSearchStatsRequest()).get();
String responseAsString = statsResponse.toString();
for (DiscoveryNode dataNode : dataNodes) {
assertThat(responseAsString, containsString(dataNode.getId()));
}
statsResponse.getNodes().forEach(nodeStats -> {
AsynchronousSearchCountStats asCountStats = nodeStats.getAsynchronousSearchCountStats();
if (nodeStats.getNode().equals(randomDataNode)) {
assertEquals(1, asCountStats.getPersistedCount());
assertEquals(1, asCountStats.getCompletedCount());
assertEquals(1, asCountStats.getSubmittedCount());
assertEquals(1, asCountStats.getInitializedCount());
assertEquals(0, asCountStats.getFailedCount());
assertEquals(0, asCountStats.getRunningCount());
assertEquals(0, asCountStats.getCancelledCount());
} else {
assertEquals(0, asCountStats.getPersistedCount());
assertEquals(0, asCountStats.getCompletedCount());
assertEquals(0, asCountStats.getFailedCount());
assertEquals(0, asCountStats.getRunningCount());
}
});
} catch (Exception e) {
fail(e.getMessage());
}
}
Aggregations