use of org.opensearch.action.ActionFuture in project OpenSearch by opensearch-project.
the class RethrottleTests method testCase.
private void testCase(AbstractBulkByScrollRequestBuilder<?, ?> request, String actionName) throws Exception {
logger.info("Starting test for [{}] with [{}] slices", actionName, request.request().getSlices());
/* Add ten documents per slice so most slices will have many documents to process, having to go to multiple batches.
* We can't rely on the slices being evenly sized but 10 means we have some pretty big slices.
*/
createIndex("test");
int numSlices = expectedSlices(request.request().getSlices(), "test");
List<IndexRequestBuilder> docs = new ArrayList<>();
for (int i = 0; i < numSlices * 10; i++) {
docs.add(client().prepareIndex("test").setId(Integer.toString(i)).setSource("foo", "bar"));
}
indexRandom(true, docs);
// Start a request that will never finish unless we rethrottle it
// Throttle "forever"
request.setRequestsPerSecond(.000001f);
// Make sure we use multiple batches
request.source().setSize(1);
ActionFuture<? extends BulkByScrollResponse> responseListener = request.execute();
TaskGroup taskGroupToRethrottle = findTaskToRethrottle(actionName, numSlices);
TaskId taskToRethrottle = taskGroupToRethrottle.getTaskInfo().getTaskId();
if (numSlices == 1) {
assertThat(taskGroupToRethrottle.getChildTasks(), empty());
} else {
// There should be a sane number of child tasks running
assertThat(taskGroupToRethrottle.getChildTasks(), hasSize(allOf(greaterThanOrEqualTo(1), lessThanOrEqualTo(numSlices))));
// Wait for all of the sub tasks to start (or finish, some might finish early, all that matters is that not all do)
assertBusy(() -> {
BulkByScrollTask.Status parent = (BulkByScrollTask.Status) client().admin().cluster().prepareGetTask(taskToRethrottle).get().getTask().getTask().getStatus();
long finishedSubTasks = parent.getSliceStatuses().stream().filter(Objects::nonNull).count();
ListTasksResponse list = client().admin().cluster().prepareListTasks().setParentTaskId(taskToRethrottle).get();
list.rethrowFailures("subtasks");
assertThat(finishedSubTasks + list.getTasks().size(), greaterThanOrEqualTo((long) numSlices));
assertThat(list.getTasks().size(), greaterThan(0));
});
}
// Now rethrottle it so it'll finish
// No throttle or "very fast"
float newRequestsPerSecond = randomBoolean() ? Float.POSITIVE_INFINITY : between(1, 1000) * 100000;
ListTasksResponse rethrottleResponse = rethrottleTask(taskToRethrottle, newRequestsPerSecond);
BulkByScrollTask.Status status = (BulkByScrollTask.Status) rethrottleResponse.getTasks().get(0).getStatus();
// Now check the resulting requests per second.
if (numSlices == 1) {
// If there is a single slice it should match perfectly
assertEquals(newRequestsPerSecond, status.getRequestsPerSecond(), Float.MIN_NORMAL);
} else {
/* Check that at least one slice was rethrottled. We won't always rethrottle all of them because they might have completed.
* With multiple slices these numbers might not add up perfectly, thus the 1.01F. */
long unfinished = status.getSliceStatuses().stream().filter(Objects::nonNull).filter(slice -> slice.getStatus().getTotal() > slice.getStatus().getSuccessfullyProcessed()).count();
float maxExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ? Float.POSITIVE_INFINITY : (newRequestsPerSecond / unfinished) * 1.01F;
float minExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ? Float.POSITIVE_INFINITY : (newRequestsPerSecond / numSlices) * 0.99F;
boolean oneSliceRethrottled = false;
float totalRequestsPerSecond = 0;
for (BulkByScrollTask.StatusOrException statusOrException : status.getSliceStatuses()) {
if (statusOrException == null) {
/* The slice can be null here because it was completed but hadn't reported its success back to the task when the
* rethrottle request came through. */
continue;
}
assertNull(statusOrException.getException());
BulkByScrollTask.Status slice = statusOrException.getStatus();
if (slice.getTotal() > slice.getSuccessfullyProcessed()) {
// This slice reports as not having completed so it should have been processed.
assertThat(slice.getRequestsPerSecond(), both(greaterThanOrEqualTo(minExpectedSliceRequestsPerSecond)).and(lessThanOrEqualTo(maxExpectedSliceRequestsPerSecond)));
}
if (minExpectedSliceRequestsPerSecond <= slice.getRequestsPerSecond() && slice.getRequestsPerSecond() <= maxExpectedSliceRequestsPerSecond) {
oneSliceRethrottled = true;
}
totalRequestsPerSecond += slice.getRequestsPerSecond();
}
assertTrue("At least one slice must be rethrottled", oneSliceRethrottled);
/* Now assert that the parent request has the total requests per second. This is a much weaker assertion than that the parent
* actually has the newRequestsPerSecond. For the most part it will. Sometimes it'll be greater because only unfinished requests
* are rethrottled, the finished ones just keep whatever requests per second they had while they were running. But it might
* also be less than newRequestsPerSecond because the newRequestsPerSecond is divided among running sub-requests and then the
* requests are rethrottled. If one request finishes in between the division and the application of the new throttle then it
* won't be rethrottled, thus only contributing its lower total. */
assertEquals(totalRequestsPerSecond, status.getRequestsPerSecond(), totalRequestsPerSecond * 0.0001f);
}
// Now the response should come back quickly because we've rethrottled the request
BulkByScrollResponse response = responseListener.get();
// It'd be bad if the entire require completed in a single batch. The test wouldn't be testing anything.
assertThat("Entire request completed in a single batch. This may invalidate the test as throttling is done between batches.", response.getBatches(), greaterThanOrEqualTo(numSlices));
}
use of org.opensearch.action.ActionFuture in project OpenSearch by opensearch-project.
the class ConcurrentSnapshotsIT method testEquivalentDeletesAreDeduplicated.
public void testEquivalentDeletesAreDeduplicated() throws Exception {
final String masterName = internalCluster().startMasterOnlyNode();
internalCluster().startDataOnlyNode();
final String repoName = "test-repo";
createRepository(repoName, "mock");
createIndexWithContent("index-test");
createNSnapshots(repoName, randomIntBetween(1, 5));
blockNodeOnAnyFiles(repoName, masterName);
final int deletes = randomIntBetween(2, 10);
final List<ActionFuture<AcknowledgedResponse>> deleteResponses = new ArrayList<>(deletes);
for (int i = 0; i < deletes; ++i) {
deleteResponses.add(client().admin().cluster().prepareDeleteSnapshot(repoName, "*").execute());
}
waitForBlock(masterName, repoName, TimeValue.timeValueSeconds(30L));
awaitNDeletionsInProgress(1);
for (ActionFuture<AcknowledgedResponse> deleteResponse : deleteResponses) {
assertFalse(deleteResponse.isDone());
}
awaitNDeletionsInProgress(1);
unblockNode(repoName, masterName);
for (ActionFuture<AcknowledgedResponse> deleteResponse : deleteResponses) {
assertAcked(deleteResponse.get());
}
}
use of org.opensearch.action.ActionFuture in project OpenSearch by opensearch-project.
the class CancellableTasksIT method testBanOnlyNodesWithOutstandingDescendantTasks.
public void testBanOnlyNodesWithOutstandingDescendantTasks() throws Exception {
if (randomBoolean()) {
internalCluster().startNodes(randomIntBetween(1, 3));
}
Set<DiscoveryNode> nodes = StreamSupport.stream(clusterService().state().nodes().spliterator(), false).collect(Collectors.toSet());
final TestRequest rootRequest = generateTestRequest(nodes, 0, between(1, 4));
ActionFuture<TestResponse> rootTaskFuture = client().execute(TransportTestAction.ACTION, rootRequest);
Set<TestRequest> pendingRequests = allowPartialRequest(rootRequest);
TaskId rootTaskId = getRootTaskId(rootRequest);
ActionFuture<CancelTasksResponse> cancelFuture = client().admin().cluster().prepareCancelTasks().setTaskId(rootTaskId).waitForCompletion(true).execute();
if (randomBoolean()) {
List<TaskInfo> runningTasks = client().admin().cluster().prepareListTasks().setActions(TransportTestAction.ACTION.name()).setDetailed(true).get().getTasks();
for (TaskInfo subTask : randomSubsetOf(runningTasks)) {
client().admin().cluster().prepareCancelTasks().setTaskId(subTask.getTaskId()).waitForCompletion(false).get();
}
}
assertBusy(() -> {
for (DiscoveryNode node : nodes) {
TaskManager taskManager = internalCluster().getInstance(TransportService.class, node.getName()).getTaskManager();
Set<TaskId> expectedBans = new HashSet<>();
for (TestRequest req : pendingRequests) {
if (req.node.equals(node)) {
List<Task> childTasks = taskManager.getTasks().values().stream().filter(t -> t.getParentTaskId() != null && t.getDescription().equals(req.taskDescription())).collect(Collectors.toList());
assertThat(childTasks, hasSize(1));
CancellableTask childTask = (CancellableTask) childTasks.get(0);
assertTrue(childTask.isCancelled());
expectedBans.add(childTask.getParentTaskId());
}
}
assertThat(taskManager.getBannedTaskIds(), equalTo(expectedBans));
}
}, 30, TimeUnit.SECONDS);
allowEntireRequest(rootRequest);
cancelFuture.actionGet();
waitForRootTask(rootTaskFuture);
ensureAllBansRemoved();
}
use of org.opensearch.action.ActionFuture in project OpenSearch by opensearch-project.
the class DedicatedClusterSnapshotRestoreIT method testAbortWaitsOnDataNode.
public void testAbortWaitsOnDataNode() throws Exception {
internalCluster().startMasterOnlyNode();
final String dataNodeName = internalCluster().startDataOnlyNode();
final String indexName = "test-index";
createIndex(indexName);
index(indexName, "_doc", "some_id", "foo", "bar");
final String otherDataNode = internalCluster().startDataOnlyNode();
final String repoName = "test-repo";
createRepository(repoName, "mock");
blockAllDataNodes(repoName);
final String snapshotName = "test-snap";
final ActionFuture<CreateSnapshotResponse> snapshotResponse = startFullSnapshot(repoName, snapshotName);
waitForBlock(dataNodeName, repoName, TimeValue.timeValueSeconds(30L));
final AtomicBoolean blocked = new AtomicBoolean(true);
final TransportService transportService = internalCluster().getInstance(TransportService.class, otherDataNode);
transportService.addMessageListener(new TransportMessageListener() {
@Override
public void onRequestSent(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions finalOptions) {
if (blocked.get() && action.equals(SnapshotsService.UPDATE_SNAPSHOT_STATUS_ACTION_NAME)) {
throw new AssertionError("Node had no assigned shard snapshots so it shouldn't send out shard state updates");
}
}
});
logger.info("--> abort snapshot");
final ActionFuture<AcknowledgedResponse> deleteResponse = startDeleteSnapshot(repoName, snapshotName);
awaitClusterState(otherDataNode, state -> state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY).entries().stream().anyMatch(entry -> entry.state() == SnapshotsInProgress.State.ABORTED));
assertFalse("delete should not be able to finish until data node is unblocked", deleteResponse.isDone());
blocked.set(false);
unblockAllDataNodes(repoName);
assertAcked(deleteResponse.get());
assertThat(snapshotResponse.get().getSnapshotInfo().state(), is(SnapshotState.FAILED));
}
use of org.opensearch.action.ActionFuture in project OpenSearch by opensearch-project.
the class RelocationIT method testRelocateWhileContinuouslyIndexingAndWaitingForRefresh.
public void testRelocateWhileContinuouslyIndexingAndWaitingForRefresh() throws Exception {
logger.info("--> starting [node1] ...");
final String node1 = internalCluster().startNode();
logger.info("--> creating test index ...");
prepareCreate("test", // we
Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).put("index.refresh_interval", -1)).get();
logger.info("--> index 10 docs");
for (int i = 0; i < 10; i++) {
client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet();
}
logger.info("--> flush so we have an actual index");
client().admin().indices().prepareFlush().execute().actionGet();
logger.info("--> index more docs so we have something in the translog");
final List<ActionFuture<IndexResponse>> pendingIndexResponses = new ArrayList<>();
for (int i = 10; i < 20; i++) {
pendingIndexResponses.add(client().prepareIndex("test").setId(Integer.toString(i)).setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL).setSource("field", "value" + i).execute());
}
logger.info("--> start another node");
final String node2 = internalCluster().startNode();
ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").execute().actionGet();
assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
logger.info("--> relocate the shard from node1 to node2");
ActionFuture<ClusterRerouteResponse> relocationListener = client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, node1, node2)).execute();
logger.info("--> index 100 docs while relocating");
for (int i = 20; i < 120; i++) {
pendingIndexResponses.add(client().prepareIndex("test").setId(Integer.toString(i)).setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL).setSource("field", "value" + i).execute());
}
relocationListener.actionGet();
clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNoRelocatingShards(true).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
logger.info("--> verifying count");
assertBusy(() -> {
client().admin().indices().prepareRefresh().execute().actionGet();
assertTrue(pendingIndexResponses.stream().allMatch(ActionFuture::isDone));
}, 1, TimeUnit.MINUTES);
assertThat(client().prepareSearch("test").setSize(0).execute().actionGet().getHits().getTotalHits().value, equalTo(120L));
}
Aggregations