Search in sources :

Example 1 with ActionFuture

use of org.opensearch.action.ActionFuture in project OpenSearch by opensearch-project.

the class RethrottleTests method testCase.

private void testCase(AbstractBulkByScrollRequestBuilder<?, ?> request, String actionName) throws Exception {
    logger.info("Starting test for [{}] with [{}] slices", actionName, request.request().getSlices());
    /* Add ten documents per slice so most slices will have many documents to process, having to go to multiple batches.
         * We can't rely on the slices being evenly sized but 10 means we have some pretty big slices.
         */
    createIndex("test");
    int numSlices = expectedSlices(request.request().getSlices(), "test");
    List<IndexRequestBuilder> docs = new ArrayList<>();
    for (int i = 0; i < numSlices * 10; i++) {
        docs.add(client().prepareIndex("test").setId(Integer.toString(i)).setSource("foo", "bar"));
    }
    indexRandom(true, docs);
    // Start a request that will never finish unless we rethrottle it
    // Throttle "forever"
    request.setRequestsPerSecond(.000001f);
    // Make sure we use multiple batches
    request.source().setSize(1);
    ActionFuture<? extends BulkByScrollResponse> responseListener = request.execute();
    TaskGroup taskGroupToRethrottle = findTaskToRethrottle(actionName, numSlices);
    TaskId taskToRethrottle = taskGroupToRethrottle.getTaskInfo().getTaskId();
    if (numSlices == 1) {
        assertThat(taskGroupToRethrottle.getChildTasks(), empty());
    } else {
        // There should be a sane number of child tasks running
        assertThat(taskGroupToRethrottle.getChildTasks(), hasSize(allOf(greaterThanOrEqualTo(1), lessThanOrEqualTo(numSlices))));
        // Wait for all of the sub tasks to start (or finish, some might finish early, all that matters is that not all do)
        assertBusy(() -> {
            BulkByScrollTask.Status parent = (BulkByScrollTask.Status) client().admin().cluster().prepareGetTask(taskToRethrottle).get().getTask().getTask().getStatus();
            long finishedSubTasks = parent.getSliceStatuses().stream().filter(Objects::nonNull).count();
            ListTasksResponse list = client().admin().cluster().prepareListTasks().setParentTaskId(taskToRethrottle).get();
            list.rethrowFailures("subtasks");
            assertThat(finishedSubTasks + list.getTasks().size(), greaterThanOrEqualTo((long) numSlices));
            assertThat(list.getTasks().size(), greaterThan(0));
        });
    }
    // Now rethrottle it so it'll finish
    // No throttle or "very fast"
    float newRequestsPerSecond = randomBoolean() ? Float.POSITIVE_INFINITY : between(1, 1000) * 100000;
    ListTasksResponse rethrottleResponse = rethrottleTask(taskToRethrottle, newRequestsPerSecond);
    BulkByScrollTask.Status status = (BulkByScrollTask.Status) rethrottleResponse.getTasks().get(0).getStatus();
    // Now check the resulting requests per second.
    if (numSlices == 1) {
        // If there is a single slice it should match perfectly
        assertEquals(newRequestsPerSecond, status.getRequestsPerSecond(), Float.MIN_NORMAL);
    } else {
        /* Check that at least one slice was rethrottled. We won't always rethrottle all of them because they might have completed.
             * With multiple slices these numbers might not add up perfectly, thus the 1.01F. */
        long unfinished = status.getSliceStatuses().stream().filter(Objects::nonNull).filter(slice -> slice.getStatus().getTotal() > slice.getStatus().getSuccessfullyProcessed()).count();
        float maxExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ? Float.POSITIVE_INFINITY : (newRequestsPerSecond / unfinished) * 1.01F;
        float minExpectedSliceRequestsPerSecond = newRequestsPerSecond == Float.POSITIVE_INFINITY ? Float.POSITIVE_INFINITY : (newRequestsPerSecond / numSlices) * 0.99F;
        boolean oneSliceRethrottled = false;
        float totalRequestsPerSecond = 0;
        for (BulkByScrollTask.StatusOrException statusOrException : status.getSliceStatuses()) {
            if (statusOrException == null) {
                /* The slice can be null here because it was completed but hadn't reported its success back to the task when the
                     * rethrottle request came through. */
                continue;
            }
            assertNull(statusOrException.getException());
            BulkByScrollTask.Status slice = statusOrException.getStatus();
            if (slice.getTotal() > slice.getSuccessfullyProcessed()) {
                // This slice reports as not having completed so it should have been processed.
                assertThat(slice.getRequestsPerSecond(), both(greaterThanOrEqualTo(minExpectedSliceRequestsPerSecond)).and(lessThanOrEqualTo(maxExpectedSliceRequestsPerSecond)));
            }
            if (minExpectedSliceRequestsPerSecond <= slice.getRequestsPerSecond() && slice.getRequestsPerSecond() <= maxExpectedSliceRequestsPerSecond) {
                oneSliceRethrottled = true;
            }
            totalRequestsPerSecond += slice.getRequestsPerSecond();
        }
        assertTrue("At least one slice must be rethrottled", oneSliceRethrottled);
        /* Now assert that the parent request has the total requests per second. This is a much weaker assertion than that the parent
             * actually has the newRequestsPerSecond. For the most part it will. Sometimes it'll be greater because only unfinished requests
             * are rethrottled, the finished ones just keep whatever requests per second they had while they were running. But it might
             * also be less than newRequestsPerSecond because the newRequestsPerSecond is divided among running sub-requests and then the
             * requests are rethrottled. If one request finishes in between the division and the application of the new throttle then it
             * won't be rethrottled, thus only contributing its lower total. */
        assertEquals(totalRequestsPerSecond, status.getRequestsPerSecond(), totalRequestsPerSecond * 0.0001f);
    }
    // Now the response should come back quickly because we've rethrottled the request
    BulkByScrollResponse response = responseListener.get();
    // It'd be bad if the entire require completed in a single batch. The test wouldn't be testing anything.
    assertThat("Entire request completed in a single batch. This may invalidate the test as throttling is done between batches.", response.getBatches(), greaterThanOrEqualTo(numSlices));
}
Also used : IndexRequestBuilder(org.opensearch.action.index.IndexRequestBuilder) QueryBuilders(org.opensearch.index.query.QueryBuilders) TaskGroup(org.opensearch.action.admin.cluster.node.tasks.list.TaskGroup) Matchers.empty(org.hamcrest.Matchers.empty) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) Matchers.allOf(org.hamcrest.Matchers.allOf) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) ExceptionsHelper(org.opensearch.ExceptionsHelper) TaskId(org.opensearch.tasks.TaskId) OpenSearchException(org.opensearch.OpenSearchException) AtomicReference(java.util.concurrent.atomic.AtomicReference) ActionFuture(org.opensearch.action.ActionFuture) ArrayList(java.util.ArrayList) Matchers.both(org.hamcrest.Matchers.both) Objects(java.util.Objects) TimeUnit(java.util.concurrent.TimeUnit) ListTasksResponse(org.opensearch.action.admin.cluster.node.tasks.list.ListTasksResponse) List(java.util.List) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Matchers.lessThan(org.hamcrest.Matchers.lessThan) Matchers.hasSize(org.hamcrest.Matchers.hasSize) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) TaskId(org.opensearch.tasks.TaskId) ArrayList(java.util.ArrayList) ListTasksResponse(org.opensearch.action.admin.cluster.node.tasks.list.ListTasksResponse) IndexRequestBuilder(org.opensearch.action.index.IndexRequestBuilder) Objects(java.util.Objects) TaskGroup(org.opensearch.action.admin.cluster.node.tasks.list.TaskGroup)

Example 2 with ActionFuture

use of org.opensearch.action.ActionFuture in project OpenSearch by opensearch-project.

the class ConcurrentSnapshotsIT method testEquivalentDeletesAreDeduplicated.

public void testEquivalentDeletesAreDeduplicated() throws Exception {
    final String masterName = internalCluster().startMasterOnlyNode();
    internalCluster().startDataOnlyNode();
    final String repoName = "test-repo";
    createRepository(repoName, "mock");
    createIndexWithContent("index-test");
    createNSnapshots(repoName, randomIntBetween(1, 5));
    blockNodeOnAnyFiles(repoName, masterName);
    final int deletes = randomIntBetween(2, 10);
    final List<ActionFuture<AcknowledgedResponse>> deleteResponses = new ArrayList<>(deletes);
    for (int i = 0; i < deletes; ++i) {
        deleteResponses.add(client().admin().cluster().prepareDeleteSnapshot(repoName, "*").execute());
    }
    waitForBlock(masterName, repoName, TimeValue.timeValueSeconds(30L));
    awaitNDeletionsInProgress(1);
    for (ActionFuture<AcknowledgedResponse> deleteResponse : deleteResponses) {
        assertFalse(deleteResponse.isDone());
    }
    awaitNDeletionsInProgress(1);
    unblockNode(repoName, masterName);
    for (ActionFuture<AcknowledgedResponse> deleteResponse : deleteResponses) {
        assertAcked(deleteResponse.get());
    }
}
Also used : PlainActionFuture(org.opensearch.action.support.PlainActionFuture) ActionFuture(org.opensearch.action.ActionFuture) ArrayList(java.util.ArrayList) AcknowledgedResponse(org.opensearch.action.support.master.AcknowledgedResponse) Matchers.containsString(org.hamcrest.Matchers.containsString)

Example 3 with ActionFuture

use of org.opensearch.action.ActionFuture in project OpenSearch by opensearch-project.

the class CancellableTasksIT method testBanOnlyNodesWithOutstandingDescendantTasks.

public void testBanOnlyNodesWithOutstandingDescendantTasks() throws Exception {
    if (randomBoolean()) {
        internalCluster().startNodes(randomIntBetween(1, 3));
    }
    Set<DiscoveryNode> nodes = StreamSupport.stream(clusterService().state().nodes().spliterator(), false).collect(Collectors.toSet());
    final TestRequest rootRequest = generateTestRequest(nodes, 0, between(1, 4));
    ActionFuture<TestResponse> rootTaskFuture = client().execute(TransportTestAction.ACTION, rootRequest);
    Set<TestRequest> pendingRequests = allowPartialRequest(rootRequest);
    TaskId rootTaskId = getRootTaskId(rootRequest);
    ActionFuture<CancelTasksResponse> cancelFuture = client().admin().cluster().prepareCancelTasks().setTaskId(rootTaskId).waitForCompletion(true).execute();
    if (randomBoolean()) {
        List<TaskInfo> runningTasks = client().admin().cluster().prepareListTasks().setActions(TransportTestAction.ACTION.name()).setDetailed(true).get().getTasks();
        for (TaskInfo subTask : randomSubsetOf(runningTasks)) {
            client().admin().cluster().prepareCancelTasks().setTaskId(subTask.getTaskId()).waitForCompletion(false).get();
        }
    }
    assertBusy(() -> {
        for (DiscoveryNode node : nodes) {
            TaskManager taskManager = internalCluster().getInstance(TransportService.class, node.getName()).getTaskManager();
            Set<TaskId> expectedBans = new HashSet<>();
            for (TestRequest req : pendingRequests) {
                if (req.node.equals(node)) {
                    List<Task> childTasks = taskManager.getTasks().values().stream().filter(t -> t.getParentTaskId() != null && t.getDescription().equals(req.taskDescription())).collect(Collectors.toList());
                    assertThat(childTasks, hasSize(1));
                    CancellableTask childTask = (CancellableTask) childTasks.get(0);
                    assertTrue(childTask.isCancelled());
                    expectedBans.add(childTask.getParentTaskId());
                }
            }
            assertThat(taskManager.getBannedTaskIds(), equalTo(expectedBans));
        }
    }, 30, TimeUnit.SECONDS);
    allowEntireRequest(rootRequest);
    cancelFuture.actionGet();
    waitForRootTask(rootTaskFuture);
    ensureAllBansRemoved();
}
Also used : ActionPlugin(org.opensearch.plugins.ActionPlugin) AbstractRunnable(org.opensearch.common.util.concurrent.AbstractRunnable) ActionRequest(org.opensearch.action.ActionRequest) ConcurrentCollections(org.opensearch.common.util.concurrent.ConcurrentCollections) GroupedActionListener(org.opensearch.action.support.GroupedActionListener) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) LatchedActionListener(org.opensearch.action.LatchedActionListener) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) Map(java.util.Map) Inject(org.opensearch.common.inject.Inject) ActionListener(org.opensearch.action.ActionListener) ActionResponse(org.opensearch.action.ActionResponse) ActionType(org.opensearch.action.ActionType) CancelTasksResponse(org.opensearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse) CancellableTask(org.opensearch.tasks.CancellableTask) NodeClient(org.opensearch.client.node.NodeClient) Collection(java.util.Collection) ExceptionsHelper(org.opensearch.ExceptionsHelper) Set(java.util.Set) Task(org.opensearch.tasks.Task) TransportService(org.opensearch.transport.TransportService) Collectors(java.util.stream.Collectors) TaskManager(org.opensearch.tasks.TaskManager) Objects(java.util.Objects) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) ActionFilters(org.opensearch.action.support.ActionFilters) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Matchers.equalTo(org.hamcrest.Matchers.equalTo) TaskCancelledException(org.opensearch.tasks.TaskCancelledException) Matchers.anyOf(org.hamcrest.Matchers.anyOf) OpenSearchIntegTestCase(org.opensearch.test.OpenSearchIntegTestCase) Matchers.containsString(org.hamcrest.Matchers.containsString) TransportException(org.opensearch.transport.TransportException) HandledTransportAction(org.opensearch.action.support.HandledTransportAction) ActionRunnable(org.opensearch.action.ActionRunnable) ThreadPool(org.opensearch.threadpool.ThreadPool) StreamOutput(org.opensearch.common.io.stream.StreamOutput) ResourceNotFoundException(org.opensearch.ResourceNotFoundException) InternalTestCluster(org.opensearch.test.InternalTestCluster) ActionRequestValidationException(org.opensearch.action.ActionRequestValidationException) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ListTasksResponse(org.opensearch.action.admin.cluster.node.tasks.list.ListTasksResponse) Matchers.hasSize(org.hamcrest.Matchers.hasSize) StreamSupport(java.util.stream.StreamSupport) Before(org.junit.Before) StreamInput(org.opensearch.common.io.stream.StreamInput) Matchers.empty(org.hamcrest.Matchers.empty) SetOnce(org.apache.lucene.util.SetOnce) TaskId(org.opensearch.tasks.TaskId) TransportResponseHandler(org.opensearch.transport.TransportResponseHandler) IOException(java.io.IOException) Plugin(org.opensearch.plugins.Plugin) ActionFuture(org.opensearch.action.ActionFuture) TimeUnit(java.util.concurrent.TimeUnit) Sets(org.opensearch.common.util.set.Sets) TaskInfo(org.opensearch.tasks.TaskInfo) Collections(java.util.Collections) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) CancellableTask(org.opensearch.tasks.CancellableTask) Task(org.opensearch.tasks.Task) TaskId(org.opensearch.tasks.TaskId) CancelTasksResponse(org.opensearch.action.admin.cluster.node.tasks.cancel.CancelTasksResponse) TaskInfo(org.opensearch.tasks.TaskInfo) TaskManager(org.opensearch.tasks.TaskManager) CancellableTask(org.opensearch.tasks.CancellableTask) TransportService(org.opensearch.transport.TransportService) HashSet(java.util.HashSet)

Example 4 with ActionFuture

use of org.opensearch.action.ActionFuture in project OpenSearch by opensearch-project.

the class DedicatedClusterSnapshotRestoreIT method testAbortWaitsOnDataNode.

public void testAbortWaitsOnDataNode() throws Exception {
    internalCluster().startMasterOnlyNode();
    final String dataNodeName = internalCluster().startDataOnlyNode();
    final String indexName = "test-index";
    createIndex(indexName);
    index(indexName, "_doc", "some_id", "foo", "bar");
    final String otherDataNode = internalCluster().startDataOnlyNode();
    final String repoName = "test-repo";
    createRepository(repoName, "mock");
    blockAllDataNodes(repoName);
    final String snapshotName = "test-snap";
    final ActionFuture<CreateSnapshotResponse> snapshotResponse = startFullSnapshot(repoName, snapshotName);
    waitForBlock(dataNodeName, repoName, TimeValue.timeValueSeconds(30L));
    final AtomicBoolean blocked = new AtomicBoolean(true);
    final TransportService transportService = internalCluster().getInstance(TransportService.class, otherDataNode);
    transportService.addMessageListener(new TransportMessageListener() {

        @Override
        public void onRequestSent(DiscoveryNode node, long requestId, String action, TransportRequest request, TransportRequestOptions finalOptions) {
            if (blocked.get() && action.equals(SnapshotsService.UPDATE_SNAPSHOT_STATUS_ACTION_NAME)) {
                throw new AssertionError("Node had no assigned shard snapshots so it shouldn't send out shard state updates");
            }
        }
    });
    logger.info("--> abort snapshot");
    final ActionFuture<AcknowledgedResponse> deleteResponse = startDeleteSnapshot(repoName, snapshotName);
    awaitClusterState(otherDataNode, state -> state.custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY).entries().stream().anyMatch(entry -> entry.state() == SnapshotsInProgress.State.ABORTED));
    assertFalse("delete should not be able to finish until data node is unblocked", deleteResponse.isDone());
    blocked.set(false);
    unblockAllDataNodes(repoName);
    assertAcked(deleteResponse.get());
    assertThat(snapshotResponse.get().getSnapshotInfo().state(), is(SnapshotState.FAILED));
}
Also used : RepositoryMissingException(org.opensearch.repositories.RepositoryMissingException) Arrays(java.util.Arrays) Metadata(org.opensearch.cluster.metadata.Metadata) CheckedFunction(org.opensearch.common.CheckedFunction) Matchers.not(org.hamcrest.Matchers.not) ClusterScope(org.opensearch.test.OpenSearchIntegTestCase.ClusterScope) Version(org.opensearch.Version) SnapshotsStatusResponse(org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse) Strings(org.opensearch.common.Strings) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) RecoveryState(org.opensearch.indices.recovery.RecoveryState) SnapshotStatus(org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus) Matchers.nullValue(org.hamcrest.Matchers.nullValue) Path(java.nio.file.Path) EnumSet(java.util.EnumSet) OpenSearchAssertions.assertRequestBuilderThrows(org.opensearch.test.hamcrest.OpenSearchAssertions.assertRequestBuilderThrows) Client(org.opensearch.client.Client) TimeValue(org.opensearch.common.unit.TimeValue) NodeClient(org.opensearch.client.node.NodeClient) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) Matchers.allOf(org.hamcrest.Matchers.allOf) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) Settings(org.opensearch.common.settings.Settings) TestCustomMetadata(org.opensearch.test.TestCustomMetadata) Scope(org.opensearch.test.OpenSearchIntegTestCase.Scope) TransportService(org.opensearch.transport.TransportService) RetentionLeaseActions(org.opensearch.index.seqno.RetentionLeaseActions) UncheckedIOException(java.io.UncheckedIOException) FileVisitResult(java.nio.file.FileVisitResult) CountDownLatch(java.util.concurrent.CountDownLatch) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Matchers.containsString(org.hamcrest.Matchers.containsString) IndexNameExpressionResolver(org.opensearch.cluster.metadata.IndexNameExpressionResolver) Priority(org.opensearch.common.Priority) Node(org.opensearch.node.Node) ParseField(org.opensearch.common.ParseField) Writeable(org.opensearch.common.io.stream.Writeable) MockTransportService(org.opensearch.test.transport.MockTransportService) ArrayList(java.util.ArrayList) RestoreSnapshotResponse(org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse) ClusterState(org.opensearch.cluster.ClusterState) BusyMasterServiceDisruption(org.opensearch.test.disruption.BusyMasterServiceDisruption) Matchers.lessThan(org.hamcrest.Matchers.lessThan) Matchers.hasSize(org.hamcrest.Matchers.hasSize) AbstractRestChannel(org.opensearch.rest.AbstractRestChannel) Environment(org.opensearch.env.Environment) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) Files(java.nio.file.Files) RETAIN_ALL(org.opensearch.index.seqno.RetentionLeaseActions.RETAIN_ALL) IOException(java.io.IOException) TransportMessageListener(org.opensearch.transport.TransportMessageListener) Plugin(org.opensearch.plugins.Plugin) NamedXContentRegistry(org.opensearch.common.xcontent.NamedXContentRegistry) RetentionLeases(org.opensearch.index.seqno.RetentionLeases) ClusterService(org.opensearch.cluster.service.ClusterService) ShardStats(org.opensearch.action.admin.indices.stats.ShardStats) IndexRequestBuilder(org.opensearch.action.index.IndexRequestBuilder) BlobStoreRepository(org.opensearch.repositories.blobstore.BlobStoreRepository) NodeRoles.nonMasterNode(org.opensearch.test.NodeRoles.nonMasterNode) ThreadContext(org.opensearch.common.util.concurrent.ThreadContext) XContentParser(org.opensearch.common.xcontent.XContentParser) FakeRestRequest(org.opensearch.test.rest.FakeRestRequest) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) Locale(java.util.Locale) SimpleFileVisitor(java.nio.file.SimpleFileVisitor) Collection(java.util.Collection) RestStatus(org.opensearch.rest.RestStatus) ServiceDisruptionScheme(org.opensearch.test.disruption.ServiceDisruptionScheme) MockRepository(org.opensearch.snapshots.mockstore.MockRepository) List(java.util.List) Matchers.equalTo(org.hamcrest.Matchers.equalTo) SnapshotStats(org.opensearch.action.admin.cluster.snapshots.status.SnapshotStats) RestGetRepositoriesAction(org.opensearch.rest.action.admin.cluster.RestGetRepositoriesAction) SnapshotsInProgress(org.opensearch.cluster.SnapshotsInProgress) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) InternalTestCluster(org.opensearch.test.InternalTestCluster) AtomicReference(java.util.concurrent.atomic.AtomicReference) NamedWriteableRegistry(org.opensearch.common.io.stream.NamedWriteableRegistry) PeerRecoveryTargetService(org.opensearch.indices.recovery.PeerRecoveryTargetService) StreamInput(org.opensearch.common.io.stream.StreamInput) SettingsFilter(org.opensearch.common.settings.SettingsFilter) OpenSearchAssertions.assertAcked(org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked) CreateSnapshotResponse(org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse) Setting(org.opensearch.common.settings.Setting) TransportRequest(org.opensearch.transport.TransportRequest) RestRequest(org.opensearch.rest.RestRequest) IntHashSet(com.carrotsearch.hppc.IntHashSet) IntSet(com.carrotsearch.hppc.IntSet) RestClusterStateAction(org.opensearch.rest.action.admin.cluster.RestClusterStateAction) BasicFileAttributes(java.nio.file.attribute.BasicFileAttributes) OpenSearchAssertions.assertFutureThrows(org.opensearch.test.hamcrest.OpenSearchAssertions.assertFutureThrows) RestResponse(org.opensearch.rest.RestResponse) ActionFuture(org.opensearch.action.ActionFuture) ActiveShardCount(org.opensearch.action.support.ActiveShardCount) AcknowledgedResponse(org.opensearch.action.support.master.AcknowledgedResponse) ShardId(org.opensearch.index.shard.ShardId) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) Sets(org.opensearch.common.util.set.Sets) CreateSnapshotRequest(org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest) GetSnapshotsResponse(org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse) NamedDiff(org.opensearch.cluster.NamedDiff) Collections(java.util.Collections) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) TransportRequest(org.opensearch.transport.TransportRequest) AcknowledgedResponse(org.opensearch.action.support.master.AcknowledgedResponse) Matchers.containsString(org.hamcrest.Matchers.containsString) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CreateSnapshotResponse(org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse) TransportService(org.opensearch.transport.TransportService) MockTransportService(org.opensearch.test.transport.MockTransportService) TransportRequestOptions(org.opensearch.transport.TransportRequestOptions) TransportMessageListener(org.opensearch.transport.TransportMessageListener)

Example 5 with ActionFuture

use of org.opensearch.action.ActionFuture in project OpenSearch by opensearch-project.

the class RelocationIT method testRelocateWhileContinuouslyIndexingAndWaitingForRefresh.

public void testRelocateWhileContinuouslyIndexingAndWaitingForRefresh() throws Exception {
    logger.info("--> starting [node1] ...");
    final String node1 = internalCluster().startNode();
    logger.info("--> creating test index ...");
    prepareCreate("test", // we
    Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0).put("index.refresh_interval", -1)).get();
    logger.info("--> index 10 docs");
    for (int i = 0; i < 10; i++) {
        client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", "value" + i).execute().actionGet();
    }
    logger.info("--> flush so we have an actual index");
    client().admin().indices().prepareFlush().execute().actionGet();
    logger.info("--> index more docs so we have something in the translog");
    final List<ActionFuture<IndexResponse>> pendingIndexResponses = new ArrayList<>();
    for (int i = 10; i < 20; i++) {
        pendingIndexResponses.add(client().prepareIndex("test").setId(Integer.toString(i)).setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL).setSource("field", "value" + i).execute());
    }
    logger.info("--> start another node");
    final String node2 = internalCluster().startNode();
    ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNodes("2").execute().actionGet();
    assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
    logger.info("--> relocate the shard from node1 to node2");
    ActionFuture<ClusterRerouteResponse> relocationListener = client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, node1, node2)).execute();
    logger.info("--> index 100 docs while relocating");
    for (int i = 20; i < 120; i++) {
        pendingIndexResponses.add(client().prepareIndex("test").setId(Integer.toString(i)).setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL).setSource("field", "value" + i).execute());
    }
    relocationListener.actionGet();
    clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNoRelocatingShards(true).setTimeout(ACCEPTABLE_RELOCATION_TIME).execute().actionGet();
    assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
    logger.info("--> verifying count");
    assertBusy(() -> {
        client().admin().indices().prepareRefresh().execute().actionGet();
        assertTrue(pendingIndexResponses.stream().allMatch(ActionFuture::isDone));
    }, 1, TimeUnit.MINUTES);
    assertThat(client().prepareSearch("test").setSize(0).execute().actionGet().getHits().getTotalHits().value, equalTo(120L));
}
Also used : ClusterHealthResponse(org.opensearch.action.admin.cluster.health.ClusterHealthResponse) ActionFuture(org.opensearch.action.ActionFuture) ArrayList(java.util.ArrayList) MoveAllocationCommand(org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand) ClusterRerouteResponse(org.opensearch.action.admin.cluster.reroute.ClusterRerouteResponse)

Aggregations

ArrayList (java.util.ArrayList)8 ActionFuture (org.opensearch.action.ActionFuture)8 Matchers.containsString (org.hamcrest.Matchers.containsString)5 PlainActionFuture (org.opensearch.action.support.PlainActionFuture)5 AcknowledgedResponse (org.opensearch.action.support.master.AcknowledgedResponse)4 List (java.util.List)3 TimeUnit (java.util.concurrent.TimeUnit)3 Matchers.equalTo (org.hamcrest.Matchers.equalTo)3 Matchers.hasSize (org.hamcrest.Matchers.hasSize)3 IOException (java.io.IOException)2 Collection (java.util.Collection)2 Collections (java.util.Collections)2 Objects (java.util.Objects)2 CountDownLatch (java.util.concurrent.CountDownLatch)2 AtomicReference (java.util.concurrent.atomic.AtomicReference)2 Matchers.allOf (org.hamcrest.Matchers.allOf)2 Matchers.empty (org.hamcrest.Matchers.empty)2 Matchers.greaterThan (org.hamcrest.Matchers.greaterThan)2 Matchers.greaterThanOrEqualTo (org.hamcrest.Matchers.greaterThanOrEqualTo)2 Matchers.lessThan (org.hamcrest.Matchers.lessThan)2