Examples with ClusterRerouteRequest org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest used on opensource projects

Example 1 with ClusterRerouteRequest

use of org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest in project OpenSearch by opensearch-project.

the class FailedNodeRoutingTests method testRandomClusterPromotesNewestReplica.

public void testRandomClusterPromotesNewestReplica() throws InterruptedException {
    ThreadPool threadPool = new TestThreadPool(getClass().getName());
    ClusterStateChanges cluster = new ClusterStateChanges(xContentRegistry(), threadPool);
    ClusterState state = randomInitialClusterState();
    // randomly add nodes of mixed versions
    logger.info("--> adding random nodes");
    for (int i = 0; i < randomIntBetween(4, 8); i++) {
        DiscoveryNodes newNodes = DiscoveryNodes.builder(state.nodes()).add(createNode()).build();
        state = ClusterState.builder(state).nodes(newNodes).build();
        // always reroute after adding node
        state = cluster.reroute(state, new ClusterRerouteRequest());
    }
    // Log the node versions (for debugging if necessary)
    for (ObjectCursor<DiscoveryNode> cursor : state.nodes().getDataNodes().values()) {
        Version nodeVer = cursor.value.getVersion();
        logger.info("--> node [{}] has version [{}]", cursor.value.getId(), nodeVer);
    }
    // randomly create some indices
    logger.info("--> creating some indices");
    for (int i = 0; i < randomIntBetween(2, 5); i++) {
        String name = "index_" + randomAlphaOfLength(8).toLowerCase(Locale.ROOT);
        Settings.Builder settingsBuilder = Settings.builder().put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 4)).put(SETTING_NUMBER_OF_REPLICAS, randomIntBetween(2, 4));
        CreateIndexRequest request = new CreateIndexRequest(name, settingsBuilder.build()).waitForActiveShards(ActiveShardCount.NONE);
        state = cluster.createIndex(state, request);
        assertTrue(state.metadata().hasIndex(name));
    }
    logger.info("--> starting shards");
    state = cluster.applyStartedShards(state, state.getRoutingNodes().shardsWithState(INITIALIZING));
    logger.info("--> starting replicas a random number of times");
    for (int i = 0; i < randomIntBetween(1, 10); i++) {
        state = cluster.applyStartedShards(state, state.getRoutingNodes().shardsWithState(INITIALIZING));
    }
    boolean keepGoing = true;
    while (keepGoing) {
        List<ShardRouting> primaries = state.getRoutingNodes().shardsWithState(STARTED).stream().filter(ShardRouting::primary).collect(Collectors.toList());
        // Pick a random subset of primaries to fail
        List<FailedShard> shardsToFail = new ArrayList<>();
        List<ShardRouting> failedPrimaries = randomSubsetOf(primaries);
        failedPrimaries.stream().forEach(sr -> {
            shardsToFail.add(new FailedShard(randomFrom(sr), "failed primary", new Exception(), randomBoolean()));
        });
        logger.info("--> state before failing shards: {}", state);
        state = cluster.applyFailedShards(state, shardsToFail);
        final ClusterState compareState = state;
        failedPrimaries.forEach(shardRouting -> {
            logger.info("--> verifying version for {}", shardRouting);
            ShardRouting newPrimary = compareState.routingTable().index(shardRouting.index()).shard(shardRouting.id()).primaryShard();
            Version newPrimaryVersion = getNodeVersion(newPrimary, compareState);
            logger.info("--> new primary is on version {}: {}", newPrimaryVersion, newPrimary);
            compareState.routingTable().shardRoutingTable(newPrimary.shardId()).shardsWithState(STARTED).stream().forEach(sr -> {
                Version candidateVer = getNodeVersion(sr, compareState);
                if (candidateVer != null) {
                    logger.info("--> candidate on {} node; shard routing: {}", candidateVer, sr);
                    assertTrue("candidate was not on the newest version, new primary is on " + newPrimaryVersion + " and there is a candidate on " + candidateVer, candidateVer.onOrBefore(newPrimaryVersion));
                }
            });
        });
        keepGoing = randomBoolean();
    }
    terminate(threadPool);
}

Example 2 with ClusterRerouteRequest

use of org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest in project OpenSearch by opensearch-project.

the class IndicesClusterStateServiceRandomUpdatesTests method randomlyUpdateClusterState.

public ClusterState randomlyUpdateClusterState(ClusterState state, Map<DiscoveryNode, IndicesClusterStateService> clusterStateServiceMap, Supplier<MockIndicesService> indicesServiceSupplier) {
    // randomly remove no_master blocks
    if (randomBoolean() && state.blocks().hasGlobalBlockWithId(NoMasterBlockService.NO_MASTER_BLOCK_ID)) {
        state = ClusterState.builder(state).blocks(ClusterBlocks.builder().blocks(state.blocks()).removeGlobalBlock(NoMasterBlockService.NO_MASTER_BLOCK_ID)).build();
    }
    // randomly add no_master blocks
    if (rarely() && state.blocks().hasGlobalBlockWithId(NoMasterBlockService.NO_MASTER_BLOCK_ID) == false) {
        ClusterBlock block = randomBoolean() ? NoMasterBlockService.NO_MASTER_BLOCK_ALL : NoMasterBlockService.NO_MASTER_BLOCK_WRITES;
        state = ClusterState.builder(state).blocks(ClusterBlocks.builder().blocks(state.blocks()).addGlobalBlock(block)).build();
    }
    // if no_master block is in place, make no other cluster state changes
    if (state.blocks().hasGlobalBlockWithId(NoMasterBlockService.NO_MASTER_BLOCK_ID)) {
        return state;
    }
    // randomly create new indices (until we have 200 max)
    for (int i = 0; i < randomInt(5); i++) {
        if (state.metadata().indices().size() > 200) {
            break;
        }
        String name = "index_" + randomAlphaOfLength(15).toLowerCase(Locale.ROOT);
        Settings.Builder settingsBuilder = Settings.builder().put(SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 3));
        if (randomBoolean()) {
            int min = randomInt(2);
            int max = min + randomInt(3);
            settingsBuilder.put(SETTING_AUTO_EXPAND_REPLICAS, randomBoolean() ? min + "-" + max : min + "-all");
        } else {
            settingsBuilder.put(SETTING_NUMBER_OF_REPLICAS, randomInt(2));
        }
        CreateIndexRequest request = new CreateIndexRequest(name, settingsBuilder.build()).waitForActiveShards(ActiveShardCount.NONE);
        state = cluster.createIndex(state, request);
        assertTrue(state.metadata().hasIndex(name));
    }
    // randomly delete indices
    Set<String> indicesToDelete = new HashSet<>();
    int numberOfIndicesToDelete = randomInt(Math.min(2, state.metadata().indices().size()));
    for (String index : randomSubsetOf(numberOfIndicesToDelete, state.metadata().indices().keys().toArray(String.class))) {
        indicesToDelete.add(state.metadata().index(index).getIndex().getName());
    }
    if (indicesToDelete.isEmpty() == false) {
        DeleteIndexRequest deleteRequest = new DeleteIndexRequest(indicesToDelete.toArray(new String[indicesToDelete.size()]));
        state = cluster.deleteIndices(state, deleteRequest);
        for (String index : indicesToDelete) {
            assertFalse(state.metadata().hasIndex(index));
        }
    }
    // randomly close indices
    int numberOfIndicesToClose = randomInt(Math.min(1, state.metadata().indices().size()));
    for (String index : randomSubsetOf(numberOfIndicesToClose, state.metadata().indices().keys().toArray(String.class))) {
        CloseIndexRequest closeIndexRequest = new CloseIndexRequest(state.metadata().index(index).getIndex().getName());
        state = cluster.closeIndices(state, closeIndexRequest);
    }
    // randomly open indices
    int numberOfIndicesToOpen = randomInt(Math.min(1, state.metadata().indices().size()));
    for (String index : randomSubsetOf(numberOfIndicesToOpen, state.metadata().indices().keys().toArray(String.class))) {
        OpenIndexRequest openIndexRequest = new OpenIndexRequest(state.metadata().index(index).getIndex().getName());
        state = cluster.openIndices(state, openIndexRequest);
    }
    // randomly update settings
    Set<String> indicesToUpdate = new HashSet<>();
    boolean containsClosedIndex = false;
    int numberOfIndicesToUpdate = randomInt(Math.min(2, state.metadata().indices().size()));
    for (String index : randomSubsetOf(numberOfIndicesToUpdate, state.metadata().indices().keys().toArray(String.class))) {
        indicesToUpdate.add(state.metadata().index(index).getIndex().getName());
        if (state.metadata().index(index).getState() == IndexMetadata.State.CLOSE) {
            containsClosedIndex = true;
        }
    }
    if (indicesToUpdate.isEmpty() == false) {
        UpdateSettingsRequest updateSettingsRequest = new UpdateSettingsRequest(indicesToUpdate.toArray(new String[indicesToUpdate.size()]));
        Settings.Builder settings = Settings.builder();
        if (containsClosedIndex == false) {
            settings.put(SETTING_NUMBER_OF_REPLICAS, randomInt(2));
        }
        settings.put("index.refresh_interval", randomIntBetween(1, 5) + "s");
        updateSettingsRequest.settings(settings.build());
        state = cluster.updateSettings(state, updateSettingsRequest);
    }
    // randomly reroute
    if (rarely()) {
        state = cluster.reroute(state, new ClusterRerouteRequest());
    }
    // randomly start and fail allocated shards
    final Map<ShardRouting, Long> startedShards = new HashMap<>();
    List<FailedShard> failedShards = new ArrayList<>();
    for (DiscoveryNode node : state.nodes()) {
        IndicesClusterStateService indicesClusterStateService = clusterStateServiceMap.get(node);
        MockIndicesService indicesService = (MockIndicesService) indicesClusterStateService.indicesService;
        for (MockIndexService indexService : indicesService) {
            for (MockIndexShard indexShard : indexService) {
                ShardRouting persistedShardRouting = indexShard.routingEntry();
                if (persistedShardRouting.initializing() && randomBoolean()) {
                    startedShards.put(persistedShardRouting, indexShard.term());
                } else if (rarely()) {
                    failedShards.add(new FailedShard(persistedShardRouting, "fake shard failure", new Exception(), randomBoolean()));
                }
            }
        }
    }
    state = cluster.applyFailedShards(state, failedShards);
    state = cluster.applyStartedShards(state, startedShards);
    // randomly add and remove nodes (except current master)
    if (rarely()) {
        if (randomBoolean()) {
            // add node
            if (state.nodes().getSize() < 10) {
                state = cluster.addNodes(state, Collections.singletonList(createNode()));
                updateNodes(state, clusterStateServiceMap, indicesServiceSupplier);
            }
        } else {
            // remove node
            if (state.nodes().getDataNodes().size() > 3) {
                DiscoveryNode discoveryNode = randomFrom(state.nodes().getNodes().values().toArray(DiscoveryNode.class));
                if (discoveryNode.equals(state.nodes().getMasterNode()) == false) {
                    state = cluster.removeNodes(state, Collections.singletonList(discoveryNode));
                    updateNodes(state, clusterStateServiceMap, indicesServiceSupplier);
                }
                if (randomBoolean()) {
                    // and add it back
                    state = cluster.addNodes(state, Collections.singletonList(discoveryNode));
                    updateNodes(state, clusterStateServiceMap, indicesServiceSupplier);
                }
            }
        }
    }
    return state;
}

Example 3 with ClusterRerouteRequest

use of org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest in project OpenSearch by opensearch-project.

the class AutoExpandReplicasTests method testAutoExpandWhenNodeLeavesAndPossiblyRejoins.

/**
 * Checks that when nodes leave the cluster that the auto-expand-replica functionality only triggers after failing the shards on
 * the removed nodes. This ensures that active shards on other live nodes are not failed if the primary resided on a now dead node.
 * Instead, one of the replicas on the live nodes first gets promoted to primary, and the auto-expansion (removing replicas) only
 * triggers in a follow-up step.
 */
public void testAutoExpandWhenNodeLeavesAndPossiblyRejoins() throws InterruptedException {
    final ThreadPool threadPool = new TestThreadPool(getClass().getName());
    final ClusterStateChanges cluster = new ClusterStateChanges(xContentRegistry(), threadPool);
    try {
        List<DiscoveryNode> allNodes = new ArrayList<>();
        // local node is the master
        DiscoveryNode localNode = createNode(DiscoveryNodeRole.MASTER_ROLE);
        allNodes.add(localNode);
        int numDataNodes = randomIntBetween(3, 5);
        List<DiscoveryNode> dataNodes = new ArrayList<>(numDataNodes);
        for (int i = 0; i < numDataNodes; i++) {
            dataNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE));
        }
        allNodes.addAll(dataNodes);
        ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[0]));
        CreateIndexRequest request = new CreateIndexRequest("index", Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_AUTO_EXPAND_REPLICAS, "0-all").build()).waitForActiveShards(ActiveShardCount.NONE);
        state = cluster.createIndex(state, request);
        assertTrue(state.metadata().hasIndex("index"));
        while (state.routingTable().index("index").shard(0).allShardsStarted() == false) {
            logger.info(state);
            state = cluster.applyStartedShards(state, state.routingTable().index("index").shard(0).shardsWithState(ShardRoutingState.INITIALIZING));
            state = cluster.reroute(state, new ClusterRerouteRequest());
        }
        IndexShardRoutingTable preTable = state.routingTable().index("index").shard(0);
        final Set<String> unchangedNodeIds;
        final IndexShardRoutingTable postTable;
        if (randomBoolean()) {
            // simulate node removal
            List<DiscoveryNode> nodesToRemove = randomSubsetOf(2, dataNodes);
            unchangedNodeIds = dataNodes.stream().filter(n -> nodesToRemove.contains(n) == false).map(DiscoveryNode::getId).collect(Collectors.toSet());
            state = cluster.removeNodes(state, nodesToRemove);
            postTable = state.routingTable().index("index").shard(0);
            assertTrue("not all shards started in " + state.toString(), postTable.allShardsStarted());
            assertThat(postTable.toString(), postTable.getAllAllocationIds(), everyItem(is(in(preTable.getAllAllocationIds()))));
        } else {
            // fake an election where conflicting nodes are removed and readded
            state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).masterNodeId(null).build()).build();
            List<DiscoveryNode> conflictingNodes = randomSubsetOf(2, dataNodes);
            unchangedNodeIds = dataNodes.stream().filter(n -> conflictingNodes.contains(n) == false).map(DiscoveryNode::getId).collect(Collectors.toSet());
            List<DiscoveryNode> nodesToAdd = conflictingNodes.stream().map(n -> new DiscoveryNode(n.getName(), n.getId(), buildNewFakeTransportAddress(), n.getAttributes(), n.getRoles(), n.getVersion())).collect(Collectors.toList());
            if (randomBoolean()) {
                nodesToAdd.add(createNode(DiscoveryNodeRole.DATA_ROLE));
            }
            state = cluster.joinNodesAndBecomeMaster(state, nodesToAdd);
            postTable = state.routingTable().index("index").shard(0);
        }
        Set<String> unchangedAllocationIds = preTable.getShards().stream().filter(shr -> unchangedNodeIds.contains(shr.currentNodeId())).map(shr -> shr.allocationId().getId()).collect(Collectors.toSet());
        assertThat(postTable.toString(), unchangedAllocationIds, everyItem(is(in(postTable.getAllAllocationIds()))));
        postTable.getShards().forEach(shardRouting -> {
            if (shardRouting.assignedToNode() && unchangedAllocationIds.contains(shardRouting.allocationId().getId())) {
                assertTrue("Shard should be active: " + shardRouting, shardRouting.active());
            }
        });
    } finally {
        terminate(threadPool);
    }
}

Example 4 with ClusterRerouteRequest

use of org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest in project OpenSearch by opensearch-project.

the class SnapshotResiliencyTests method testSnapshotPrimaryRelocations.

/**
 * Simulates concurrent restarts of data and master nodes as well as relocating a primary shard, while starting and subsequently
 * deleting a snapshot.
 */
public void testSnapshotPrimaryRelocations() {
    final int masterNodeCount = randomFrom(1, 3, 5);
    setupTestCluster(masterNodeCount, randomIntBetween(2, 5));
    String repoName = "repo";
    String snapshotName = "snapshot";
    final String index = "test";
    final int shards = randomIntBetween(1, 5);
    final TestClusterNodes.TestClusterNode masterNode = testClusterNodes.currentMaster(testClusterNodes.nodes.values().iterator().next().clusterService.state());
    final AtomicBoolean createdSnapshot = new AtomicBoolean();
    final AdminClient masterAdminClient = masterNode.client.admin();
    final StepListener<ClusterStateResponse> clusterStateResponseStepListener = new StepListener<>();
    continueOrDie(createRepoAndIndex(repoName, index, shards), createIndexResponse -> client().admin().cluster().state(new ClusterStateRequest(), clusterStateResponseStepListener));
    continueOrDie(clusterStateResponseStepListener, clusterStateResponse -> {
        final ShardRouting shardToRelocate = clusterStateResponse.getState().routingTable().allShards(index).get(0);
        final TestClusterNodes.TestClusterNode currentPrimaryNode = testClusterNodes.nodeById(shardToRelocate.currentNodeId());
        final TestClusterNodes.TestClusterNode otherNode = testClusterNodes.randomDataNodeSafe(currentPrimaryNode.node.getName());
        scheduleNow(() -> testClusterNodes.stopNode(currentPrimaryNode));
        scheduleNow(new Runnable() {

            @Override
            public void run() {
                final StepListener<ClusterStateResponse> updatedClusterStateResponseStepListener = new StepListener<>();
                masterAdminClient.cluster().state(new ClusterStateRequest(), updatedClusterStateResponseStepListener);
                continueOrDie(updatedClusterStateResponseStepListener, updatedClusterState -> {
                    final ShardRouting shardRouting = updatedClusterState.getState().routingTable().shardRoutingTable(shardToRelocate.shardId()).primaryShard();
                    if (shardRouting.unassigned() && shardRouting.unassignedInfo().getReason() == UnassignedInfo.Reason.NODE_LEFT) {
                        if (masterNodeCount > 1) {
                            scheduleNow(() -> testClusterNodes.stopNode(masterNode));
                        }
                        testClusterNodes.randomDataNodeSafe().client.admin().cluster().prepareCreateSnapshot(repoName, snapshotName).execute(ActionListener.wrap(() -> {
                            createdSnapshot.set(true);
                            testClusterNodes.randomDataNodeSafe().client.admin().cluster().deleteSnapshot(new DeleteSnapshotRequest(repoName, snapshotName), noopListener());
                        }));
                        scheduleNow(() -> testClusterNodes.randomMasterNodeSafe().client.admin().cluster().reroute(new ClusterRerouteRequest().add(new AllocateEmptyPrimaryAllocationCommand(index, shardRouting.shardId().id(), otherNode.node.getName(), true)), noopListener()));
                    } else {
                        scheduleSoon(this);
                    }
                });
            }
        });
    });
    runUntil(() -> testClusterNodes.randomMasterNode().map(master -> {
        if (createdSnapshot.get() == false) {
            return false;
        }
        return master.clusterService.state().custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY).entries().isEmpty();
    }).orElse(false), TimeUnit.MINUTES.toMillis(1L));
    clearDisruptionsAndAwaitSync();
    assertTrue(createdSnapshot.get());
    assertThat(testClusterNodes.randomDataNodeSafe().clusterService.state().custom(SnapshotsInProgress.TYPE, SnapshotsInProgress.EMPTY).entries(), empty());
    final Repository repository = testClusterNodes.randomMasterNodeSafe().repositoriesService.repository(repoName);
    Collection<SnapshotId> snapshotIds = getRepositoryData(repository).getSnapshotIds();
    assertThat(snapshotIds, either(hasSize(1)).or(hasSize(0)));
}

Example 5 with ClusterRerouteRequest

use of org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest in project OpenSearch by opensearch-project.

the class CloseWhileRelocatingShardsIT method testCloseWhileRelocatingShards.

public void testCloseWhileRelocatingShards() throws Exception {
    final String[] indices = new String[randomIntBetween(3, 5)];
    final Map<String, Long> docsPerIndex = new HashMap<>();
    final Map<String, BackgroundIndexer> indexers = new HashMap<>();
    for (int i = 0; i < indices.length; i++) {
        final String indexName = "index-" + i;
        int nbDocs = 0;
        switch(i) {
            case 0:
                logger.debug("creating empty index {}", indexName);
                createIndex(indexName);
                break;
            case 1:
                nbDocs = scaledRandomIntBetween(1, 100);
                logger.debug("creating index {} with {} documents", indexName, nbDocs);
                createIndex(indexName);
                indexRandom(randomBoolean(), IntStream.range(0, nbDocs).mapToObj(n -> client().prepareIndex(indexName).setSource("num", n)).collect(Collectors.toList()));
                break;
            default:
                logger.debug("creating index {} with background indexing", indexName);
                final BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), -1, 1);
                indexers.put(indexName, indexer);
                indexer.setFailureAssertion(t -> assertException(t, indexName));
                waitForDocs(1, indexer);
        }
        docsPerIndex.put(indexName, (long) nbDocs);
        indices[i] = indexName;
    }
    ensureGreen(TimeValue.timeValueSeconds(60L), indices);
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), Rebalance.NONE.toString())));
    final String targetNode = internalCluster().startDataOnlyNode();
    // wait for the master to finish processing join.
    ensureClusterSizeConsistency();
    try {
        final ClusterService clusterService = internalCluster().getInstance(ClusterService.class, internalCluster().getMasterName());
        final ClusterState state = clusterService.state();
        final CountDownLatch latch = new CountDownLatch(indices.length);
        final CountDownLatch release = new CountDownLatch(indices.length);
        // relocate one shard for every index to be closed
        final AllocationCommands commands = new AllocationCommands();
        for (final String index : indices) {
            final NumShards numShards = getNumShards(index);
            final int shardId = numShards.numPrimaries == 1 ? 0 : randomIntBetween(0, numShards.numPrimaries - 1);
            final IndexRoutingTable indexRoutingTable = state.routingTable().index(index);
            final ShardRouting primary = indexRoutingTable.shard(shardId).primaryShard();
            assertTrue(primary.started());
            String currentNodeId = primary.currentNodeId();
            if (numShards.numReplicas > 0) {
                final ShardRouting replica = indexRoutingTable.shard(shardId).replicaShards().iterator().next();
                assertTrue(replica.started());
                if (randomBoolean()) {
                    currentNodeId = replica.currentNodeId();
                }
            }
            commands.add(new MoveAllocationCommand(index, shardId, state.nodes().resolveNode(currentNodeId).getName(), targetNode));
        }
        // Build the list of shards for which recoveries will be blocked
        final Set<ShardId> blockedShards = commands.commands().stream().map(c -> (MoveAllocationCommand) c).map(c -> new ShardId(clusterService.state().metadata().index(c.index()).getIndex(), c.shardId())).collect(Collectors.toSet());
        assertThat(blockedShards, hasSize(indices.length));
        final Set<String> acknowledgedCloses = ConcurrentCollections.newConcurrentSet();
        final Set<String> interruptedRecoveries = ConcurrentCollections.newConcurrentSet();
        // Create a SendRequestBehavior that will block outgoing start recovery request
        final StubbableTransport.SendRequestBehavior sendBehavior = (connection, requestId, action, request, options) -> {
            if (PeerRecoverySourceService.Actions.START_RECOVERY.equals(action)) {
                final StartRecoveryRequest startRecoveryRequest = ((StartRecoveryRequest) request);
                if (blockedShards.contains(startRecoveryRequest.shardId())) {
                    logger.debug("blocking recovery of shard {}", startRecoveryRequest.shardId());
                    latch.countDown();
                    try {
                        release.await();
                        logger.debug("releasing recovery of shard {}", startRecoveryRequest.shardId());
                    } catch (final InterruptedException e) {
                        logger.warn(() -> new ParameterizedMessage("exception when releasing recovery of shard {}", startRecoveryRequest.shardId()), e);
                        interruptedRecoveries.add(startRecoveryRequest.shardId().getIndexName());
                        Thread.currentThread().interrupt();
                        return;
                    }
                }
            }
            connection.sendRequest(requestId, action, request, options);
        };
        final MockTransportService targetTransportService = (MockTransportService) internalCluster().getInstance(TransportService.class, targetNode);
        for (DiscoveryNode node : state.getNodes()) {
            if (node.isDataNode() && node.getName().equals(targetNode) == false) {
                final TransportService sourceTransportService = internalCluster().getInstance(TransportService.class, node.getName());
                targetTransportService.addSendBehavior(sourceTransportService, sendBehavior);
            }
        }
        assertAcked(client().admin().cluster().reroute(new ClusterRerouteRequest().commands(commands)).get());
        // start index closing threads
        final List<Thread> threads = new ArrayList<>();
        for (final String indexToClose : indices) {
            final Thread thread = new Thread(() -> {
                try {
                    latch.await();
                } catch (InterruptedException e) {
                    throw new AssertionError(e);
                } finally {
                    release.countDown();
                }
                // Closing is not always acknowledged when shards are relocating: this is the case when the target shard is initializing
                // or is catching up operations. In these cases the TransportVerifyShardBeforeCloseAction will detect that the global
                // and max sequence number don't match and will not ack the close.
                AcknowledgedResponse closeResponse = client().admin().indices().prepareClose(indexToClose).get();
                if (closeResponse.isAcknowledged()) {
                    assertTrue("Index closing should not be acknowledged twice", acknowledgedCloses.add(indexToClose));
                }
            });
            threads.add(thread);
            thread.start();
        }
        latch.countDown();
        for (Thread thread : threads) {
            thread.join();
        }
        // stop indexers first without waiting for stop to not redundantly index on some while waiting for another one to stop
        for (BackgroundIndexer indexer : indexers.values()) {
            indexer.stop();
        }
        for (Map.Entry<String, BackgroundIndexer> entry : indexers.entrySet()) {
            final BackgroundIndexer indexer = entry.getValue();
            indexer.awaitStopped();
            final String indexName = entry.getKey();
            docsPerIndex.computeIfPresent(indexName, (key, value) -> value + indexer.totalIndexedDocs());
        }
        for (String index : indices) {
            if (acknowledgedCloses.contains(index)) {
                assertIndexIsClosed(index);
            } else {
                assertIndexIsOpened(index);
            }
        }
        targetTransportService.clearAllRules();
        // If a shard recovery has been interrupted, we expect its index to be closed
        interruptedRecoveries.forEach(CloseIndexIT::assertIndexIsClosed);
        assertThat("Consider that the test failed if no indices were successfully closed", acknowledgedCloses.size(), greaterThan(0));
        assertAcked(client().admin().indices().prepareOpen("index-*"));
        ensureGreen(indices);
        for (String index : acknowledgedCloses) {
            long docsCount = client().prepareSearch(index).setSize(0).setTrackTotalHits(true).get().getHits().getTotalHits().value;
            assertEquals("Expected " + docsPerIndex.get(index) + " docs in index " + index + " but got " + docsCount + " (close acknowledged=" + acknowledgedCloses.contains(index) + ")", (long) docsPerIndex.get(index), docsCount);
        }
    } finally {
        assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().putNull(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey())));
    }
}