Examples with MoveAllocationCommand org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand used on opensource projects

Example 26 with MoveAllocationCommand

use of org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand in project elasticsearch by elastic.

the class AckIT method getAllocationCommand.

private MoveAllocationCommand getAllocationCommand() {
    String fromNodeId = null;
    String toNodeId = null;
    ShardRouting shardToBeMoved = null;
    ClusterStateResponse clusterStateResponse = client().admin().cluster().prepareState().get();
    for (RoutingNode routingNode : clusterStateResponse.getState().getRoutingNodes()) {
        if (routingNode.node().isDataNode()) {
            if (fromNodeId == null && routingNode.numberOfOwningShards() > 0) {
                fromNodeId = routingNode.nodeId();
                shardToBeMoved = routingNode.copyShards().get(randomInt(routingNode.size() - 1));
            } else {
                toNodeId = routingNode.nodeId();
            }
            if (toNodeId != null && fromNodeId != null) {
                break;
            }
        }
    }
    assertNotNull(fromNodeId);
    assertNotNull(toNodeId);
    assertNotNull(shardToBeMoved);
    logger.info("==> going to move shard [{}] from [{}] to [{}]", shardToBeMoved, fromNodeId, toNodeId);
    return new MoveAllocationCommand(shardToBeMoved.getIndexName(), shardToBeMoved.id(), fromNodeId, toNodeId);
}

Example 27 with MoveAllocationCommand

use of org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand in project elasticsearch by elastic.

the class IndexPrimaryRelocationIT method testPrimaryRelocationWhileIndexing.

@TestLogging("_root:DEBUG,org.elasticsearch.action.bulk:TRACE,org.elasticsearch.index.shard:TRACE,org.elasticsearch.cluster.service:TRACE")
public void testPrimaryRelocationWhileIndexing() throws Exception {
    internalCluster().ensureAtLeastNumDataNodes(randomIntBetween(2, 3));
    client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", 1).put("index.number_of_replicas", 0)).addMapping("type", "field", "type=text").get();
    ensureGreen("test");
    AtomicInteger numAutoGenDocs = new AtomicInteger();
    final AtomicBoolean finished = new AtomicBoolean(false);
    Thread indexingThread = new Thread() {

        @Override
        public void run() {
            while (finished.get() == false) {
                IndexResponse indexResponse = client().prepareIndex("test", "type", "id").setSource("field", "value").get();
                assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
                DeleteResponse deleteResponse = client().prepareDelete("test", "type", "id").get();
                assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
                client().prepareIndex("test", "type").setSource("auto", true).get();
                numAutoGenDocs.incrementAndGet();
            }
        }
    };
    indexingThread.start();
    ClusterState initialState = client().admin().cluster().prepareState().get().getState();
    DiscoveryNode[] dataNodes = initialState.getNodes().getDataNodes().values().toArray(DiscoveryNode.class);
    DiscoveryNode relocationSource = initialState.getNodes().getDataNodes().get(initialState.getRoutingTable().shardRoutingTable("test", 0).primaryShard().currentNodeId());
    for (int i = 0; i < RELOCATION_COUNT; i++) {
        DiscoveryNode relocationTarget = randomFrom(dataNodes);
        while (relocationTarget.equals(relocationSource)) {
            relocationTarget = randomFrom(dataNodes);
        }
        logger.info("--> [iteration {}] relocating from {} to {} ", i, relocationSource.getName(), relocationTarget.getName());
        client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, relocationSource.getId(), relocationTarget.getId())).execute().actionGet();
        ClusterHealthResponse clusterHealthResponse = client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForNoRelocatingShards(true).execute().actionGet();
        assertThat(clusterHealthResponse.isTimedOut(), equalTo(false));
        logger.info("--> [iteration {}] relocation complete", i);
        relocationSource = relocationTarget;
        if (indexingThread.isAlive() == false) {
            // indexing process aborted early, no need for more relocations as test has already failed
            break;
        }
        if (i > 0 && i % 5 == 0) {
            logger.info("--> [iteration {}] flushing index", i);
            client().admin().indices().prepareFlush("test").get();
        }
    }
    finished.set(true);
    indexingThread.join();
    refresh("test");
    ElasticsearchAssertions.assertHitCount(client().prepareSearch("test").get(), numAutoGenDocs.get());
    ElasticsearchAssertions.assertHitCount(// extra paranoia ;)
    client().prepareSearch("test").setQuery(QueryBuilders.termQuery("auto", true)).get(), numAutoGenDocs.get());
}

Example 28 with MoveAllocationCommand

use of org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand in project elasticsearch by elastic.

the class IndexRecoveryIT method testRerouteRecovery.

public void testRerouteRecovery() throws Exception {
    logger.info("--> start node A");
    final String nodeA = internalCluster().startNode();
    logger.info("--> create index on node: {}", nodeA);
    ByteSizeValue shardSize = createAndPopulateIndex(INDEX_NAME, 1, SHARD_COUNT, REPLICA_COUNT).getShards()[0].getStats().getStore().size();
    logger.info("--> start node B");
    final String nodeB = internalCluster().startNode();
    ensureGreen();
    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);
    logger.info("--> move shard from: {} to: {}", nodeA, nodeB);
    client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(INDEX_NAME, 0, nodeA, nodeB)).execute().actionGet().getState();
    logger.info("--> waiting for recovery to start both on source and target");
    final Index index = resolveIndex(INDEX_NAME);
    assertBusy(new Runnable() {

        @Override
        public void run() {
            IndicesService indicesService = internalCluster().getInstance(IndicesService.class, nodeA);
            assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsSource(), equalTo(1));
            indicesService = internalCluster().getInstance(IndicesService.class, nodeB);
            assertThat(indicesService.indexServiceSafe(index).getShard(0).recoveryStats().currentAsTarget(), equalTo(1));
        }
    });
    logger.info("--> request recoveries");
    RecoveryResponse response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    List<RecoveryState> recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    List<RecoveryState> nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeARecoveryStates.get(0), 0, StoreRecoverySource.EMPTY_STORE_INSTANCE, true, Stage.DONE, null, nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
    assertOnGoingRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    logger.info("--> request node recovery stats");
    NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
    long nodeAThrottling = Long.MAX_VALUE;
    long nodeBThrottling = Long.MAX_VALUE;
    for (NodeStats nodeStats : statsResponse.getNodes()) {
        final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
        if (nodeStats.getNode().getName().equals(nodeA)) {
            assertThat("node A should have ongoing recovery as source", recoveryStats.currentAsSource(), equalTo(1));
            assertThat("node A should not have ongoing recovery as target", recoveryStats.currentAsTarget(), equalTo(0));
            nodeAThrottling = recoveryStats.throttleTime().millis();
        }
        if (nodeStats.getNode().getName().equals(nodeB)) {
            assertThat("node B should not have ongoing recovery as source", recoveryStats.currentAsSource(), equalTo(0));
            assertThat("node B should have ongoing recovery as target", recoveryStats.currentAsTarget(), equalTo(1));
            nodeBThrottling = recoveryStats.throttleTime().millis();
        }
    }
    logger.info("--> checking throttling increases");
    final long finalNodeAThrottling = nodeAThrottling;
    final long finalNodeBThrottling = nodeBThrottling;
    assertBusy(new Runnable() {

        @Override
        public void run() {
            NodesStatsResponse statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
            assertThat(statsResponse.getNodes(), hasSize(2));
            for (NodeStats nodeStats : statsResponse.getNodes()) {
                final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
                if (nodeStats.getNode().getName().equals(nodeA)) {
                    assertThat("node A throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeAThrottling));
                }
                if (nodeStats.getNode().getName().equals(nodeB)) {
                    assertThat("node B throttling should increase", recoveryStats.throttleTime().millis(), greaterThan(finalNodeBThrottling));
                }
            }
        }
    });
    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();
    // wait for it to be finished
    ensureGreen();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    assertThat(recoveryStates.size(), equalTo(1));
    assertRecoveryState(recoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(recoveryStates.get(0).getIndex());
    statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
    assertThat(statsResponse.getNodes(), hasSize(2));
    for (NodeStats nodeStats : statsResponse.getNodes()) {
        final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
        assertThat(recoveryStats.currentAsSource(), equalTo(0));
        assertThat(recoveryStats.currentAsTarget(), equalTo(0));
        if (nodeStats.getNode().getName().equals(nodeA)) {
            assertThat("node A throttling should be >0", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
        if (nodeStats.getNode().getName().equals(nodeB)) {
            assertThat("node B throttling should be >0 ", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
    }
    logger.info("--> bump replica count");
    client().admin().indices().prepareUpdateSettings(INDEX_NAME).setSettings(Settings.builder().put("number_of_replicas", 1)).execute().actionGet();
    ensureGreen();
    statsResponse = client().admin().cluster().prepareNodesStats().clear().setIndices(new CommonStatsFlags(CommonStatsFlags.Flag.Recovery)).get();
    assertThat(statsResponse.getNodes(), hasSize(2));
    for (NodeStats nodeStats : statsResponse.getNodes()) {
        final RecoveryStats recoveryStats = nodeStats.getIndices().getRecoveryStats();
        assertThat(recoveryStats.currentAsSource(), equalTo(0));
        assertThat(recoveryStats.currentAsTarget(), equalTo(0));
        if (nodeStats.getNode().getName().equals(nodeA)) {
            assertThat("node A throttling should be >0", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
        if (nodeStats.getNode().getName().equals(nodeB)) {
            assertThat("node B throttling should be >0 ", recoveryStats.throttleTime().millis(), greaterThan(0L));
        }
    }
    logger.info("--> start node C");
    String nodeC = internalCluster().startNode();
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("3").get().isTimedOut());
    logger.info("--> slowing down recoveries");
    slowDownRecovery(shardSize);
    logger.info("--> move replica shard from: {} to: {}", nodeA, nodeC);
    client().admin().cluster().prepareReroute().add(new MoveAllocationCommand(INDEX_NAME, 0, nodeA, nodeC)).execute().actionGet().getState();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(1));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    List<RecoveryState> nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeARecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeB, nodeA);
    validateIndexRecoveryState(nodeARecoveryStates.get(0).getIndex());
    assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
    assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
    if (randomBoolean()) {
        // shutdown node with relocation source of replica shard and check if recovery continues
        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(nodeA));
        ensureStableCluster(2);
        response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
        recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
        nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
        assertThat(nodeARecoveryStates.size(), equalTo(0));
        nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
        assertThat(nodeBRecoveryStates.size(), equalTo(1));
        nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
        assertThat(nodeCRecoveryStates.size(), equalTo(1));
        assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
        validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
        assertOnGoingRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, nodeB, nodeC);
        validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
    }
    logger.info("--> speeding up recoveries");
    restoreRecoverySpeed();
    ensureGreen();
    response = client().admin().indices().prepareRecoveries(INDEX_NAME).execute().actionGet();
    recoveryStates = response.shardRecoveryStates().get(INDEX_NAME);
    nodeARecoveryStates = findRecoveriesForTargetNode(nodeA, recoveryStates);
    assertThat(nodeARecoveryStates.size(), equalTo(0));
    nodeBRecoveryStates = findRecoveriesForTargetNode(nodeB, recoveryStates);
    assertThat(nodeBRecoveryStates.size(), equalTo(1));
    nodeCRecoveryStates = findRecoveriesForTargetNode(nodeC, recoveryStates);
    assertThat(nodeCRecoveryStates.size(), equalTo(1));
    assertRecoveryState(nodeBRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, true, Stage.DONE, nodeA, nodeB);
    validateIndexRecoveryState(nodeBRecoveryStates.get(0).getIndex());
    // relocations of replicas are marked as REPLICA and the source node is the node holding the primary (B)
    assertRecoveryState(nodeCRecoveryStates.get(0), 0, PeerRecoverySource.INSTANCE, false, Stage.DONE, nodeB, nodeC);
    validateIndexRecoveryState(nodeCRecoveryStates.get(0).getIndex());
}

Example 29 with MoveAllocationCommand

use of org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand in project elasticsearch by elastic.

the class IndicesStoreIntegrationIT method testShardCleanupIfShardDeletionAfterRelocationFailedAndIndexDeleted.

/* Test that shard is deleted in case ShardActiveRequest after relocation and next incoming cluster state is an index delete. */
public void testShardCleanupIfShardDeletionAfterRelocationFailedAndIndexDeleted() throws Exception {
    final String node_1 = internalCluster().startNode();
    logger.info("--> creating index [test] with one shard and on replica");
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(indexSettings()).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)));
    ensureGreen("test");
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    Index index = state.metaData().index("test").getIndex();
    assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
    assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
    final String node_2 = internalCluster().startDataOnlyNode(Settings.builder().build());
    assertFalse(client().admin().cluster().prepareHealth().setWaitForNodes("2").get().isTimedOut());
    assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
    assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(true));
    assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(false));
    assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(false));
    // add a transport delegate that will prevent the shard active request to succeed the first time after relocation has finished.
    // node_1 will then wait for the next cluster state change before it tries a next attempt to delete the shard.
    MockTransportService transportServiceNode_1 = (MockTransportService) internalCluster().getInstance(TransportService.class, node_1);
    TransportService transportServiceNode_2 = internalCluster().getInstance(TransportService.class, node_2);
    final CountDownLatch shardActiveRequestSent = new CountDownLatch(1);
    transportServiceNode_1.addDelegate(transportServiceNode_2, new MockTransportService.DelegateTransport(transportServiceNode_1.original()) {

        @Override
        protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
            if (action.equals("internal:index/shard/exists") && shardActiveRequestSent.getCount() > 0) {
                shardActiveRequestSent.countDown();
                logger.info("prevent shard active request from being sent");
                throw new ConnectTransportException(connection.getNode(), "DISCONNECT: simulated");
            }
            super.sendRequest(connection, requestId, action, request, options);
        }
    });
    logger.info("--> move shard from {} to {}, and wait for relocation to finish", node_1, node_2);
    internalCluster().client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, node_1, node_2)).get();
    shardActiveRequestSent.await();
    ClusterHealthResponse clusterHealth = client().admin().cluster().prepareHealth().setWaitForNoRelocatingShards(true).get();
    assertThat(clusterHealth.isTimedOut(), equalTo(false));
    logClusterState();
    // delete the index. node_1 that still waits for the next cluster state update will then get the delete index next.
    // it must still delete the shard, even if it cannot find it anymore in indicesservice
    client().admin().indices().prepareDelete("test").get();
    assertThat(waitForShardDeletion(node_1, index, 0), equalTo(false));
    assertThat(waitForIndexDeletion(node_1, index), equalTo(false));
    assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(false));
    assertThat(Files.exists(indexDirectory(node_1, index)), equalTo(false));
    assertThat(waitForShardDeletion(node_2, index, 0), equalTo(false));
    assertThat(waitForIndexDeletion(node_2, index), equalTo(false));
    assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(false));
    assertThat(Files.exists(indexDirectory(node_2, index)), equalTo(false));
}

Example 30 with MoveAllocationCommand

use of org.elasticsearch.cluster.routing.allocation.command.MoveAllocationCommand in project elasticsearch by elastic.

the class ThrottlingAllocationTests method testOutgoingThrottlesAllocation.

public void testOutgoingThrottlesAllocation() {
    TestGatewayAllocator gatewayAllocator = new TestGatewayAllocator();
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_outgoing_recoveries", 1).build(), gatewayAllocator);
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2)).build();
    ClusterState clusterState = createRecoveryStateAndInitalizeAllocations(metaData, gatewayAllocator);
    logger.info("with one node, do reroute, only 1 should initialize");
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(0));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(1));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(2));
    logger.info("start initializing");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(1));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(0));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(2));
    logger.info("start one more node, first non-primary should start being allocated");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(1));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(1));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(1));
    assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
    logger.info("start initializing non-primary");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(2));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(0));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(1));
    assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 0);
    logger.info("start one more node, initializing second non-primary");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(2));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(1));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(0));
    assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
    logger.info("start one more node");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node4"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
    logger.info("move started non-primary to new node");
    AllocationService.CommandsResult commandsResult = strategy.reroute(clusterState, new AllocationCommands(new MoveAllocationCommand("test", 0, "node2", "node4")), true, false);
    assertEquals(commandsResult.explanations().explanations().size(), 1);
    assertEquals(commandsResult.explanations().explanations().get(0).decisions().type(), Decision.Type.THROTTLE);
    // even though it is throttled, move command still forces allocation
    clusterState = commandsResult.getClusterState();
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(1));
    assertThat(clusterState.routingTable().shardsWithState(RELOCATING).size(), equalTo(1));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(2));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(0));
    assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 2);
    assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node2"), 0);
}