Examples with RoutingNodes org.elasticsearch.cluster.routing.RoutingNodes used on opensource projects

Example 36 with RoutingNodes

use of org.elasticsearch.cluster.routing.RoutingNodes in project elasticsearch by elastic.

the class ShardsLimitAllocationTests method testClusterLevelShardsLimitAllocate.

public void testClusterLevelShardsLimitAllocate() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put(ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), 1).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTable).build();
    logger.info("Adding two nodes and performing rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
    logger.info("Start the primary shards");
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(1));
    assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(2));
    // Bump the cluster total shards to 2
    strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put(ShardsLimitAllocationDecider.CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), 2).build());
    logger.info("Do another reroute, make sure shards are now allocated");
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
    assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.INITIALIZING), equalTo(1));
    routingNodes = clusterState.getRoutingNodes();
    clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(clusterState.getRoutingNodes().node("node1").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2));
    assertThat(clusterState.getRoutingNodes().node("node2").numberOfShardsWithState(ShardRoutingState.STARTED), equalTo(2));
    assertThat(clusterState.getRoutingNodes().unassigned().size(), equalTo(0));
}

Example 37 with RoutingNodes

use of org.elasticsearch.cluster.routing.RoutingNodes in project elasticsearch by elastic.

the class SingleShardNoReplicasRoutingTests method testSingleIndexStartedShard.

public void testSingleIndexStartedShard() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
    logger.info("Adding one node and performing rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(INITIALIZING));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
    logger.info("Rerouting again, nothing should change");
    clusterState = ClusterState.builder(clusterState).build();
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    clusterState = newState;
    logger.info("Marking the shard as started");
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
    logger.info("Starting another node and making sure nothing changed");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
    logger.info("Killing node1 where the shard is, checking the shard is unassigned");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node1")).build();
    newState = strategy.deassociateDeadNodes(clusterState, true, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
    logger.info("Bring node1 back, and see it's assinged");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node1"))).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(INITIALIZING));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
    logger.info("Start another node, make sure that things remain the same (shard is in node2 and initializing)");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Start the shard on node 1");
    routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
}

Example 38 with RoutingNodes

use of org.elasticsearch.cluster.routing.RoutingNodes in project elasticsearch by elastic.

the class SingleShardNoReplicasRoutingTests method testMultiIndexEvenDistribution.

public void testMultiIndexEvenDistribution() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always").put("cluster.routing.allocation.cluster_concurrent_rebalance", -1).build());
    final int numberOfIndices = 50;
    logger.info("Building initial routing table with " + numberOfIndices + " indices");
    MetaData.Builder metaDataBuilder = MetaData.builder();
    for (int i = 0; i < numberOfIndices; i++) {
        metaDataBuilder.put(IndexMetaData.builder("test" + i).settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0));
    }
    MetaData metaData = metaDataBuilder.build();
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
    for (int i = 0; i < numberOfIndices; i++) {
        routingTableBuilder.addAsNew(metaData.index("test" + i));
    }
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTableBuilder.build()).build();
    assertThat(clusterState.routingTable().indicesRouting().size(), equalTo(numberOfIndices));
    for (int i = 0; i < numberOfIndices; i++) {
        assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).currentNodeId(), nullValue());
    }
    logger.info("Adding " + (numberOfIndices / 2) + " nodes");
    DiscoveryNodes.Builder nodesBuilder = DiscoveryNodes.builder();
    List<DiscoveryNode> nodes = new ArrayList<>();
    for (int i = 0; i < (numberOfIndices / 2); i++) {
        nodesBuilder.add(newNode("node" + i));
    }
    clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    for (int i = 0; i < numberOfIndices; i++) {
        assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).unassigned(), equalTo(false));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state(), equalTo(INITIALIZING));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).primary(), equalTo(true));
        // make sure we still have 2 shards initializing per node on the first 25 nodes
        String nodeId = clusterState.routingTable().index("test" + i).shard(0).shards().get(0).currentNodeId();
        int nodeIndex = Integer.parseInt(nodeId.substring("node".length()));
        assertThat(nodeIndex, lessThan(25));
    }
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    Set<String> encounteredIndices = new HashSet<>();
    for (RoutingNode routingNode : routingNodes) {
        assertThat(routingNode.numberOfShardsWithState(STARTED), equalTo(0));
        assertThat(routingNode.size(), equalTo(2));
        // make sure we still have 2 shards initializing per node on the only 25 nodes
        int nodeIndex = Integer.parseInt(routingNode.nodeId().substring("node".length()));
        assertThat(nodeIndex, lessThan(25));
        // check that we don't have a shard associated with a node with the same index name (we have a single shard)
        for (ShardRouting shardRoutingEntry : routingNode) {
            assertThat(encounteredIndices, not(hasItem(shardRoutingEntry.getIndexName())));
            encounteredIndices.add(shardRoutingEntry.getIndexName());
        }
    }
    logger.info("Adding additional " + (numberOfIndices / 2) + " nodes, nothing should change");
    nodesBuilder = DiscoveryNodes.builder(clusterState.nodes());
    for (int i = (numberOfIndices / 2); i < numberOfIndices; i++) {
        nodesBuilder.add(newNode("node" + i));
    }
    clusterState = ClusterState.builder(clusterState).nodes(nodesBuilder).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Marking the shard as started");
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    int numberOfRelocatingShards = 0;
    int numberOfStartedShards = 0;
    for (int i = 0; i < numberOfIndices; i++) {
        assertThat(clusterState.routingTable().index("test" + i).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).unassigned(), equalTo(false));
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state(), anyOf(equalTo(STARTED), equalTo(RELOCATING)));
        if (clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state() == STARTED) {
            numberOfStartedShards++;
        } else if (clusterState.routingTable().index("test" + i).shard(0).shards().get(0).state() == RELOCATING) {
            numberOfRelocatingShards++;
        }
        assertThat(clusterState.routingTable().index("test" + i).shard(0).shards().get(0).primary(), equalTo(true));
        // make sure we still have 2 shards either relocating or started on the first 25 nodes (still)
        String nodeId = clusterState.routingTable().index("test" + i).shard(0).shards().get(0).currentNodeId();
        int nodeIndex = Integer.parseInt(nodeId.substring("node".length()));
        assertThat(nodeIndex, lessThan(25));
    }
    assertThat(numberOfRelocatingShards, equalTo(25));
    assertThat(numberOfStartedShards, equalTo(25));
}

Example 39 with RoutingNodes

use of org.elasticsearch.cluster.routing.RoutingNodes in project elasticsearch by elastic.

the class SingleShardNoReplicasRoutingTests method testSingleIndexShardFailed.

public void testSingleIndexShardFailed() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(0)).build();
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder().addAsNew(metaData.index("test"));
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(routingTableBuilder.build()).build();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
    logger.info("Adding one node and rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).unassigned(), equalTo(false));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(INITIALIZING));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), equalTo("node1"));
    logger.info("Marking the shard as failed");
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyFailedShard(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING).get(0));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
}

Example 40 with RoutingNodes

use of org.elasticsearch.cluster.routing.RoutingNodes in project elasticsearch by elastic.

the class IndexBalanceTests method testBalanceAllNodesStartedAddIndex.

public void testBalanceAllNodesStartedAddIndex() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).put("cluster.routing.allocation.node_initial_primaries_recoveries", 10).put(ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(), "always").put("cluster.routing.allocation.cluster_concurrent_rebalance", -1).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(1)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(1).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(0).currentNodeId(), nullValue());
        assertThat(clusterState.routingTable().index("test").shard(i).shards().get(1).currentNodeId(), nullValue());
    }
    logger.info("Adding three node and performing rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2")).add(newNode("node3"))).build();
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(INITIALIZING));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().get(0).currentNodeId(), nullValue());
    }
    logger.info("Another round of rebalancing");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().size(), equalTo(1));
        // backup shards are initializing as well, we make sure that they
        // recover from primary *started* shards in the
        // IndicesClusterStateService
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().get(0).state(), equalTo(INITIALIZING));
    }
    logger.info("Reroute, nothing should change");
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Start the more shards");
    routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    routingNodes = clusterState.getRoutingNodes();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test").shard(i).replicaShards().size(), equalTo(1));
    }
    assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), equalTo(2));
    assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(2));
    assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(2));
    assertThat(routingNodes.node("node1").shardsWithState("test", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node2").shardsWithState("test", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node3").shardsWithState("test", STARTED).size(), equalTo(2));
    logger.info("Add new index 3 shards 1 replica");
    MetaData updatedMetaData = MetaData.builder(clusterState.metaData()).put(IndexMetaData.builder("test1").settings(settings(Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 3).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 1))).build();
    RoutingTable updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()).addAsNew(updatedMetaData.index("test1")).build();
    clusterState = ClusterState.builder(clusterState).metaData(updatedMetaData).routingTable(updatedRoutingTable).build();
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).primaryShard().state(), equalTo(INITIALIZING));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().size(), equalTo(1));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().get(0).state(), equalTo(UNASSIGNED));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().get(0).currentNodeId(), nullValue());
    }
    logger.info("Another round of rebalancing");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes())).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().size(), equalTo(1));
        // backup shards are initializing as well, we make sure that they
        // recover from primary *started* shards in the
        // IndicesClusterStateService
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().get(0).state(), equalTo(INITIALIZING));
    }
    logger.info("Reroute, nothing should change");
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Start the more shards");
    routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    routingNodes = clusterState.getRoutingNodes();
    assertThat(clusterState.routingTable().index("test1").shards().size(), equalTo(3));
    for (int i = 0; i < clusterState.routingTable().index("test1").shards().size(); i++) {
        assertThat(clusterState.routingTable().index("test1").shard(i).size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).shards().size(), equalTo(2));
        assertThat(clusterState.routingTable().index("test1").shard(i).primaryShard().state(), equalTo(STARTED));
        assertThat(clusterState.routingTable().index("test1").shard(i).replicaShards().size(), equalTo(1));
    }
    assertThat(routingNodes.node("node1").numberOfShardsWithState(STARTED), equalTo(4));
    assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(4));
    assertThat(routingNodes.node("node3").numberOfShardsWithState(STARTED), equalTo(4));
    assertThat(routingNodes.node("node1").shardsWithState("test1", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node2").shardsWithState("test1", STARTED).size(), equalTo(2));
    assertThat(routingNodes.node("node3").shardsWithState("test1", STARTED).size(), equalTo(2));
}