Examples with RoutingTable org.opensearch.cluster.routing.RoutingTable used on opensource projects

Example 56 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class NodeLoadAwareAllocationTests method testThreeZoneTwoReplicaLimitsShardAllocationOnOverloadAcrossZones.

public void testThreeZoneTwoReplicaLimitsShardAllocationOnOverloadAcrossZones() {
    AllocationService strategy = createAllocationServiceWithAdditionalSettings(org.opensearch.common.collect.Map.of(NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_PROVISIONED_CAPACITY_SETTING.getKey(), 9, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_SKEW_FACTOR_SETTING.getKey(), 10, "cluster.routing.allocation.awareness.force.zone.values", "zone_1,zone_2,zone_3"));
    logger.info("Building initial routing table for 'testThreeZoneTwoReplicaLimitsShardAllocationOnOverloadAcrossZones'");
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(21).numberOfReplicas(2)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(initialRoutingTable).build();
    logger.info("--> adding three nodes on same zone and do rerouting");
    clusterState = addNodes(clusterState, strategy, "zone_1", "node1", "node2", "node3");
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(21));
    logger.info("--> start the shards (primaries)");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    logger.info("--> replica will not start because we have only one rack value");
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(21));
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(0));
    logger.info("--> add three new node with a new rack and reroute");
    clusterState = addNodes(clusterState, strategy, "zone_2", "node4", "node5", "node6");
    assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(21));
    assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).size(), equalTo(21));
    assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.INITIALIZING).get(0).currentNodeId(), equalTo("node4"));
    logger.info("--> complete relocation");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(42));
    logger.info("--> do another reroute, make sure nothing moves");
    assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable()));
    logger.info("--> add another node with a new rack, make sure nothing moves");
    ClusterState newState = addNodes(clusterState, strategy, "zone_3", "node7", "node8", "node9");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(63));
    logger.info("--> Remove two nodes from zones");
    // remove one nodes in one zone to cause distribution zone1->2 , zone2->3, zone3->2
    newState = removeNodes(newState, strategy, "node7", "node2");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    // ensure minority zone doesn't get overloaded
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(53));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(10));
    for (ShardRouting shard : newState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
        assertEquals(shard.unassignedInfo().getReason(), UnassignedInfo.Reason.NODE_LEFT);
    }
    newState = ClusterState.builder(newState).nodes(DiscoveryNodes.builder(newState.nodes()).add(newNode("node7", singletonMap("zone", "zone_3"))).add(newNode("node2", singletonMap("zone", "zone_1")))).build();
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    for (RoutingNode node : newState.getRoutingNodes()) {
        assertThat(node.size(), equalTo(7));
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(63));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(0));
}

Example 57 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class NodeLoadAwareAllocationTests method testSingleZoneOneReplicaLimitsShardAllocationOnOverload.

public void testSingleZoneOneReplicaLimitsShardAllocationOnOverload() {
    GatewayAllocator gatewayAllocator = new TestGatewayAllocator();
    AllocationService strategy = createAllocationServiceWithAdditionalSettings(org.opensearch.common.collect.Map.of(NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_PROVISIONED_CAPACITY_SETTING.getKey(), 5, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_SKEW_FACTOR_SETTING.getKey(), 20, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_ALLOW_UNASSIGNED_PRIMARIES_SETTING.getKey(), true), gatewayAllocator);
    logger.info("Building initial routing table for 'testSingleZoneOneReplicaLimitsShardAllocationOnOverload'");
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(20).numberOfReplicas(1)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(initialRoutingTable).build();
    logger.info("--> adding five nodes on same zone and do rerouting");
    clusterState = addNodes(clusterState, strategy, "zone_1", "node1", "node2", "node3", "node4", "node5");
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(20));
    logger.info("--> start the shards (primaries)");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(20));
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(20));
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(40));
    logger.info("--> Remove node1 from zone");
    ClusterState newState = removeNodes(clusterState, strategy, "node1");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(40));
    for (RoutingNode node : newState.getRoutingNodes()) {
        assertThat(node.size(), equalTo(10));
    }
    logger.info("--> Remove node2 when the limit of overload is reached");
    newState = removeNodes(newState, strategy, "node2");
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(30));
    for (ShardRouting shard : newState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
        assertEquals(shard.unassignedInfo().getReason(), UnassignedInfo.Reason.NODE_LEFT);
        assertFalse(shard.primary());
    }
    logger.info("add another index with 20 shards");
    metadata = Metadata.builder(newState.metadata()).put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 20).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1))).build();
    RoutingTable updatedRoutingTable = RoutingTable.builder(newState.routingTable()).addAsNew(metadata.index("test1")).build();
    // increases avg shard per node to 80/5 = 16, overload factor 1.2, total allowed 20
    newState = ClusterState.builder(newState).metadata(metadata).routingTable(updatedRoutingTable).build();
    newState = strategy.reroute(newState, "reroute");
    newState = startInitializingShardsAndReroute(strategy, newState);
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(60));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(20));
    logger.info("add another index with 60 shards");
    metadata = Metadata.builder(newState.metadata()).put(IndexMetadata.builder("test2").settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 60).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0))).build();
    updatedRoutingTable = RoutingTable.builder(newState.routingTable()).addAsNew(metadata.index("test2")).build();
    // increases avg shard per node to 140/5 = 28, overload factor 1.2, total allowed 34 per node but still ALL primaries get assigned
    newState = ClusterState.builder(newState).metadata(metadata).routingTable(updatedRoutingTable).build();
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(120));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(20));
    logger.info("change settings to allow unassigned primaries");
    strategy = createAllocationServiceWithAdditionalSettings(org.opensearch.common.collect.Map.of(NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_PROVISIONED_CAPACITY_SETTING.getKey(), 5, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_SKEW_FACTOR_SETTING.getKey(), 20, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_ALLOW_UNASSIGNED_PRIMARIES_SETTING.getKey(), false), gatewayAllocator);
    for (RoutingNode node : newState.getRoutingNodes()) {
        assertThat(node.size(), equalTo(40));
    }
    logger.info("add another index with 5 shards");
    metadata = Metadata.builder(newState.metadata()).put(IndexMetadata.builder("test3").settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0))).build();
    updatedRoutingTable = RoutingTable.builder(newState.routingTable()).addAsNew(metadata.index("test3")).build();
    // increases avg shard per node to 145/5 = 29, overload factor 1.2, total allowed 35 per node and NO primaries get assigned
    // since total owning shards are 40 per node already
    newState = ClusterState.builder(newState).metadata(metadata).routingTable(updatedRoutingTable).build();
    newState = strategy.reroute(newState, "reroute");
    newState = startInitializingShardsAndReroute(strategy, newState);
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(120));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(25));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().filter(ShardRouting::primary).count(), equalTo(5L));
}

Example 58 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class NodeLoadAwareAllocationTests method testSingleZoneTwoReplicaLimitsReplicaAllocationOnOverload.

public void testSingleZoneTwoReplicaLimitsReplicaAllocationOnOverload() {
    AllocationService strategy = createAllocationServiceWithAdditionalSettings(org.opensearch.common.collect.Map.of(NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_PROVISIONED_CAPACITY_SETTING.getKey(), 3, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_SKEW_FACTOR_SETTING.getKey(), 10));
    logger.info("Building initial routing table for 'testSingleZoneTwoReplicaLimitsReplicaAllocationOnOverload'");
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(3).numberOfReplicas(2)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(initialRoutingTable).build();
    logger.info("--> adding three nodes on same rack and do rerouting");
    clusterState = addNodes(clusterState, strategy, "zone1", "node1", "node2", "node3");
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(3));
    logger.info("--> start the shards (primaries)");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    logger.info("--> replicas are initializing");
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(3));
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(6));
    logger.info("--> start the shards (replicas)");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    logger.info("--> all shards are started");
    assertThat(clusterState.getRoutingNodes().shardsWithState(ShardRoutingState.STARTED).size(), equalTo(9));
    logger.info("--> do another reroute, make sure nothing moves");
    assertThat(strategy.reroute(clusterState, "reroute").routingTable(), sameInstance(clusterState.routingTable()));
    // remove one node to make zone1 skewed
    clusterState = removeNodes(clusterState, strategy, randomFrom("node1", "node2", "node3"));
    while (clusterState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    }
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(6));
    assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(3));
    for (ShardRouting shard : clusterState.getRoutingNodes().shardsWithState(UNASSIGNED)) {
        assertEquals(shard.unassignedInfo().getReason(), UnassignedInfo.Reason.NODE_LEFT);
        assertFalse(shard.primary());
    }
}

Example 59 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class NodeLoadAwareAllocationTests method testExistingPrimariesAllocationOnOverload.

public void testExistingPrimariesAllocationOnOverload() {
    GatewayAllocator gatewayAllocator = new TestGatewayAllocator();
    AllocationService strategy = createAllocationServiceWithAdditionalSettings(org.opensearch.common.collect.Map.of(NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_PROVISIONED_CAPACITY_SETTING.getKey(), 5, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_SKEW_FACTOR_SETTING.getKey(), 50, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_ALLOW_UNASSIGNED_PRIMARIES_SETTING.getKey(), false), gatewayAllocator);
    logger.info("Building initial routing table for 'testExistingPrimariesAllocationOnOverload'");
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(20).numberOfReplicas(0)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(initialRoutingTable).build();
    logger.info("--> adding five nodes on same zone and do rerouting");
    clusterState = addNodes(clusterState, strategy, "zone_1", "node1", "node2", "node3", "node4", "node5");
    assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(20));
    logger.info("--> start the shards (primaries)");
    clusterState = startInitializingShardsAndReroute(strategy, clusterState);
    assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(20));
    logger.info("--> Remove nodes from zone holding primaries");
    ClusterState newState = removeNodes(clusterState, strategy, "node1", "node2", "node3");
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(8));
    logger.info("add another index with 20 shards");
    metadata = Metadata.builder(newState.metadata()).put(IndexMetadata.builder("test1").settings(settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 20).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0))).build();
    RoutingTable updatedRoutingTable = RoutingTable.builder(newState.routingTable()).addAsNew(metadata.index("test1")).build();
    newState = ClusterState.builder(newState).metadata(metadata).routingTable(updatedRoutingTable).build();
    newState = strategy.reroute(newState, "reroute");
    newState = startInitializingShardsAndReroute(strategy, newState);
    logger.info("limits should be applied on newly create primaries");
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(24));
    assertThat(newState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(16));
    assertEquals(12L, newState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().filter(r -> r.unassignedInfo().getReason() == UnassignedInfo.Reason.NODE_LEFT).count());
    assertEquals(4L, newState.getRoutingNodes().shardsWithState(UNASSIGNED).stream().filter(r -> r.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED).count());
    assertThat(newState.getRoutingNodes().node("node4").size(), equalTo(12));
    logger.info("--> Remove node4 from zone holding primaries");
    newState = removeNodes(newState, strategy, "node4");
    logger.info("--> change the overload load factor to zero and verify if unassigned primaries on disk get assigned despite overload");
    strategy = createAllocationServiceWithAdditionalSettings(org.opensearch.common.collect.Map.of(NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_PROVISIONED_CAPACITY_SETTING.getKey(), 5, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_SKEW_FACTOR_SETTING.getKey(), 0, NodeLoadAwareAllocationDecider.CLUSTER_ROUTING_ALLOCATION_LOAD_AWARENESS_ALLOW_UNASSIGNED_PRIMARIES_SETTING.getKey(), false), gatewayAllocator);
    newState = strategy.reroute(newState, "reroute");
    logger.info("--> Add back node4 and ensure existing primaries are assigned");
    newState = ClusterState.builder(newState).nodes(DiscoveryNodes.builder(newState.nodes()).add(newNode("node4", singletonMap("zone", "zone_1")))).build();
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    logger.info("--> do another reroute, make sure nothing moves");
    assertThat(strategy.reroute(newState, "reroute").routingTable(), sameInstance(newState.routingTable()));
    assertThat(newState.getRoutingNodes().node("node4").size(), equalTo(12));
    assertThat(newState.getRoutingNodes().node("node5").size(), equalTo(12));
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(24));
    newState = ClusterState.builder(newState).nodes(DiscoveryNodes.builder(newState.nodes()).add(newNode("node1", singletonMap("zone", "zone_1")))).build();
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(32));
    // add back node2 when skewness is still breached
    newState = ClusterState.builder(newState).nodes(DiscoveryNodes.builder(newState.nodes()).add(newNode("node2", singletonMap("zone", "zone_1")))).build();
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(36));
    // add back node3
    newState = ClusterState.builder(newState).nodes(DiscoveryNodes.builder(newState.nodes()).add(newNode("node3", singletonMap("zone", "zone_1")))).build();
    newState = strategy.reroute(newState, "reroute");
    while (newState.getRoutingNodes().shardsWithState(INITIALIZING).isEmpty() == false) {
        newState = startInitializingShardsAndReroute(strategy, newState);
    }
    assertThat(newState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(40));
}

Example 60 with RoutingTable

use of org.opensearch.cluster.routing.RoutingTable in project OpenSearch by opensearch-project.

the class PrimaryElectionRoutingTests method testBackupElectionToPrimaryWhenPrimaryCanBeAllocatedToAnotherNode.

public void testBackupElectionToPrimaryWhenPrimaryCanBeAllocatedToAnotherNode() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    logger.info("Building initial routing table");
    Metadata metadata = Metadata.builder().put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metadata(metadata).routingTable(routingTable).build();
    logger.info("Adding two nodes and performing rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    logger.info("Start the primary shard (on node1)");
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    clusterState = startInitializingShardsAndReroute(strategy, clusterState, routingNodes.node("node1"));
    logger.info("Start the backup shard (on node2)");
    routingNodes = clusterState.getRoutingNodes();
    clusterState = startInitializingShardsAndReroute(strategy, clusterState, routingNodes.node("node2"));
    logger.info("Adding third node and reroute and kill first node");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3")).remove("node1")).build();
    RoutingTable prevRoutingTable = clusterState.routingTable();
    clusterState = strategy.disassociateDeadNodes(clusterState, true, "reroute");
    routingNodes = clusterState.getRoutingNodes();
    routingTable = clusterState.routingTable();
    assertThat(prevRoutingTable != routingTable, equalTo(true));
    assertThat(routingTable.index("test").shards().size(), equalTo(1));
    assertThat(routingNodes.node("node1"), nullValue());
    assertThat(routingNodes.node("node2").numberOfShardsWithState(STARTED), equalTo(1));
    assertThat(routingNodes.node("node3").numberOfShardsWithState(INITIALIZING), equalTo(1));
    // verify where the primary is
    assertThat(routingTable.index("test").shard(0).primaryShard().currentNodeId(), equalTo("node2"));
    assertThat(clusterState.metadata().index("test").primaryTerm(0), equalTo(2L));
    assertThat(routingTable.index("test").shard(0).replicaShards().get(0).currentNodeId(), equalTo("node3"));
}