Examples with MetaData org.elasticsearch.cluster.metadata.MetaData used on opensource projects

Example 51 with MetaData

use of org.elasticsearch.cluster.metadata.MetaData in project elasticsearch by elastic.

the class SingleShardOneReplicaRoutingTests method testSingleIndexFirstStartPrimaryThenBackups.

public void testSingleIndexFirstStartPrimaryThenBackups() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(1).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
    assertThat(clusterState.routingTable().index("test").shard(0).shards().get(1).currentNodeId(), nullValue());
    logger.info("Adding one node and performing rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    ClusterState newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(INITIALIZING));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo("node1"));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(), nullValue());
    logger.info("Add another node and perform rerouting, nothing will happen since primary shards not started");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2"))).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Start the primary shard (on node1)");
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.node("node1").shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo("node1"));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().size(), equalTo(1));
    // backup shards are initializing as well, we make sure that they recover from primary *started* shards in the IndicesClusterStateService
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).state(), equalTo(INITIALIZING));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(), equalTo("node2"));
    logger.info("Reroute, nothing should change");
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
    logger.info("Start the backup shard");
    routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.node("node2").shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo("node1"));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(), equalTo("node2"));
    logger.info("Kill node1, backup shard should become primary");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).remove("node1")).build();
    newState = strategy.deassociateDeadNodes(clusterState, true, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo("node2"));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().size(), equalTo(1));
    // backup shards are initializing as well, we make sure that they recover from primary *started* shards in the IndicesClusterStateService
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(), nullValue());
    logger.info("Start another node, backup shard should start initializing");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo("node2"));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().size(), equalTo(1));
    // backup shards are initializing as well, we make sure that they recover from primary *started* shards in the IndicesClusterStateService
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).state(), equalTo(INITIALIZING));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(), equalTo("node3"));
}

Example 52 with MetaData

use of org.elasticsearch.cluster.metadata.MetaData in project elasticsearch by elastic.

the class ThrottlingAllocationTests method testPrimaryRecoveryThrottling.

public void testPrimaryRecoveryThrottling() {
    TestGatewayAllocator gatewayAllocator = new TestGatewayAllocator();
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 3).put("cluster.routing.allocation.node_initial_primaries_recoveries", 3).build(), gatewayAllocator);
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(10).numberOfReplicas(1)).build();
    ClusterState clusterState = createRecoveryStateAndInitalizeAllocations(metaData, gatewayAllocator);
    logger.info("start one node, do reroute, only 3 should initialize");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(0));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(3));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(17));
    logger.info("start initializing, another 3 should initialize");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(3));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(3));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(14));
    logger.info("start initializing, another 3 should initialize");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(6));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(3));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(11));
    logger.info("start initializing, another 1 should initialize");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(9));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(1));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(10));
    logger.info("start initializing, all primaries should be started");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(10));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(0));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(10));
}

Example 53 with MetaData

use of org.elasticsearch.cluster.metadata.MetaData in project elasticsearch by elastic.

the class ThrottlingAllocationTests method createRecoveryStateAndInitalizeAllocations.

private ClusterState createRecoveryStateAndInitalizeAllocations(MetaData metaData, TestGatewayAllocator gatewayAllocator) {
    DiscoveryNode node1 = newNode("node1");
    MetaData.Builder metaDataBuilder = new MetaData.Builder(metaData);
    RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
    for (ObjectCursor<IndexMetaData> cursor : metaData.indices().values()) {
        Index index = cursor.value.getIndex();
        IndexMetaData.Builder indexMetaDataBuilder = IndexMetaData.builder(cursor.value);
        final int recoveryType = randomInt(5);
        if (recoveryType <= 4) {
            addInSyncAllocationIds(index, indexMetaDataBuilder, gatewayAllocator, node1);
        }
        IndexMetaData indexMetaData = indexMetaDataBuilder.build();
        metaDataBuilder.put(indexMetaData, false);
        switch(recoveryType) {
            case 0:
                routingTableBuilder.addAsRecovery(indexMetaData);
                break;
            case 1:
                routingTableBuilder.addAsFromCloseToOpen(indexMetaData);
                break;
            case 2:
                routingTableBuilder.addAsFromDangling(indexMetaData);
                break;
            case 3:
                routingTableBuilder.addAsNewRestore(indexMetaData, new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT, indexMetaData.getIndex().getName()), new IntHashSet());
                break;
            case 4:
                routingTableBuilder.addAsRestore(indexMetaData, new SnapshotRecoverySource(new Snapshot("repo", new SnapshotId("snap", "randomId")), Version.CURRENT, indexMetaData.getIndex().getName()));
                break;
            case 5:
                routingTableBuilder.addAsNew(indexMetaData);
                break;
            default:
                throw new IndexOutOfBoundsException();
        }
    }
    return ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).nodes(DiscoveryNodes.builder().add(node1)).metaData(metaDataBuilder.build()).routingTable(routingTableBuilder.build()).build();
}

Example 54 with MetaData

use of org.elasticsearch.cluster.metadata.MetaData in project elasticsearch by elastic.

the class ThrottlingAllocationTests method testThrottleIncomingAndOutgoing.

public void testThrottleIncomingAndOutgoing() {
    TestGatewayAllocator gatewayAllocator = new TestGatewayAllocator();
    Settings settings = Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 5).put("cluster.routing.allocation.node_initial_primaries_recoveries", 5).put("cluster.routing.allocation.cluster_concurrent_rebalance", 5).build();
    AllocationService strategy = createAllocationService(settings, gatewayAllocator);
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(9).numberOfReplicas(0)).build();
    ClusterState clusterState = createRecoveryStateAndInitalizeAllocations(metaData, gatewayAllocator);
    logger.info("with one node, do reroute, only 5 should initialize");
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(0));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(5));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(4));
    assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 5);
    logger.info("start initializing, all primaries should be started");
    clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(5));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(4));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(0));
    clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
    logger.info("start another 2 nodes, 5 shards should be relocating - at most 5 are allowed per node");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node2")).add(newNode("node3"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(4));
    assertThat(clusterState.routingTable().shardsWithState(RELOCATING).size(), equalTo(5));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(5));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(0));
    assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node2"), 3);
    assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node3"), 2);
    assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 0);
    assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 5);
    clusterState = strategy.applyStartedShards(clusterState, clusterState.routingTable().shardsWithState(INITIALIZING));
    logger.info("start the relocating shards, one more shard should relocate away from node1");
    assertThat(clusterState.routingTable().shardsWithState(STARTED).size(), equalTo(8));
    assertThat(clusterState.routingTable().shardsWithState(RELOCATING).size(), equalTo(1));
    assertThat(clusterState.routingTable().shardsWithState(INITIALIZING).size(), equalTo(1));
    assertThat(clusterState.routingTable().shardsWithState(UNASSIGNED).size(), equalTo(0));
    assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node2"), 0);
    assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node3"), 1);
    assertEquals(clusterState.getRoutingNodes().getIncomingRecoveries("node1"), 0);
    assertEquals(clusterState.getRoutingNodes().getOutgoingRecoveries("node1"), 1);
}

Example 55 with MetaData

use of org.elasticsearch.cluster.metadata.MetaData in project elasticsearch by elastic.

the class UpdateNumberOfReplicasTests method testUpdateNumberOfReplicas.

public void testUpdateNumberOfReplicas() {
    AllocationService strategy = createAllocationService(Settings.builder().put("cluster.routing.allocation.node_concurrent_recoveries", 10).build());
    logger.info("Building initial routing table");
    MetaData metaData = MetaData.builder().put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1)).build();
    RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metaData.index("test")).build();
    ClusterState clusterState = ClusterState.builder(org.elasticsearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).metaData(metaData).routingTable(initialRoutingTable).build();
    assertThat(initialRoutingTable.index("test").shards().size(), equalTo(1));
    assertThat(initialRoutingTable.index("test").shard(0).size(), equalTo(2));
    assertThat(initialRoutingTable.index("test").shard(0).shards().size(), equalTo(2));
    assertThat(initialRoutingTable.index("test").shard(0).shards().get(0).state(), equalTo(UNASSIGNED));
    assertThat(initialRoutingTable.index("test").shard(0).shards().get(1).state(), equalTo(UNASSIGNED));
    assertThat(initialRoutingTable.index("test").shard(0).shards().get(0).currentNodeId(), nullValue());
    assertThat(initialRoutingTable.index("test").shard(0).shards().get(1).currentNodeId(), nullValue());
    logger.info("Adding two nodes and performing rerouting");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(newNode("node1")).add(newNode("node2"))).build();
    clusterState = strategy.reroute(clusterState, "reroute");
    logger.info("Start all the primary shards");
    RoutingNodes routingNodes = clusterState.getRoutingNodes();
    clusterState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    logger.info("Start all the replica shards");
    routingNodes = clusterState.getRoutingNodes();
    ClusterState newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    final String nodeHoldingPrimary = clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId();
    final String nodeHoldingReplica = clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId();
    assertThat(nodeHoldingPrimary, not(equalTo(nodeHoldingReplica)));
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).shards().size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo(nodeHoldingPrimary));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(), equalTo(nodeHoldingReplica));
    logger.info("add another replica");
    routingNodes = clusterState.getRoutingNodes();
    RoutingTable updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()).updateNumberOfReplicas(2).build();
    metaData = MetaData.builder(clusterState.metaData()).updateNumberOfReplicas(2).build();
    clusterState = ClusterState.builder(clusterState).routingTable(updatedRoutingTable).metaData(metaData).build();
    assertThat(clusterState.metaData().index("test").getNumberOfReplicas(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(3));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo(nodeHoldingPrimary));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(), equalTo(nodeHoldingReplica));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(1).state(), equalTo(UNASSIGNED));
    logger.info("Add another node and start the added replica");
    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder(clusterState.nodes()).add(newNode("node3"))).build();
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(3));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo(nodeHoldingPrimary));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShardsWithState(STARTED).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShardsWithState(STARTED).get(0).currentNodeId(), equalTo(nodeHoldingReplica));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShardsWithState(INITIALIZING).size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShardsWithState(INITIALIZING).get(0).currentNodeId(), equalTo("node3"));
    routingNodes = clusterState.getRoutingNodes();
    newState = strategy.applyStartedShards(clusterState, routingNodes.shardsWithState(INITIALIZING));
    assertThat(newState, not(equalTo(clusterState)));
    clusterState = newState;
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(3));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo(nodeHoldingPrimary));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShardsWithState(STARTED).size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShardsWithState(STARTED).get(0).currentNodeId(), anyOf(equalTo(nodeHoldingReplica), equalTo("node3")));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShardsWithState(STARTED).get(1).currentNodeId(), anyOf(equalTo(nodeHoldingReplica), equalTo("node3")));
    logger.info("now remove a replica");
    routingNodes = clusterState.getRoutingNodes();
    updatedRoutingTable = RoutingTable.builder(clusterState.routingTable()).updateNumberOfReplicas(1).build();
    metaData = MetaData.builder(clusterState.metaData()).updateNumberOfReplicas(1).build();
    clusterState = ClusterState.builder(clusterState).routingTable(updatedRoutingTable).metaData(metaData).build();
    assertThat(clusterState.metaData().index("test").getNumberOfReplicas(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).size(), equalTo(2));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).primaryShard().currentNodeId(), equalTo(nodeHoldingPrimary));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().size(), equalTo(1));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).state(), equalTo(STARTED));
    assertThat(clusterState.routingTable().index("test").shard(0).replicaShards().get(0).currentNodeId(), anyOf(equalTo(nodeHoldingReplica), equalTo("node3")));
    logger.info("do a reroute, should remain the same");
    newState = strategy.reroute(clusterState, "reroute");
    assertThat(newState, equalTo(clusterState));
}