Search in sources :

Example 1 with SETTING_AUTO_EXPAND_REPLICAS

use of org.opensearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS in project OpenSearch by opensearch-project.

the class AutoExpandReplicasTests method testAutoExpandWhenNodeLeavesAndPossiblyRejoins.

/**
 * Checks that when nodes leave the cluster that the auto-expand-replica functionality only triggers after failing the shards on
 * the removed nodes. This ensures that active shards on other live nodes are not failed if the primary resided on a now dead node.
 * Instead, one of the replicas on the live nodes first gets promoted to primary, and the auto-expansion (removing replicas) only
 * triggers in a follow-up step.
 */
public void testAutoExpandWhenNodeLeavesAndPossiblyRejoins() throws InterruptedException {
    final ThreadPool threadPool = new TestThreadPool(getClass().getName());
    final ClusterStateChanges cluster = new ClusterStateChanges(xContentRegistry(), threadPool);
    try {
        List<DiscoveryNode> allNodes = new ArrayList<>();
        // local node is the cluster-manager
        DiscoveryNode localNode = createNode(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE);
        allNodes.add(localNode);
        int numDataNodes = randomIntBetween(3, 5);
        List<DiscoveryNode> dataNodes = new ArrayList<>(numDataNodes);
        for (int i = 0; i < numDataNodes; i++) {
            dataNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE));
        }
        allNodes.addAll(dataNodes);
        ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[0]));
        CreateIndexRequest request = new CreateIndexRequest("index", Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_AUTO_EXPAND_REPLICAS, "0-all").build()).waitForActiveShards(ActiveShardCount.NONE);
        state = cluster.createIndex(state, request);
        assertTrue(state.metadata().hasIndex("index"));
        while (state.routingTable().index("index").shard(0).allShardsStarted() == false) {
            logger.info(state);
            state = cluster.applyStartedShards(state, state.routingTable().index("index").shard(0).shardsWithState(ShardRoutingState.INITIALIZING));
            state = cluster.reroute(state, new ClusterRerouteRequest());
        }
        IndexShardRoutingTable preTable = state.routingTable().index("index").shard(0);
        final Set<String> unchangedNodeIds;
        final IndexShardRoutingTable postTable;
        if (randomBoolean()) {
            // simulate node removal
            List<DiscoveryNode> nodesToRemove = randomSubsetOf(2, dataNodes);
            unchangedNodeIds = dataNodes.stream().filter(n -> nodesToRemove.contains(n) == false).map(DiscoveryNode::getId).collect(Collectors.toSet());
            state = cluster.removeNodes(state, nodesToRemove);
            postTable = state.routingTable().index("index").shard(0);
            assertTrue("not all shards started in " + state.toString(), postTable.allShardsStarted());
            assertThat(postTable.toString(), postTable.getAllAllocationIds(), everyItem(is(in(preTable.getAllAllocationIds()))));
        } else {
            // fake an election where conflicting nodes are removed and readded
            state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).masterNodeId(null).build()).build();
            List<DiscoveryNode> conflictingNodes = randomSubsetOf(2, dataNodes);
            unchangedNodeIds = dataNodes.stream().filter(n -> conflictingNodes.contains(n) == false).map(DiscoveryNode::getId).collect(Collectors.toSet());
            List<DiscoveryNode> nodesToAdd = conflictingNodes.stream().map(n -> new DiscoveryNode(n.getName(), n.getId(), buildNewFakeTransportAddress(), n.getAttributes(), n.getRoles(), n.getVersion())).collect(Collectors.toList());
            if (randomBoolean()) {
                nodesToAdd.add(createNode(DiscoveryNodeRole.DATA_ROLE));
            }
            state = cluster.joinNodesAndBecomeMaster(state, nodesToAdd);
            postTable = state.routingTable().index("index").shard(0);
        }
        Set<String> unchangedAllocationIds = preTable.getShards().stream().filter(shr -> unchangedNodeIds.contains(shr.currentNodeId())).map(shr -> shr.allocationId().getId()).collect(Collectors.toSet());
        assertThat(postTable.toString(), unchangedAllocationIds, everyItem(is(in(postTable.getAllAllocationIds()))));
        postTable.getShards().forEach(shardRouting -> {
            if (shardRouting.assignedToNode() && unchangedAllocationIds.contains(shardRouting.allocationId().getId())) {
                assertTrue("Shard should be active: " + shardRouting, shardRouting.active());
            }
        });
    } finally {
        terminate(threadPool);
    }
}
Also used : DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) UpdateSettingsRequest(org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest) SETTING_AUTO_EXPAND_REPLICAS(org.opensearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS) ThreadPool(org.opensearch.threadpool.ThreadPool) TestThreadPool(org.opensearch.threadpool.TestThreadPool) ClusterStateChanges(org.opensearch.indices.cluster.ClusterStateChanges) Version(org.opensearch.Version) ClusterRerouteRequest(org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest) ClusterStateCreationUtils(org.opensearch.action.support.replication.ClusterStateCreationUtils) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) Matchers.everyItem(org.hamcrest.Matchers.everyItem) VersionUtils(org.opensearch.test.VersionUtils) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) DiscoveryNodeRole(org.opensearch.cluster.node.DiscoveryNodeRole) Collectors(java.util.stream.Collectors) ActiveShardCount(org.opensearch.action.support.ActiveShardCount) List(java.util.List) Matchers.equalTo(org.hamcrest.Matchers.equalTo) SETTING_NUMBER_OF_SHARDS(org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS) Matchers.is(org.hamcrest.Matchers.is) Collections(java.util.Collections) Matchers.in(org.hamcrest.Matchers.in) ClusterState(org.opensearch.cluster.ClusterState) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) ThreadPool(org.opensearch.threadpool.ThreadPool) TestThreadPool(org.opensearch.threadpool.TestThreadPool) ArrayList(java.util.ArrayList) TestThreadPool(org.opensearch.threadpool.TestThreadPool) ClusterStateChanges(org.opensearch.indices.cluster.ClusterStateChanges) ClusterRerouteRequest(org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest)

Aggregations

ArrayList (java.util.ArrayList)1 Collections (java.util.Collections)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Locale (java.util.Locale)1 Set (java.util.Set)1 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)1 Collectors (java.util.stream.Collectors)1 Matchers.equalTo (org.hamcrest.Matchers.equalTo)1 Matchers.everyItem (org.hamcrest.Matchers.everyItem)1 Matchers.in (org.hamcrest.Matchers.in)1 Matchers.is (org.hamcrest.Matchers.is)1 Version (org.opensearch.Version)1 ClusterRerouteRequest (org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest)1 CreateIndexRequest (org.opensearch.action.admin.indices.create.CreateIndexRequest)1 UpdateSettingsRequest (org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest)1 ActiveShardCount (org.opensearch.action.support.ActiveShardCount)1 ClusterStateCreationUtils (org.opensearch.action.support.replication.ClusterStateCreationUtils)1 ClusterState (org.opensearch.cluster.ClusterState)1 SETTING_AUTO_EXPAND_REPLICAS (org.opensearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS)1