Search in sources :

Example 31 with IndexShardRoutingTable

use of org.opensearch.cluster.routing.IndexShardRoutingTable in project OpenSearch by opensearch-project.

the class ReplicationOperationTests method testAddedReplicaAfterPrimaryOperation.

public void testAddedReplicaAfterPrimaryOperation() throws Exception {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    final ClusterState initialState = stateWithActivePrimary(index, true, 0);
    Set<String> inSyncAllocationIds = initialState.metadata().index(index).inSyncAllocationIds(0);
    IndexShardRoutingTable shardRoutingTable = initialState.getRoutingTable().shardRoutingTable(shardId);
    Set<String> trackedShards = new HashSet<>();
    addTrackingInfo(shardRoutingTable, null, trackedShards, new HashSet<>());
    ReplicationGroup initialReplicationGroup = new ReplicationGroup(shardRoutingTable, inSyncAllocationIds, trackedShards, 0);
    final ClusterState stateWithAddedReplicas;
    if (randomBoolean()) {
        stateWithAddedReplicas = state(index, true, ShardRoutingState.STARTED, randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.STARTED);
    } else {
        stateWithAddedReplicas = state(index, true, ShardRoutingState.RELOCATING);
    }
    inSyncAllocationIds = stateWithAddedReplicas.metadata().index(index).inSyncAllocationIds(0);
    shardRoutingTable = stateWithAddedReplicas.getRoutingTable().shardRoutingTable(shardId);
    trackedShards = new HashSet<>();
    addTrackingInfo(shardRoutingTable, null, trackedShards, new HashSet<>());
    ReplicationGroup updatedReplicationGroup = new ReplicationGroup(shardRoutingTable, inSyncAllocationIds, trackedShards, 0);
    final AtomicReference<ReplicationGroup> replicationGroup = new AtomicReference<>(initialReplicationGroup);
    logger.debug("--> using initial replicationGroup:\n{}", replicationGroup.get());
    final long primaryTerm = initialState.getMetadata().index(shardId.getIndexName()).primaryTerm(shardId.id());
    final ShardRouting primaryShard = updatedReplicationGroup.getRoutingTable().primaryShard();
    final TestPrimary primary = new TestPrimary(primaryShard, replicationGroup::get, threadPool) {

        @Override
        public void perform(Request request, ActionListener<Result> listener) {
            super.perform(request, ActionListener.map(listener, result -> {
                replicationGroup.set(updatedReplicationGroup);
                logger.debug("--> state after primary operation:\n{}", replicationGroup.get());
                return result;
            }));
        }
    };
    Request request = new Request(shardId);
    PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
    final TestReplicationOperation op = new TestReplicationOperation(request, primary, listener, new TestReplicaProxy(), primaryTerm);
    op.execute();
    assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true));
    Set<ShardRouting> expectedReplicas = getExpectedReplicas(shardId, stateWithAddedReplicas, trackedShards);
    assertThat(request.processedOnReplicas, equalTo(expectedReplicas));
}
Also used : OpenSearchRejectedExecutionException(org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException) ShardInfo(org.opensearch.action.support.replication.ReplicationResponse.ShardInfo) TestThreadPool(org.opensearch.threadpool.TestThreadPool) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) Version(org.opensearch.Version) CircuitBreaker(org.opensearch.common.breaker.CircuitBreaker) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) ConcurrentCollections(org.opensearch.common.util.concurrent.ConcurrentCollections) InetAddress(java.net.InetAddress) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) NodeClosedException(org.opensearch.node.NodeClosedException) ActionListener(org.opensearch.action.ActionListener) UnavailableShardsException(org.opensearch.action.UnavailableShardsException) ShardStateAction(org.opensearch.cluster.action.shard.ShardStateAction) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) TimeValue(org.opensearch.common.unit.TimeValue) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) RemoteTransportException(org.opensearch.transport.RemoteTransportException) Set(java.util.Set) ClusterStateCreationUtils.state(org.opensearch.action.support.replication.ClusterStateCreationUtils.state) TransportAddress(org.opensearch.common.transport.TransportAddress) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Matchers.equalTo(org.hamcrest.Matchers.equalTo) ConnectTransportException(org.opensearch.transport.ConnectTransportException) Matchers.arrayWithSize(org.hamcrest.Matchers.arrayWithSize) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) IndexShardState(org.opensearch.index.shard.IndexShardState) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ThreadPool(org.opensearch.threadpool.ThreadPool) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) CircuitBreakingException(org.opensearch.common.breaker.CircuitBreakingException) IndexShardNotStartedException(org.opensearch.index.shard.IndexShardNotStartedException) ClusterStateCreationUtils.stateWithActivePrimary(org.opensearch.action.support.replication.ClusterStateCreationUtils.stateWithActivePrimary) ActiveShardCount(org.opensearch.action.support.ActiveShardCount) SendRequestTransportException(org.opensearch.transport.SendRequestTransportException) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ShardId(org.opensearch.index.shard.ShardId) ExecutionException(java.util.concurrent.ExecutionException) Sets(org.opensearch.common.util.set.Sets) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) Collections(java.util.Collections) ClusterState(org.opensearch.cluster.ClusterState) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) AtomicReference(java.util.concurrent.atomic.AtomicReference) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) ShardId(org.opensearch.index.shard.ShardId) ActionListener(org.opensearch.action.ActionListener) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) ShardRouting(org.opensearch.cluster.routing.ShardRouting) HashSet(java.util.HashSet)

Example 32 with IndexShardRoutingTable

use of org.opensearch.cluster.routing.IndexShardRoutingTable in project OpenSearch by opensearch-project.

the class ReplicationOperationTests method testWaitForActiveShards.

public void testWaitForActiveShards() throws Exception {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    final int assignedReplicas = randomInt(2);
    final int unassignedReplicas = randomInt(2);
    final int totalShards = 1 + assignedReplicas + unassignedReplicas;
    final int activeShardCount = randomIntBetween(0, totalShards);
    Request request = new Request(shardId).waitForActiveShards(activeShardCount == totalShards ? ActiveShardCount.ALL : ActiveShardCount.from(activeShardCount));
    final boolean passesActiveShardCheck = activeShardCount <= assignedReplicas + 1;
    ShardRoutingState[] replicaStates = new ShardRoutingState[assignedReplicas + unassignedReplicas];
    for (int i = 0; i < assignedReplicas; i++) {
        replicaStates[i] = randomFrom(ShardRoutingState.STARTED, ShardRoutingState.RELOCATING);
    }
    for (int i = assignedReplicas; i < replicaStates.length; i++) {
        replicaStates[i] = ShardRoutingState.UNASSIGNED;
    }
    final ClusterState state = state(index, true, ShardRoutingState.STARTED, replicaStates);
    logger.debug("using active shard count of [{}], assigned shards [{}], total shards [{}]." + " expecting op to [{}]. using state: \n{}", request.waitForActiveShards(), 1 + assignedReplicas, 1 + assignedReplicas + unassignedReplicas, passesActiveShardCheck ? "succeed" : "retry", state);
    final long primaryTerm = state.metadata().index(index).primaryTerm(shardId.id());
    final IndexShardRoutingTable shardRoutingTable = state.routingTable().index(index).shard(shardId.id());
    final Set<String> inSyncAllocationIds = state.metadata().index(index).inSyncAllocationIds(0);
    Set<String> trackedShards = new HashSet<>();
    addTrackingInfo(shardRoutingTable, null, trackedShards, new HashSet<>());
    final ReplicationGroup initialReplicationGroup = new ReplicationGroup(shardRoutingTable, inSyncAllocationIds, trackedShards, 0);
    PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
    final ShardRouting primaryShard = shardRoutingTable.primaryShard();
    final TestReplicationOperation op = new TestReplicationOperation(request, new TestPrimary(primaryShard, () -> initialReplicationGroup, threadPool), listener, new TestReplicaProxy(), logger, threadPool, "test", primaryTerm);
    if (passesActiveShardCheck) {
        assertThat(op.checkActiveShardCount(), nullValue());
        op.execute();
        assertTrue("operations should have been performed, active shard count is met", request.processedOnPrimary.get());
    } else {
        assertThat(op.checkActiveShardCount(), notNullValue());
        op.execute();
        assertFalse("operations should not have been perform, active shard count is *NOT* met", request.processedOnPrimary.get());
        assertListenerThrows("should throw exception to trigger retry", listener, UnavailableShardsException.class);
    }
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) ShardId(org.opensearch.index.shard.ShardId) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) ShardRouting(org.opensearch.cluster.routing.ShardRouting) HashSet(java.util.HashSet)

Example 33 with IndexShardRoutingTable

use of org.opensearch.cluster.routing.IndexShardRoutingTable in project OpenSearch by opensearch-project.

the class TransportReplicationActionTests method testReplicaProxy.

public void testReplicaProxy() throws InterruptedException, ExecutionException {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    ClusterState state = stateWithActivePrimary(index, true, 1 + randomInt(3), randomInt(2));
    logger.info("using state: {}", state);
    setState(clusterService, state);
    final long primaryTerm = state.metadata().index(index).primaryTerm(0);
    ReplicationOperation.Replicas<Request> proxy = action.newReplicasProxy();
    // check that at unknown node fails
    PlainActionFuture<ReplicaResponse> listener = new PlainActionFuture<>();
    ShardRoutingState routingState = randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.STARTED, ShardRoutingState.RELOCATING);
    proxy.performOn(TestShardRouting.newShardRouting(shardId, "NOT THERE", routingState == ShardRoutingState.RELOCATING ? state.nodes().iterator().next().getId() : null, false, routingState), new Request(NO_SHARD_ID), primaryTerm, randomNonNegativeLong(), randomNonNegativeLong(), listener);
    assertTrue(listener.isDone());
    assertListenerThrows("non existent node should throw a NoNodeAvailableException", listener, NoNodeAvailableException.class);
    final IndexShardRoutingTable shardRoutings = state.routingTable().shardRoutingTable(shardId);
    final ShardRouting replica = randomFrom(shardRoutings.replicaShards().stream().filter(ShardRouting::assignedToNode).collect(Collectors.toList()));
    listener = new PlainActionFuture<>();
    proxy.performOn(replica, new Request(NO_SHARD_ID), primaryTerm, randomNonNegativeLong(), randomNonNegativeLong(), listener);
    assertFalse(listener.isDone());
    CapturingTransport.CapturedRequest[] captures = transport.getCapturedRequestsAndClear();
    assertThat(captures, arrayWithSize(1));
    if (randomBoolean()) {
        final TransportReplicationAction.ReplicaResponse response = new TransportReplicationAction.ReplicaResponse(randomLong(), randomLong());
        transport.handleResponse(captures[0].requestId, response);
        assertTrue(listener.isDone());
        assertThat(listener.get(), equalTo(response));
    } else if (randomBoolean()) {
        transport.handleRemoteError(captures[0].requestId, new OpenSearchException("simulated"));
        assertTrue(listener.isDone());
        assertListenerThrows("listener should reflect remote error", listener, OpenSearchException.class);
    } else {
        transport.handleError(captures[0].requestId, new TransportException("simulated"));
        assertTrue(listener.isDone());
        assertListenerThrows("listener should reflect remote error", listener, TransportException.class);
    }
    AtomicReference<Object> failure = new AtomicReference<>();
    AtomicBoolean success = new AtomicBoolean();
    proxy.failShardIfNeeded(replica, primaryTerm, "test", new OpenSearchException("simulated"), ActionListener.wrap(r -> success.set(true), failure::set));
    CapturingTransport.CapturedRequest[] shardFailedRequests = transport.getCapturedRequestsAndClear();
    // A replication action doesn't not fail the request
    assertEquals(0, shardFailedRequests.length);
}
Also used : Matchers.hasToString(org.hamcrest.Matchers.hasToString) Metadata(org.opensearch.cluster.metadata.Metadata) AllocationService(org.opensearch.cluster.routing.allocation.AllocationService) TestThreadPool(org.opensearch.threadpool.TestThreadPool) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) ClusterStateChanges(org.opensearch.indices.cluster.ClusterStateChanges) Version(org.opensearch.Version) ClusterServiceUtils.setState(org.opensearch.test.ClusterServiceUtils.setState) NoNodeAvailableException(org.opensearch.client.transport.NoNodeAvailableException) Transport(org.opensearch.transport.Transport) Mockito.doThrow(org.mockito.Mockito.doThrow) ClusterBlock(org.opensearch.cluster.block.ClusterBlock) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) OpenSearchAllocationTestCase(org.opensearch.cluster.OpenSearchAllocationTestCase) Matchers.nullValue(org.hamcrest.Matchers.nullValue) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Is.is(org.hamcrest.core.Is.is) ActionListener(org.opensearch.action.ActionListener) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest) ShardStateAction(org.opensearch.cluster.action.shard.ShardStateAction) EnumSet(java.util.EnumSet) AfterClass(org.junit.AfterClass) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) Index(org.opensearch.index.Index) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) TransportService(org.opensearch.transport.TransportService) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) ActionFilters(org.opensearch.action.support.ActionFilters) ActionTestUtils(org.opensearch.action.support.ActionTestUtils) RoutingNode(org.opensearch.cluster.routing.RoutingNode) TransportException(org.opensearch.transport.TransportException) Mockito.eq(org.mockito.Mockito.eq) Mockito.mock(org.mockito.Mockito.mock) Matchers.arrayWithSize(org.hamcrest.Matchers.arrayWithSize) IndexShardState(org.opensearch.index.shard.IndexShardState) ThreadPool(org.opensearch.threadpool.ThreadPool) Releasable(org.opensearch.common.lease.Releasable) MockTransportService(org.opensearch.test.transport.MockTransportService) ClusterState(org.opensearch.cluster.ClusterState) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) Mockito.anyLong(org.mockito.Mockito.anyLong) ClusterBlocks(org.opensearch.cluster.block.ClusterBlocks) Mockito.anyString(org.mockito.Mockito.anyString) Before(org.junit.Before) MockNioTransport(org.opensearch.transport.nio.MockNioTransport) IOException(java.io.IOException) ShardNotFoundException(org.opensearch.index.shard.ShardNotFoundException) IndexService(org.opensearch.index.IndexService) SETTING_WAIT_FOR_ACTIVE_SHARDS(org.opensearch.cluster.metadata.IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS) ExecutionException(java.util.concurrent.ExecutionException) Matcher(org.hamcrest.Matcher) ClusterService(org.opensearch.cluster.service.ClusterService) NetworkService(org.opensearch.common.network.NetworkService) Mockito.anyInt(org.mockito.Mockito.anyInt) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) Assert(org.junit.Assert) PageCacheRecycler(org.opensearch.common.util.PageCacheRecycler) NoneCircuitBreakerService(org.opensearch.indices.breaker.NoneCircuitBreakerService) OpenSearchException(org.opensearch.OpenSearchException) TestTransportChannel(org.opensearch.transport.TestTransportChannel) ReplicaResponse(org.opensearch.action.support.replication.ReplicationOperation.ReplicaResponse) Collections.singleton(java.util.Collections.singleton) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) After(org.junit.After) UnavailableShardsException(org.opensearch.action.UnavailableShardsException) CloseIndexRequest(org.opensearch.action.admin.indices.close.CloseIndexRequest) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) TransportChannel(org.opensearch.transport.TransportChannel) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) IndicesService(org.opensearch.indices.IndicesService) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IndexShardClosedException(org.opensearch.index.shard.IndexShardClosedException) TransportResponse(org.opensearch.transport.TransportResponse) RestStatus(org.opensearch.rest.RestStatus) ClusterStateCreationUtils.state(org.opensearch.action.support.replication.ClusterStateCreationUtils.state) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) List(java.util.List) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Mockito.any(org.mockito.Mockito.any) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) BeforeClass(org.junit.BeforeClass) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ShardNotInPrimaryModeException(org.opensearch.index.shard.ShardNotInPrimaryModeException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) StreamOutput(org.opensearch.common.io.stream.StreamOutput) AtomicReference(java.util.concurrent.atomic.AtomicReference) NamedWriteableRegistry(org.opensearch.common.io.stream.NamedWriteableRegistry) HashSet(java.util.HashSet) IndexShard(org.opensearch.index.shard.IndexShard) IndexClosedException(org.opensearch.indices.IndexClosedException) StreamInput(org.opensearch.common.io.stream.StreamInput) ClusterServiceUtils.createClusterService(org.opensearch.test.ClusterServiceUtils.createClusterService) TransportRequest(org.opensearch.transport.TransportRequest) AllocationId(org.opensearch.cluster.routing.AllocationId) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ClusterStateCreationUtils.stateWithActivePrimary(org.opensearch.action.support.replication.ClusterStateCreationUtils.stateWithActivePrimary) Mockito.when(org.mockito.Mockito.when) ActiveShardCount(org.opensearch.action.support.ActiveShardCount) Mockito.verify(org.mockito.Mockito.verify) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ShardId(org.opensearch.index.shard.ShardId) TimeUnit(java.util.concurrent.TimeUnit) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting) CapturingTransport(org.opensearch.test.transport.CapturingTransport) Collections(java.util.Collections) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) Matchers.hasToString(org.hamcrest.Matchers.hasToString) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Mockito.anyString(org.mockito.Mockito.anyString) ShardId(org.opensearch.index.shard.ShardId) ClusterState(org.opensearch.cluster.ClusterState) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest) CloseIndexRequest(org.opensearch.action.admin.indices.close.CloseIndexRequest) TransportRequest(org.opensearch.transport.TransportRequest) AtomicReference(java.util.concurrent.atomic.AtomicReference) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) TransportException(org.opensearch.transport.TransportException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) ReplicaResponse(org.opensearch.action.support.replication.ReplicationOperation.ReplicaResponse) OpenSearchException(org.opensearch.OpenSearchException) ShardRouting(org.opensearch.cluster.routing.ShardRouting) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting)

Example 34 with IndexShardRoutingTable

use of org.opensearch.cluster.routing.IndexShardRoutingTable in project OpenSearch by opensearch-project.

the class AutoExpandReplicasTests method testAutoExpandWhenNodeLeavesAndPossiblyRejoins.

/**
 * Checks that when nodes leave the cluster that the auto-expand-replica functionality only triggers after failing the shards on
 * the removed nodes. This ensures that active shards on other live nodes are not failed if the primary resided on a now dead node.
 * Instead, one of the replicas on the live nodes first gets promoted to primary, and the auto-expansion (removing replicas) only
 * triggers in a follow-up step.
 */
public void testAutoExpandWhenNodeLeavesAndPossiblyRejoins() throws InterruptedException {
    final ThreadPool threadPool = new TestThreadPool(getClass().getName());
    final ClusterStateChanges cluster = new ClusterStateChanges(xContentRegistry(), threadPool);
    try {
        List<DiscoveryNode> allNodes = new ArrayList<>();
        // local node is the cluster-manager
        DiscoveryNode localNode = createNode(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE);
        allNodes.add(localNode);
        int numDataNodes = randomIntBetween(3, 5);
        List<DiscoveryNode> dataNodes = new ArrayList<>(numDataNodes);
        for (int i = 0; i < numDataNodes; i++) {
            dataNodes.add(createNode(DiscoveryNodeRole.DATA_ROLE));
        }
        allNodes.addAll(dataNodes);
        ClusterState state = ClusterStateCreationUtils.state(localNode, localNode, allNodes.toArray(new DiscoveryNode[0]));
        CreateIndexRequest request = new CreateIndexRequest("index", Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_AUTO_EXPAND_REPLICAS, "0-all").build()).waitForActiveShards(ActiveShardCount.NONE);
        state = cluster.createIndex(state, request);
        assertTrue(state.metadata().hasIndex("index"));
        while (state.routingTable().index("index").shard(0).allShardsStarted() == false) {
            logger.info(state);
            state = cluster.applyStartedShards(state, state.routingTable().index("index").shard(0).shardsWithState(ShardRoutingState.INITIALIZING));
            state = cluster.reroute(state, new ClusterRerouteRequest());
        }
        IndexShardRoutingTable preTable = state.routingTable().index("index").shard(0);
        final Set<String> unchangedNodeIds;
        final IndexShardRoutingTable postTable;
        if (randomBoolean()) {
            // simulate node removal
            List<DiscoveryNode> nodesToRemove = randomSubsetOf(2, dataNodes);
            unchangedNodeIds = dataNodes.stream().filter(n -> nodesToRemove.contains(n) == false).map(DiscoveryNode::getId).collect(Collectors.toSet());
            state = cluster.removeNodes(state, nodesToRemove);
            postTable = state.routingTable().index("index").shard(0);
            assertTrue("not all shards started in " + state.toString(), postTable.allShardsStarted());
            assertThat(postTable.toString(), postTable.getAllAllocationIds(), everyItem(is(in(preTable.getAllAllocationIds()))));
        } else {
            // fake an election where conflicting nodes are removed and readded
            state = ClusterState.builder(state).nodes(DiscoveryNodes.builder(state.nodes()).masterNodeId(null).build()).build();
            List<DiscoveryNode> conflictingNodes = randomSubsetOf(2, dataNodes);
            unchangedNodeIds = dataNodes.stream().filter(n -> conflictingNodes.contains(n) == false).map(DiscoveryNode::getId).collect(Collectors.toSet());
            List<DiscoveryNode> nodesToAdd = conflictingNodes.stream().map(n -> new DiscoveryNode(n.getName(), n.getId(), buildNewFakeTransportAddress(), n.getAttributes(), n.getRoles(), n.getVersion())).collect(Collectors.toList());
            if (randomBoolean()) {
                nodesToAdd.add(createNode(DiscoveryNodeRole.DATA_ROLE));
            }
            state = cluster.joinNodesAndBecomeMaster(state, nodesToAdd);
            postTable = state.routingTable().index("index").shard(0);
        }
        Set<String> unchangedAllocationIds = preTable.getShards().stream().filter(shr -> unchangedNodeIds.contains(shr.currentNodeId())).map(shr -> shr.allocationId().getId()).collect(Collectors.toSet());
        assertThat(postTable.toString(), unchangedAllocationIds, everyItem(is(in(postTable.getAllAllocationIds()))));
        postTable.getShards().forEach(shardRouting -> {
            if (shardRouting.assignedToNode() && unchangedAllocationIds.contains(shardRouting.allocationId().getId())) {
                assertTrue("Shard should be active: " + shardRouting, shardRouting.active());
            }
        });
    } finally {
        terminate(threadPool);
    }
}
Also used : DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) UpdateSettingsRequest(org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest) SETTING_AUTO_EXPAND_REPLICAS(org.opensearch.cluster.metadata.IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS) ThreadPool(org.opensearch.threadpool.ThreadPool) TestThreadPool(org.opensearch.threadpool.TestThreadPool) ClusterStateChanges(org.opensearch.indices.cluster.ClusterStateChanges) Version(org.opensearch.Version) ClusterRerouteRequest(org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest) ClusterStateCreationUtils(org.opensearch.action.support.replication.ClusterStateCreationUtils) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) Matchers.everyItem(org.hamcrest.Matchers.everyItem) VersionUtils(org.opensearch.test.VersionUtils) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) DiscoveryNodeRole(org.opensearch.cluster.node.DiscoveryNodeRole) Collectors(java.util.stream.Collectors) ActiveShardCount(org.opensearch.action.support.ActiveShardCount) List(java.util.List) Matchers.equalTo(org.hamcrest.Matchers.equalTo) SETTING_NUMBER_OF_SHARDS(org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS) Matchers.is(org.hamcrest.Matchers.is) Collections(java.util.Collections) Matchers.in(org.hamcrest.Matchers.in) ClusterState(org.opensearch.cluster.ClusterState) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) ThreadPool(org.opensearch.threadpool.ThreadPool) TestThreadPool(org.opensearch.threadpool.TestThreadPool) ArrayList(java.util.ArrayList) TestThreadPool(org.opensearch.threadpool.TestThreadPool) ClusterStateChanges(org.opensearch.indices.cluster.ClusterStateChanges) ClusterRerouteRequest(org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest)

Example 35 with IndexShardRoutingTable

use of org.opensearch.cluster.routing.IndexShardRoutingTable in project OpenSearch by opensearch-project.

the class DecisionsImpactOnClusterHealthTests method testPrimaryShardYesDecisionOnIndexCreation.

public void testPrimaryShardYesDecisionOnIndexCreation() throws IOException {
    final String indexName = "test-idx";
    Settings settings = Settings.builder().put(Environment.PATH_HOME_SETTING.getKey(), createTempDir().toAbsolutePath().toString()).build();
    AllocationDecider decider = new TestAllocateDecision(Decision.YES) {

        @Override
        public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation) {
            if (node.getByShardId(shardRouting.shardId()) == null) {
                return Decision.YES;
            } else {
                return Decision.NO;
            }
        }
    };
    // if deciders say YES to allocating primary shards, stay in YELLOW state
    ClusterState clusterState = runAllocationTest(settings, indexName, Collections.singleton(decider), ClusterHealthStatus.YELLOW);
    // make sure primaries are initialized
    RoutingTable routingTable = clusterState.routingTable();
    for (IndexShardRoutingTable indexShardRoutingTable : routingTable.index(indexName)) {
        assertTrue(indexShardRoutingTable.primaryShard().initializing());
    }
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) RoutingNode(org.opensearch.cluster.routing.RoutingNode) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) RoutingTable(org.opensearch.cluster.routing.RoutingTable) ShardRouting(org.opensearch.cluster.routing.ShardRouting) Settings(org.opensearch.common.settings.Settings) AllocationDecider(org.opensearch.cluster.routing.allocation.decider.AllocationDecider)

Aggregations

IndexShardRoutingTable (org.opensearch.cluster.routing.IndexShardRoutingTable)86 ShardRouting (org.opensearch.cluster.routing.ShardRouting)61 ClusterState (org.opensearch.cluster.ClusterState)50 ShardId (org.opensearch.index.shard.ShardId)37 IndexMetadata (org.opensearch.cluster.metadata.IndexMetadata)32 IndexRoutingTable (org.opensearch.cluster.routing.IndexRoutingTable)27 HashSet (java.util.HashSet)22 Settings (org.opensearch.common.settings.Settings)22 PlainActionFuture (org.opensearch.action.support.PlainActionFuture)18 RoutingTable (org.opensearch.cluster.routing.RoutingTable)18 ArrayList (java.util.ArrayList)17 List (java.util.List)17 Matchers.equalTo (org.hamcrest.Matchers.equalTo)17 ShardRoutingState (org.opensearch.cluster.routing.ShardRoutingState)17 Collections (java.util.Collections)16 Collectors (java.util.stream.Collectors)15 ActionListener (org.opensearch.action.ActionListener)15 Set (java.util.Set)14 IndexShard (org.opensearch.index.shard.IndexShard)14 IOException (java.io.IOException)13