Search in sources :

Example 1 with ReplicationGroup

use of org.opensearch.index.shard.ReplicationGroup in project OpenSearch by opensearch-project.

the class TransportVerifyShardBeforeCloseActionTests method testUnavailableShardsMarkedAsStale.

public void testUnavailableShardsMarkedAsStale() throws Exception {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    final int nbReplicas = randomIntBetween(1, 10);
    final ShardRoutingState[] replicaStates = new ShardRoutingState[nbReplicas];
    for (int i = 0; i < replicaStates.length; i++) {
        replicaStates[i] = ShardRoutingState.STARTED;
    }
    final ClusterState clusterState = state(index, true, ShardRoutingState.STARTED, replicaStates);
    setState(clusterService, clusterState);
    IndexShardRoutingTable shardRoutingTable = clusterState.routingTable().index(index).shard(shardId.id());
    final IndexMetadata indexMetadata = clusterState.getMetadata().index(index);
    final ShardRouting primaryRouting = shardRoutingTable.primaryShard();
    final long primaryTerm = indexMetadata.primaryTerm(0);
    final Set<String> inSyncAllocationIds = indexMetadata.inSyncAllocationIds(0);
    final Set<String> trackedShards = shardRoutingTable.getAllAllocationIds();
    List<ShardRouting> unavailableShards = randomSubsetOf(randomIntBetween(1, nbReplicas), shardRoutingTable.replicaShards());
    IndexShardRoutingTable.Builder shardRoutingTableBuilder = new IndexShardRoutingTable.Builder(shardRoutingTable);
    unavailableShards.forEach(shardRoutingTableBuilder::removeShard);
    shardRoutingTable = shardRoutingTableBuilder.build();
    final ReplicationGroup replicationGroup = new ReplicationGroup(shardRoutingTable, inSyncAllocationIds, trackedShards, 0);
    assertThat(replicationGroup.getUnavailableInSyncShards().size(), greaterThan(0));
    final PlainActionFuture<PrimaryResult> listener = new PlainActionFuture<>();
    TaskId taskId = new TaskId(clusterService.localNode().getId(), 0L);
    TransportVerifyShardBeforeCloseAction.ShardRequest request = new TransportVerifyShardBeforeCloseAction.ShardRequest(shardId, clusterBlock, false, taskId);
    ReplicationOperation.Replicas<TransportVerifyShardBeforeCloseAction.ShardRequest> proxy = action.newReplicasProxy();
    ReplicationOperation<TransportVerifyShardBeforeCloseAction.ShardRequest, TransportVerifyShardBeforeCloseAction.ShardRequest, PrimaryResult> operation = new ReplicationOperation<>(request, createPrimary(primaryRouting, replicationGroup), listener, proxy, logger, threadPool, "test", primaryTerm, TimeValue.timeValueMillis(20), TimeValue.timeValueSeconds(60));
    operation.execute();
    final CapturingTransport.CapturedRequest[] capturedRequests = transport.getCapturedRequestsAndClear();
    assertThat(capturedRequests.length, equalTo(nbReplicas));
    for (CapturingTransport.CapturedRequest capturedRequest : capturedRequests) {
        final String actionName = capturedRequest.action;
        if (actionName.startsWith(ShardStateAction.SHARD_FAILED_ACTION_NAME)) {
            assertThat(capturedRequest.request, instanceOf(ShardStateAction.FailedShardEntry.class));
            String allocationId = ((ShardStateAction.FailedShardEntry) capturedRequest.request).getAllocationId();
            assertTrue(unavailableShards.stream().anyMatch(shardRouting -> shardRouting.allocationId().getId().equals(allocationId)));
            transport.handleResponse(capturedRequest.requestId, TransportResponse.Empty.INSTANCE);
        } else if (actionName.startsWith(TransportVerifyShardBeforeCloseAction.NAME)) {
            assertThat(capturedRequest.request, instanceOf(ConcreteShardRequest.class));
            String allocationId = ((ConcreteShardRequest) capturedRequest.request).getTargetAllocationID();
            assertFalse(unavailableShards.stream().anyMatch(shardRouting -> shardRouting.allocationId().getId().equals(allocationId)));
            assertTrue(inSyncAllocationIds.stream().anyMatch(inSyncAllocationId -> inSyncAllocationId.equals(allocationId)));
            transport.handleResponse(capturedRequest.requestId, new TransportReplicationAction.ReplicaResponse(0L, 0L));
        } else {
            fail("Test does not support action " + capturedRequest.action);
        }
    }
    final ReplicationResponse.ShardInfo shardInfo = listener.get().getShardInfo();
    assertThat(shardInfo.getFailed(), equalTo(0));
    assertThat(shardInfo.getFailures(), arrayWithSize(0));
    assertThat(shardInfo.getSuccessful(), equalTo(1 + nbReplicas - unavailableShards.size()));
}
Also used : TestThreadPool(org.opensearch.threadpool.TestThreadPool) ClusterServiceUtils.setState(org.opensearch.test.ClusterServiceUtils.setState) Mockito.doThrow(org.mockito.Mockito.doThrow) ClusterBlock(org.opensearch.cluster.block.ClusterBlock) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) After(org.junit.After) ActionListener(org.opensearch.action.ActionListener) ShardStateAction(org.opensearch.cluster.action.shard.ShardStateAction) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) AfterClass(org.junit.AfterClass) TimeValue(org.opensearch.common.unit.TimeValue) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) IndicesService(org.opensearch.indices.IndicesService) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) TransportResponse(org.opensearch.transport.TransportResponse) Mockito.doNothing(org.mockito.Mockito.doNothing) ClusterStateCreationUtils.state(org.opensearch.action.support.replication.ClusterStateCreationUtils.state) TransportService(org.opensearch.transport.TransportService) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) ActionFilters(org.opensearch.action.support.ActionFilters) List(java.util.List) Matchers.equalTo(org.hamcrest.Matchers.equalTo) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Mockito.any(org.mockito.Mockito.any) Mockito.mock(org.mockito.Mockito.mock) Matchers.arrayWithSize(org.hamcrest.Matchers.arrayWithSize) BeforeClass(org.junit.BeforeClass) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) FlushRequest(org.opensearch.action.admin.indices.flush.FlushRequest) ThreadPool(org.opensearch.threadpool.ThreadPool) ConcreteShardRequest(org.opensearch.action.support.replication.TransportReplicationAction.ConcreteShardRequest) ClusterState(org.opensearch.cluster.ClusterState) IndexShard(org.opensearch.index.shard.IndexShard) ArgumentCaptor(org.mockito.ArgumentCaptor) ReplicationOperation(org.opensearch.action.support.replication.ReplicationOperation) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) ClusterBlocks(org.opensearch.cluster.block.ClusterBlocks) MetadataIndexStateService(org.opensearch.cluster.metadata.MetadataIndexStateService) Before(org.junit.Before) ClusterServiceUtils.createClusterService(org.opensearch.test.ClusterServiceUtils.createClusterService) SetOnce(org.apache.lucene.util.SetOnce) PendingReplicationActions(org.opensearch.action.support.replication.PendingReplicationActions) TaskId(org.opensearch.tasks.TaskId) Mockito.times(org.mockito.Mockito.times) Mockito.when(org.mockito.Mockito.when) TransportReplicationAction(org.opensearch.action.support.replication.TransportReplicationAction) Mockito.verify(org.mockito.Mockito.verify) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ShardId(org.opensearch.index.shard.ShardId) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) ClusterService(org.opensearch.cluster.service.ClusterService) ClusterName(org.opensearch.cluster.ClusterName) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) CapturingTransport(org.opensearch.test.transport.CapturingTransport) Collections(java.util.Collections) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) TaskId(org.opensearch.tasks.TaskId) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) ReplicationResponse(org.opensearch.action.support.replication.ReplicationResponse) ShardId(org.opensearch.index.shard.ShardId) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ReplicationOperation(org.opensearch.action.support.replication.ReplicationOperation) ClusterState(org.opensearch.cluster.ClusterState) CapturingTransport(org.opensearch.test.transport.CapturingTransport) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) ConcreteShardRequest(org.opensearch.action.support.replication.TransportReplicationAction.ConcreteShardRequest) ShardRouting(org.opensearch.cluster.routing.ShardRouting)

Example 2 with ReplicationGroup

use of org.opensearch.index.shard.ReplicationGroup in project OpenSearch by opensearch-project.

the class ReplicationOperationTests method testWaitForActiveShards.

public void testWaitForActiveShards() throws Exception {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    final int assignedReplicas = randomInt(2);
    final int unassignedReplicas = randomInt(2);
    final int totalShards = 1 + assignedReplicas + unassignedReplicas;
    final int activeShardCount = randomIntBetween(0, totalShards);
    Request request = new Request(shardId).waitForActiveShards(activeShardCount == totalShards ? ActiveShardCount.ALL : ActiveShardCount.from(activeShardCount));
    final boolean passesActiveShardCheck = activeShardCount <= assignedReplicas + 1;
    ShardRoutingState[] replicaStates = new ShardRoutingState[assignedReplicas + unassignedReplicas];
    for (int i = 0; i < assignedReplicas; i++) {
        replicaStates[i] = randomFrom(ShardRoutingState.STARTED, ShardRoutingState.RELOCATING);
    }
    for (int i = assignedReplicas; i < replicaStates.length; i++) {
        replicaStates[i] = ShardRoutingState.UNASSIGNED;
    }
    final ClusterState state = state(index, true, ShardRoutingState.STARTED, replicaStates);
    logger.debug("using active shard count of [{}], assigned shards [{}], total shards [{}]." + " expecting op to [{}]. using state: \n{}", request.waitForActiveShards(), 1 + assignedReplicas, 1 + assignedReplicas + unassignedReplicas, passesActiveShardCheck ? "succeed" : "retry", state);
    final long primaryTerm = state.metadata().index(index).primaryTerm(shardId.id());
    final IndexShardRoutingTable shardRoutingTable = state.routingTable().index(index).shard(shardId.id());
    final Set<String> inSyncAllocationIds = state.metadata().index(index).inSyncAllocationIds(0);
    Set<String> trackedShards = new HashSet<>();
    addTrackingInfo(shardRoutingTable, null, trackedShards, new HashSet<>());
    final ReplicationGroup initialReplicationGroup = new ReplicationGroup(shardRoutingTable, inSyncAllocationIds, trackedShards, 0);
    PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
    final ShardRouting primaryShard = shardRoutingTable.primaryShard();
    final TestReplicationOperation op = new TestReplicationOperation(request, new TestPrimary(primaryShard, () -> initialReplicationGroup, threadPool), listener, new TestReplicaProxy(), logger, threadPool, "test", primaryTerm);
    if (passesActiveShardCheck) {
        assertThat(op.checkActiveShardCount(), nullValue());
        op.execute();
        assertTrue("operations should have been performed, active shard count is met", request.processedOnPrimary.get());
    } else {
        assertThat(op.checkActiveShardCount(), notNullValue());
        op.execute();
        assertFalse("operations should not have been perform, active shard count is *NOT* met", request.processedOnPrimary.get());
        assertListenerThrows("should throw exception to trigger retry", listener, UnavailableShardsException.class);
    }
}
Also used : ClusterState(org.opensearch.cluster.ClusterState) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) ShardId(org.opensearch.index.shard.ShardId) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) ShardRouting(org.opensearch.cluster.routing.ShardRouting) HashSet(java.util.HashSet)

Example 3 with ReplicationGroup

use of org.opensearch.index.shard.ReplicationGroup in project OpenSearch by opensearch-project.

the class ReplicationOperationTests method testAddedReplicaAfterPrimaryOperation.

public void testAddedReplicaAfterPrimaryOperation() throws Exception {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    final ClusterState initialState = stateWithActivePrimary(index, true, 0);
    Set<String> inSyncAllocationIds = initialState.metadata().index(index).inSyncAllocationIds(0);
    IndexShardRoutingTable shardRoutingTable = initialState.getRoutingTable().shardRoutingTable(shardId);
    Set<String> trackedShards = new HashSet<>();
    addTrackingInfo(shardRoutingTable, null, trackedShards, new HashSet<>());
    ReplicationGroup initialReplicationGroup = new ReplicationGroup(shardRoutingTable, inSyncAllocationIds, trackedShards, 0);
    final ClusterState stateWithAddedReplicas;
    if (randomBoolean()) {
        stateWithAddedReplicas = state(index, true, ShardRoutingState.STARTED, randomBoolean() ? ShardRoutingState.INITIALIZING : ShardRoutingState.STARTED);
    } else {
        stateWithAddedReplicas = state(index, true, ShardRoutingState.RELOCATING);
    }
    inSyncAllocationIds = stateWithAddedReplicas.metadata().index(index).inSyncAllocationIds(0);
    shardRoutingTable = stateWithAddedReplicas.getRoutingTable().shardRoutingTable(shardId);
    trackedShards = new HashSet<>();
    addTrackingInfo(shardRoutingTable, null, trackedShards, new HashSet<>());
    ReplicationGroup updatedReplicationGroup = new ReplicationGroup(shardRoutingTable, inSyncAllocationIds, trackedShards, 0);
    final AtomicReference<ReplicationGroup> replicationGroup = new AtomicReference<>(initialReplicationGroup);
    logger.debug("--> using initial replicationGroup:\n{}", replicationGroup.get());
    final long primaryTerm = initialState.getMetadata().index(shardId.getIndexName()).primaryTerm(shardId.id());
    final ShardRouting primaryShard = updatedReplicationGroup.getRoutingTable().primaryShard();
    final TestPrimary primary = new TestPrimary(primaryShard, replicationGroup::get, threadPool) {

        @Override
        public void perform(Request request, ActionListener<Result> listener) {
            super.perform(request, ActionListener.map(listener, result -> {
                replicationGroup.set(updatedReplicationGroup);
                logger.debug("--> state after primary operation:\n{}", replicationGroup.get());
                return result;
            }));
        }
    };
    Request request = new Request(shardId);
    PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
    final TestReplicationOperation op = new TestReplicationOperation(request, primary, listener, new TestReplicaProxy(), primaryTerm);
    op.execute();
    assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true));
    Set<ShardRouting> expectedReplicas = getExpectedReplicas(shardId, stateWithAddedReplicas, trackedShards);
    assertThat(request.processedOnReplicas, equalTo(expectedReplicas));
}
Also used : OpenSearchRejectedExecutionException(org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException) ShardInfo(org.opensearch.action.support.replication.ReplicationResponse.ShardInfo) TestThreadPool(org.opensearch.threadpool.TestThreadPool) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) Version(org.opensearch.Version) CircuitBreaker(org.opensearch.common.breaker.CircuitBreaker) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) ConcurrentCollections(org.opensearch.common.util.concurrent.ConcurrentCollections) InetAddress(java.net.InetAddress) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) NodeClosedException(org.opensearch.node.NodeClosedException) ActionListener(org.opensearch.action.ActionListener) UnavailableShardsException(org.opensearch.action.UnavailableShardsException) ShardStateAction(org.opensearch.cluster.action.shard.ShardStateAction) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) TimeValue(org.opensearch.common.unit.TimeValue) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) RemoteTransportException(org.opensearch.transport.RemoteTransportException) Set(java.util.Set) ClusterStateCreationUtils.state(org.opensearch.action.support.replication.ClusterStateCreationUtils.state) TransportAddress(org.opensearch.common.transport.TransportAddress) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Matchers.equalTo(org.hamcrest.Matchers.equalTo) ConnectTransportException(org.opensearch.transport.ConnectTransportException) Matchers.arrayWithSize(org.hamcrest.Matchers.arrayWithSize) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) IndexShardState(org.opensearch.index.shard.IndexShardState) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ThreadPool(org.opensearch.threadpool.ThreadPool) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) CircuitBreakingException(org.opensearch.common.breaker.CircuitBreakingException) IndexShardNotStartedException(org.opensearch.index.shard.IndexShardNotStartedException) ClusterStateCreationUtils.stateWithActivePrimary(org.opensearch.action.support.replication.ClusterStateCreationUtils.stateWithActivePrimary) ActiveShardCount(org.opensearch.action.support.ActiveShardCount) SendRequestTransportException(org.opensearch.transport.SendRequestTransportException) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ShardId(org.opensearch.index.shard.ShardId) ExecutionException(java.util.concurrent.ExecutionException) Sets(org.opensearch.common.util.set.Sets) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) Collections(java.util.Collections) ClusterState(org.opensearch.cluster.ClusterState) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) AtomicReference(java.util.concurrent.atomic.AtomicReference) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) ShardId(org.opensearch.index.shard.ShardId) ActionListener(org.opensearch.action.ActionListener) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) ShardRouting(org.opensearch.cluster.routing.ShardRouting) HashSet(java.util.HashSet)

Example 4 with ReplicationGroup

use of org.opensearch.index.shard.ReplicationGroup in project OpenSearch by opensearch-project.

the class ReplicationOperationTests method testPrimaryFailureHandlingReplicaResponse.

public void testPrimaryFailureHandlingReplicaResponse() throws Exception {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    final Request request = new Request(shardId);
    final ClusterState state = stateWithActivePrimary(index, true, 1, 0);
    final IndexMetadata indexMetadata = state.getMetadata().index(index);
    final long primaryTerm = indexMetadata.primaryTerm(0);
    final ShardRouting primaryRouting = state.getRoutingTable().shardRoutingTable(shardId).primaryShard();
    final Set<String> inSyncAllocationIds = indexMetadata.inSyncAllocationIds(0);
    final IndexShardRoutingTable shardRoutingTable = state.routingTable().index(index).shard(shardId.id());
    final Set<String> trackedShards = shardRoutingTable.getAllAllocationIds();
    final ReplicationGroup initialReplicationGroup = new ReplicationGroup(shardRoutingTable, inSyncAllocationIds, trackedShards, 0);
    final boolean fatal = randomBoolean();
    final AtomicBoolean primaryFailed = new AtomicBoolean();
    final ReplicationOperation.Primary<Request, Request, TestPrimary.Result> primary = new TestPrimary(primaryRouting, () -> initialReplicationGroup, threadPool) {

        @Override
        public void failShard(String message, Exception exception) {
            primaryFailed.set(true);
        }

        @Override
        public void updateLocalCheckpointForShard(String allocationId, long checkpoint) {
            if (primaryRouting.allocationId().getId().equals(allocationId)) {
                super.updateLocalCheckpointForShard(allocationId, checkpoint);
            } else {
                if (fatal) {
                    throw new NullPointerException();
                } else {
                    throw new AlreadyClosedException("already closed");
                }
            }
        }
    };
    final PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
    final ReplicationOperation.Replicas<Request> replicas = new TestReplicaProxy(Collections.emptyMap());
    TestReplicationOperation operation = new TestReplicationOperation(request, primary, listener, replicas, primaryTerm);
    operation.execute();
    assertThat(primaryFailed.get(), equalTo(fatal));
    final ShardInfo shardInfo = listener.actionGet().getShardInfo();
    assertThat(shardInfo.getFailed(), equalTo(0));
    assertThat(shardInfo.getFailures(), arrayWithSize(0));
    assertThat(shardInfo.getSuccessful(), equalTo(1 + getExpectedReplicas(shardId, state, trackedShards).size()));
}
Also used : IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) ShardId(org.opensearch.index.shard.ShardId) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ShardInfo(org.opensearch.action.support.replication.ReplicationResponse.ShardInfo) ClusterState(org.opensearch.cluster.ClusterState) OpenSearchRejectedExecutionException(org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) NodeClosedException(org.opensearch.node.NodeClosedException) UnavailableShardsException(org.opensearch.action.UnavailableShardsException) RemoteTransportException(org.opensearch.transport.RemoteTransportException) ConnectTransportException(org.opensearch.transport.ConnectTransportException) CircuitBreakingException(org.opensearch.common.breaker.CircuitBreakingException) IndexShardNotStartedException(org.opensearch.index.shard.IndexShardNotStartedException) SendRequestTransportException(org.opensearch.transport.SendRequestTransportException) ExecutionException(java.util.concurrent.ExecutionException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) ShardRouting(org.opensearch.cluster.routing.ShardRouting)

Example 5 with ReplicationGroup

use of org.opensearch.index.shard.ReplicationGroup in project OpenSearch by opensearch-project.

the class ReplicationOperationTests method testNoLongerPrimary.

public void testNoLongerPrimary() throws Exception {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    ClusterState initialState = stateWithActivePrimary(index, true, 1 + randomInt(2), randomInt(2));
    IndexMetadata indexMetadata = initialState.getMetadata().index(index);
    final long primaryTerm = indexMetadata.primaryTerm(0);
    final IndexShardRoutingTable indexShardRoutingTable = initialState.getRoutingTable().shardRoutingTable(shardId);
    ShardRouting primaryShard = indexShardRoutingTable.primaryShard();
    if (primaryShard.relocating() && randomBoolean()) {
        // simulate execution of the replication phase on the relocation target node after relocation source was marked as relocated
        initialState = ClusterState.builder(initialState).nodes(DiscoveryNodes.builder(initialState.nodes()).localNodeId(primaryShard.relocatingNodeId())).build();
        primaryShard = primaryShard.getTargetRelocatingShard();
    }
    // add an in-sync allocation id that doesn't have a corresponding routing entry
    final Set<String> staleAllocationIds = Sets.newHashSet(randomAlphaOfLength(10));
    final Set<String> inSyncAllocationIds = Sets.union(indexMetadata.inSyncAllocationIds(0), staleAllocationIds);
    final Set<String> trackedShards = new HashSet<>();
    addTrackingInfo(indexShardRoutingTable, primaryShard, trackedShards, new HashSet<>());
    trackedShards.addAll(staleAllocationIds);
    final ReplicationGroup replicationGroup = new ReplicationGroup(indexShardRoutingTable, inSyncAllocationIds, trackedShards, 0);
    final Set<ShardRouting> expectedReplicas = getExpectedReplicas(shardId, initialState, trackedShards);
    final Map<ShardRouting, Exception> expectedFailures = new HashMap<>();
    if (expectedReplicas.isEmpty()) {
        return;
    }
    final ShardRouting failedReplica = randomFrom(new ArrayList<>(expectedReplicas));
    expectedFailures.put(failedReplica, new CorruptIndexException("simulated", (String) null));
    Request request = new Request(shardId);
    PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
    final boolean testPrimaryDemotedOnStaleShardCopies = randomBoolean();
    final Exception shardActionFailure;
    if (randomBoolean()) {
        shardActionFailure = new NodeClosedException(new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT));
    } else if (randomBoolean()) {
        DiscoveryNode node = new DiscoveryNode("foo", buildNewFakeTransportAddress(), Version.CURRENT);
        shardActionFailure = new SendRequestTransportException(node, ShardStateAction.SHARD_FAILED_ACTION_NAME, new NodeClosedException(node));
    } else {
        shardActionFailure = new ShardStateAction.NoLongerPrimaryShardException(failedReplica.shardId(), "the king is dead");
    }
    final TestReplicaProxy replicasProxy = new TestReplicaProxy(expectedFailures) {

        @Override
        public void failShardIfNeeded(ShardRouting replica, long primaryTerm, String message, Exception exception, ActionListener<Void> shardActionListener) {
            if (testPrimaryDemotedOnStaleShardCopies) {
                super.failShardIfNeeded(replica, primaryTerm, message, exception, shardActionListener);
            } else {
                assertThat(replica, equalTo(failedReplica));
                shardActionListener.onFailure(shardActionFailure);
            }
        }

        @Override
        public void markShardCopyAsStaleIfNeeded(ShardId shardId, String allocationId, long primaryTerm, ActionListener<Void> shardActionListener) {
            if (testPrimaryDemotedOnStaleShardCopies) {
                shardActionListener.onFailure(shardActionFailure);
            } else {
                super.markShardCopyAsStaleIfNeeded(shardId, allocationId, primaryTerm, shardActionListener);
            }
        }
    };
    AtomicBoolean primaryFailed = new AtomicBoolean();
    final TestPrimary primary = new TestPrimary(primaryShard, () -> replicationGroup, threadPool) {

        @Override
        public void failShard(String message, Exception exception) {
            assertThat(exception, instanceOf(ShardStateAction.NoLongerPrimaryShardException.class));
            assertTrue(primaryFailed.compareAndSet(false, true));
        }
    };
    final TestReplicationOperation op = new TestReplicationOperation(request, primary, listener, replicasProxy, primaryTerm);
    op.execute();
    assertThat("request was not processed on primary", request.processedOnPrimary.get(), equalTo(true));
    assertTrue("listener is not marked as done", listener.isDone());
    if (shardActionFailure instanceof ShardStateAction.NoLongerPrimaryShardException) {
        assertTrue(primaryFailed.get());
    } else {
        assertFalse(primaryFailed.get());
    }
    assertListenerThrows("should throw exception to trigger retry", listener, ReplicationOperation.RetryOnPrimaryException.class);
}
Also used : IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) HashMap(java.util.HashMap) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) ShardId(org.opensearch.index.shard.ShardId) SendRequestTransportException(org.opensearch.transport.SendRequestTransportException) NodeClosedException(org.opensearch.node.NodeClosedException) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) OpenSearchRejectedExecutionException(org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) NodeClosedException(org.opensearch.node.NodeClosedException) UnavailableShardsException(org.opensearch.action.UnavailableShardsException) RemoteTransportException(org.opensearch.transport.RemoteTransportException) ConnectTransportException(org.opensearch.transport.ConnectTransportException) CircuitBreakingException(org.opensearch.common.breaker.CircuitBreakingException) IndexShardNotStartedException(org.opensearch.index.shard.IndexShardNotStartedException) SendRequestTransportException(org.opensearch.transport.SendRequestTransportException) ExecutionException(java.util.concurrent.ExecutionException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ActionListener(org.opensearch.action.ActionListener) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) ShardRouting(org.opensearch.cluster.routing.ShardRouting)

Aggregations

ReplicationGroup (org.opensearch.index.shard.ReplicationGroup)13 ShardRouting (org.opensearch.cluster.routing.ShardRouting)11 PlainActionFuture (org.opensearch.action.support.PlainActionFuture)9 ClusterState (org.opensearch.cluster.ClusterState)9 IndexShardRoutingTable (org.opensearch.cluster.routing.IndexShardRoutingTable)9 ShardId (org.opensearch.index.shard.ShardId)9 ExecutionException (java.util.concurrent.ExecutionException)7 AlreadyClosedException (org.apache.lucene.store.AlreadyClosedException)7 ActionListener (org.opensearch.action.ActionListener)7 UnavailableShardsException (org.opensearch.action.UnavailableShardsException)7 IndexMetadata (org.opensearch.cluster.metadata.IndexMetadata)7 HashSet (java.util.HashSet)6 CircuitBreakingException (org.opensearch.common.breaker.CircuitBreakingException)6 OpenSearchRejectedExecutionException (org.opensearch.common.util.concurrent.OpenSearchRejectedExecutionException)6 NodeClosedException (org.opensearch.node.NodeClosedException)6 ConnectTransportException (org.opensearch.transport.ConnectTransportException)6 CorruptIndexException (org.apache.lucene.index.CorruptIndexException)5 IndexShard (org.opensearch.index.shard.IndexShard)5 HashMap (java.util.HashMap)4 ShardInfo (org.opensearch.action.support.replication.ReplicationResponse.ShardInfo)4