Search in sources :

Example 1 with NoNodeAvailableException

use of org.opensearch.client.transport.NoNodeAvailableException in project OpenSearch by opensearch-project.

the class TransportWriteActionTests method testReplicaProxy.

public void testReplicaProxy() throws InterruptedException, ExecutionException {
    CapturingTransport transport = new CapturingTransport();
    TransportService transportService = transport.createTransportService(clusterService.getSettings(), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
    transportService.start();
    transportService.acceptIncomingRequests();
    ShardStateAction shardStateAction = new ShardStateAction(clusterService, transportService, null, null, threadPool);
    TestAction action = new TestAction(Settings.EMPTY, "internal:testAction", transportService, clusterService, shardStateAction, threadPool);
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    ClusterState state = ClusterStateCreationUtils.stateWithActivePrimary(index, true, 1 + randomInt(3), randomInt(2));
    logger.info("using state: {}", state);
    ClusterServiceUtils.setState(clusterService, state);
    final long primaryTerm = state.metadata().index(index).primaryTerm(0);
    ReplicationOperation.Replicas<TestRequest> proxy = action.newReplicasProxy();
    // check that at unknown node fails
    PlainActionFuture<ReplicaResponse> listener = new PlainActionFuture<>();
    ShardRoutingState routingState = randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.STARTED, ShardRoutingState.RELOCATING);
    proxy.performOn(TestShardRouting.newShardRouting(shardId, "NOT THERE", routingState == ShardRoutingState.RELOCATING ? state.nodes().iterator().next().getId() : null, false, routingState), new TestRequest(), primaryTerm, randomNonNegativeLong(), randomNonNegativeLong(), listener);
    assertTrue(listener.isDone());
    assertListenerThrows("non existent node should throw a NoNodeAvailableException", listener, NoNodeAvailableException.class);
    final IndexShardRoutingTable shardRoutings = state.routingTable().shardRoutingTable(shardId);
    final ShardRouting replica = randomFrom(shardRoutings.replicaShards().stream().filter(ShardRouting::assignedToNode).collect(Collectors.toList()));
    listener = new PlainActionFuture<>();
    proxy.performOn(replica, new TestRequest(), primaryTerm, randomNonNegativeLong(), randomNonNegativeLong(), listener);
    assertFalse(listener.isDone());
    CapturingTransport.CapturedRequest[] captures = transport.getCapturedRequestsAndClear();
    assertThat(captures, arrayWithSize(1));
    if (randomBoolean()) {
        final TransportReplicationAction.ReplicaResponse response = new TransportReplicationAction.ReplicaResponse(randomLong(), randomLong());
        transport.handleResponse(captures[0].requestId, response);
        assertTrue(listener.isDone());
        assertThat(listener.get(), equalTo(response));
    } else if (randomBoolean()) {
        transport.handleRemoteError(captures[0].requestId, new OpenSearchException("simulated"));
        assertTrue(listener.isDone());
        assertListenerThrows("listener should reflect remote error", listener, OpenSearchException.class);
    } else {
        transport.handleError(captures[0].requestId, new TransportException("simulated"));
        assertTrue(listener.isDone());
        assertListenerThrows("listener should reflect remote error", listener, TransportException.class);
    }
    AtomicReference<Object> failure = new AtomicReference<>();
    AtomicBoolean success = new AtomicBoolean();
    proxy.failShardIfNeeded(replica, primaryTerm, "test", new OpenSearchException("simulated"), ActionListener.wrap(r -> success.set(true), failure::set));
    CapturingTransport.CapturedRequest[] shardFailedRequests = transport.getCapturedRequestsAndClear();
    // A write replication action proxy should fail the shard
    assertEquals(1, shardFailedRequests.length);
    CapturingTransport.CapturedRequest shardFailedRequest = shardFailedRequests[0];
    ShardStateAction.FailedShardEntry shardEntry = (ShardStateAction.FailedShardEntry) shardFailedRequest.request;
    // the shard the request was sent to and the shard to be failed should be the same
    assertEquals(shardEntry.getShardId(), replica.shardId());
    assertEquals(shardEntry.getAllocationId(), replica.allocationId().getId());
    if (randomBoolean()) {
        // simulate success
        transport.handleResponse(shardFailedRequest.requestId, TransportResponse.Empty.INSTANCE);
        assertTrue(success.get());
        assertNull(failure.get());
    } else if (randomBoolean()) {
        // simulate the primary has been demoted
        transport.handleRemoteError(shardFailedRequest.requestId, new ShardStateAction.NoLongerPrimaryShardException(replica.shardId(), "shard-failed-test"));
        assertFalse(success.get());
        assertNotNull(failure.get());
    } else {
        // simulated a node closing exception
        transport.handleRemoteError(shardFailedRequest.requestId, new NodeClosedException(state.nodes().getLocalNode()));
        assertFalse(success.get());
        assertNotNull(failure.get());
    }
}
Also used : TestThreadPool(org.opensearch.threadpool.TestThreadPool) OpenSearchException(org.opensearch.OpenSearchException) NoNodeAvailableException(org.opensearch.client.transport.NoNodeAvailableException) Transport(org.opensearch.transport.Transport) ReplicaResponse(org.opensearch.action.support.replication.ReplicationOperation.ReplicaResponse) Mockito.doThrow(org.mockito.Mockito.doThrow) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) WriteResponse(org.opensearch.action.support.WriteResponse) After(org.junit.After) Mockito.doAnswer(org.mockito.Mockito.doAnswer) NodeClosedException(org.opensearch.node.NodeClosedException) ActionListener(org.opensearch.action.ActionListener) ShardStateAction(org.opensearch.cluster.action.shard.ShardStateAction) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) AfterClass(org.junit.AfterClass) Index(org.opensearch.index.Index) IndexingPressureService(org.opensearch.index.IndexingPressureService) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) IndicesService(org.opensearch.indices.IndicesService) Settings(org.opensearch.common.settings.Settings) TransportResponse(org.opensearch.transport.TransportResponse) TransportService(org.opensearch.transport.TransportService) Collectors(java.util.stream.Collectors) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) ActionFilters(org.opensearch.action.support.ActionFilters) ActionTestUtils(org.opensearch.action.support.ActionTestUtils) Matchers.equalTo(org.hamcrest.Matchers.equalTo) RoutingNode(org.opensearch.cluster.routing.RoutingNode) Mockito.any(org.mockito.Mockito.any) ClusterServiceUtils(org.opensearch.test.ClusterServiceUtils) TransportException(org.opensearch.transport.TransportException) RefreshPolicy(org.opensearch.action.support.WriteRequest.RefreshPolicy) Mockito.mock(org.mockito.Mockito.mock) Matchers.arrayWithSize(org.hamcrest.Matchers.arrayWithSize) BeforeClass(org.junit.BeforeClass) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ThreadPool(org.opensearch.threadpool.ThreadPool) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Releasable(org.opensearch.common.lease.Releasable) AtomicReference(java.util.concurrent.atomic.AtomicReference) HashSet(java.util.HashSet) ClusterState(org.opensearch.cluster.ClusterState) IndexShard(org.opensearch.index.shard.IndexShard) ArgumentCaptor(org.mockito.ArgumentCaptor) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) Translog(org.opensearch.index.translog.Translog) Mockito.anyLong(org.mockito.Mockito.anyLong) Mockito.anyString(org.mockito.Mockito.anyString) Before(org.junit.Before) StreamInput(org.opensearch.common.io.stream.StreamInput) Collections.emptyMap(java.util.Collections.emptyMap) ClusterServiceUtils.createClusterService(org.opensearch.test.ClusterServiceUtils.createClusterService) IOException(java.io.IOException) ShardNotFoundException(org.opensearch.index.shard.ShardNotFoundException) Mockito.when(org.mockito.Mockito.when) IndexService(org.opensearch.index.IndexService) Mockito.verify(org.mockito.Mockito.verify) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ShardId(org.opensearch.index.shard.ShardId) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting) SystemIndices(org.opensearch.indices.SystemIndices) Mockito.never(org.mockito.Mockito.never) ClusterService(org.opensearch.cluster.service.ClusterService) Mockito.anyInt(org.mockito.Mockito.anyInt) CapturingTransport(org.opensearch.test.transport.CapturingTransport) Collections(java.util.Collections) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) ShardStateAction(org.opensearch.cluster.action.shard.ShardStateAction) Mockito.anyString(org.mockito.Mockito.anyString) ShardId(org.opensearch.index.shard.ShardId) NodeClosedException(org.opensearch.node.NodeClosedException) ClusterState(org.opensearch.cluster.ClusterState) CapturingTransport(org.opensearch.test.transport.CapturingTransport) AtomicReference(java.util.concurrent.atomic.AtomicReference) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) TransportException(org.opensearch.transport.TransportException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TransportService(org.opensearch.transport.TransportService) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) ReplicaResponse(org.opensearch.action.support.replication.ReplicationOperation.ReplicaResponse) OpenSearchException(org.opensearch.OpenSearchException) ShardRouting(org.opensearch.cluster.routing.ShardRouting) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting)

Example 2 with NoNodeAvailableException

use of org.opensearch.client.transport.NoNodeAvailableException in project OpenSearch by opensearch-project.

the class OpenSearchExceptionTests method testFailureToAndFromXContentWithDetails.

public void testFailureToAndFromXContentWithDetails() throws IOException {
    final XContent xContent = randomFrom(XContentType.values()).xContent();
    Exception failure;
    Throwable failureCause;
    OpenSearchException expected;
    OpenSearchException expectedCause;
    OpenSearchException suppressed;
    switch(randomIntBetween(0, 6)) {
        case // Simple opensearch exception without cause
        0:
            failure = new NoNodeAvailableException("A");
            expected = new OpenSearchException("OpenSearch exception [type=no_node_available_exception, reason=A]");
            expected.addSuppressed(new OpenSearchException("OpenSearch exception [type=no_node_available_exception, reason=A]"));
            break;
        case // Simple opensearch exception with headers (other metadata of type number are not parsed)
        1:
            failure = new ParsingException(3, 2, "B", null);
            ((OpenSearchException) failure).addHeader("header_name", "0", "1");
            expected = new OpenSearchException("OpenSearch exception [type=parsing_exception, reason=B]");
            expected.addHeader("header_name", "0", "1");
            suppressed = new OpenSearchException("OpenSearch exception [type=parsing_exception, reason=B]");
            suppressed.addHeader("header_name", "0", "1");
            expected.addSuppressed(suppressed);
            break;
        case // OpenSearch exception with a cause, headers and parsable metadata
        2:
            failureCause = new NullPointerException("var is null");
            failure = new ScriptException("C", failureCause, singletonList("stack"), "test", "painless");
            ((OpenSearchException) failure).addHeader("script_name", "my_script");
            expectedCause = new OpenSearchException("OpenSearch exception [type=null_pointer_exception, reason=var is null]");
            expected = new OpenSearchException("OpenSearch exception [type=script_exception, reason=C]", expectedCause);
            expected.addHeader("script_name", "my_script");
            expected.addMetadata("opensearch.lang", "painless");
            expected.addMetadata("opensearch.script", "test");
            expected.addMetadata("opensearch.script_stack", "stack");
            suppressed = new OpenSearchException("OpenSearch exception [type=script_exception, reason=C]");
            suppressed.addHeader("script_name", "my_script");
            suppressed.addMetadata("opensearch.lang", "painless");
            suppressed.addMetadata("opensearch.script", "test");
            suppressed.addMetadata("opensearch.script_stack", "stack");
            expected.addSuppressed(suppressed);
            break;
        case // JDK exception without cause
        3:
            failure = new IllegalStateException("D");
            expected = new OpenSearchException("OpenSearch exception [type=illegal_state_exception, reason=D]");
            suppressed = new OpenSearchException("OpenSearch exception [type=illegal_state_exception, reason=D]");
            expected.addSuppressed(suppressed);
            break;
        case // JDK exception with cause
        4:
            failureCause = new RoutingMissingException("idx", "id");
            failure = new RuntimeException("E", failureCause);
            expectedCause = new OpenSearchException("OpenSearch exception [type=routing_missing_exception, " + "reason=routing is required for [idx]/[id]]");
            expectedCause.addMetadata("opensearch.index", "idx");
            expectedCause.addMetadata("opensearch.index_uuid", "_na_");
            expected = new OpenSearchException("OpenSearch exception [type=runtime_exception, reason=E]", expectedCause);
            suppressed = new OpenSearchException("OpenSearch exception [type=runtime_exception, reason=E]");
            expected.addSuppressed(suppressed);
            break;
        case // Wrapped exception with cause
        5:
            failureCause = new FileAlreadyExistsException("File exists");
            failure = new BroadcastShardOperationFailedException(new ShardId("_index", "_uuid", 5), "F", failureCause);
            expected = new OpenSearchException("OpenSearch exception [type=file_already_exists_exception, reason=File exists]");
            suppressed = new OpenSearchException("OpenSearch exception [type=file_already_exists_exception, reason=File exists]");
            expected.addSuppressed(suppressed);
            break;
        case // SearchPhaseExecutionException with cause and multiple failures
        6:
            DiscoveryNode node = new DiscoveryNode("node_g", buildNewFakeTransportAddress(), Version.CURRENT);
            failureCause = new NodeClosedException(node);
            failureCause = new NoShardAvailableActionException(new ShardId("_index_g", "_uuid_g", 6), "node_g", failureCause);
            ShardSearchFailure[] shardFailures = new ShardSearchFailure[] { new ShardSearchFailure(new ParsingException(0, 0, "Parsing g", null), new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 61), null, OriginalIndices.NONE)), new ShardSearchFailure(new RepositoryException("repository_g", "Repo"), new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 62), null, OriginalIndices.NONE)), new ShardSearchFailure(new SearchContextMissingException(new ShardSearchContextId(UUIDs.randomBase64UUID(), 0L)), null) };
            failure = new SearchPhaseExecutionException("phase_g", "G", failureCause, shardFailures);
            expectedCause = new OpenSearchException("OpenSearch exception [type=node_closed_exception, " + "reason=node closed " + node + "]");
            expectedCause = new OpenSearchException("OpenSearch exception [type=no_shard_available_action_exception, " + "reason=node_g]", expectedCause);
            expectedCause.addMetadata("opensearch.index", "_index_g");
            expectedCause.addMetadata("opensearch.index_uuid", "_uuid_g");
            expectedCause.addMetadata("opensearch.shard", "6");
            expected = new OpenSearchException("OpenSearch exception [type=search_phase_execution_exception, " + "reason=G]", expectedCause);
            expected.addMetadata("opensearch.phase", "phase_g");
            expected.addSuppressed(new OpenSearchException("OpenSearch exception [type=parsing_exception, reason=Parsing g]"));
            expected.addSuppressed(new OpenSearchException("OpenSearch exception [type=repository_exception, " + "reason=[repository_g] Repo]"));
            expected.addSuppressed(new OpenSearchException("OpenSearch exception [type=search_context_missing_exception, " + "reason=No search context found for id [0]]"));
            break;
        default:
            throw new UnsupportedOperationException("Failed to generate randomized failure");
    }
    Exception finalFailure = failure;
    BytesReference failureBytes = toShuffledXContent((builder, params) -> {
        OpenSearchException.generateFailureXContent(builder, params, finalFailure, true);
        return builder;
    }, xContent.type(), ToXContent.EMPTY_PARAMS, randomBoolean());
    try (XContentParser parser = createParser(xContent, failureBytes)) {
        failureBytes = BytesReference.bytes(shuffleXContent(parser, randomBoolean()));
    }
    OpenSearchException parsedFailure;
    try (XContentParser parser = createParser(xContent, failureBytes)) {
        assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
        assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
        parsedFailure = OpenSearchException.failureFromXContent(parser);
        assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
        assertNull(parser.nextToken());
    }
    assertDeepEquals(expected, parsedFailure);
}
Also used : FileAlreadyExistsException(java.nio.file.FileAlreadyExistsException) DiscoveryNode(org.opensearch.cluster.node.DiscoveryNode) SearchPhaseExecutionException(org.opensearch.action.search.SearchPhaseExecutionException) Index(org.opensearch.index.Index) ShardId(org.opensearch.index.shard.ShardId) ScriptException(org.opensearch.script.ScriptException) ToXContent(org.opensearch.common.xcontent.ToXContent) XContent(org.opensearch.common.xcontent.XContent) ParsingException(org.opensearch.common.ParsingException) NodeClosedException(org.opensearch.node.NodeClosedException) BroadcastShardOperationFailedException(org.opensearch.action.support.broadcast.BroadcastShardOperationFailedException) ShardSearchFailure(org.opensearch.action.search.ShardSearchFailure) RoutingMissingException(org.opensearch.action.RoutingMissingException) BytesReference(org.opensearch.common.bytes.BytesReference) SearchContextMissingException(org.opensearch.search.SearchContextMissingException) RepositoryException(org.opensearch.repositories.RepositoryException) NoNodeAvailableException(org.opensearch.client.transport.NoNodeAvailableException) IndexShardRecoveringException(org.opensearch.index.shard.IndexShardRecoveringException) NoNodeAvailableException(org.opensearch.client.transport.NoNodeAvailableException) ScriptException(org.opensearch.script.ScriptException) NodeClosedException(org.opensearch.node.NodeClosedException) BroadcastShardOperationFailedException(org.opensearch.action.support.broadcast.BroadcastShardOperationFailedException) ParsingException(org.opensearch.common.ParsingException) RepositoryException(org.opensearch.repositories.RepositoryException) RemoteTransportException(org.opensearch.transport.RemoteTransportException) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) QueryShardException(org.opensearch.index.query.QueryShardException) SearchParseException(org.opensearch.search.SearchParseException) SearchContextMissingException(org.opensearch.search.SearchContextMissingException) EOFException(java.io.EOFException) FileNotFoundException(java.io.FileNotFoundException) RoutingMissingException(org.opensearch.action.RoutingMissingException) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) IOException(java.io.IOException) FileAlreadyExistsException(java.nio.file.FileAlreadyExistsException) SearchPhaseExecutionException(org.opensearch.action.search.SearchPhaseExecutionException) XContentParseException(org.opensearch.common.xcontent.XContentParseException) NoShardAvailableActionException(org.opensearch.action.NoShardAvailableActionException) ShardSearchContextId(org.opensearch.search.internal.ShardSearchContextId) NoShardAvailableActionException(org.opensearch.action.NoShardAvailableActionException) SearchShardTarget(org.opensearch.search.SearchShardTarget) XContentParser(org.opensearch.common.xcontent.XContentParser)

Example 3 with NoNodeAvailableException

use of org.opensearch.client.transport.NoNodeAvailableException in project OpenSearch by opensearch-project.

the class TransportReplicationActionTests method testReplicaProxy.

public void testReplicaProxy() throws InterruptedException, ExecutionException {
    final String index = "test";
    final ShardId shardId = new ShardId(index, "_na_", 0);
    ClusterState state = stateWithActivePrimary(index, true, 1 + randomInt(3), randomInt(2));
    logger.info("using state: {}", state);
    setState(clusterService, state);
    final long primaryTerm = state.metadata().index(index).primaryTerm(0);
    ReplicationOperation.Replicas<Request> proxy = action.newReplicasProxy();
    // check that at unknown node fails
    PlainActionFuture<ReplicaResponse> listener = new PlainActionFuture<>();
    ShardRoutingState routingState = randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.STARTED, ShardRoutingState.RELOCATING);
    proxy.performOn(TestShardRouting.newShardRouting(shardId, "NOT THERE", routingState == ShardRoutingState.RELOCATING ? state.nodes().iterator().next().getId() : null, false, routingState), new Request(NO_SHARD_ID), primaryTerm, randomNonNegativeLong(), randomNonNegativeLong(), listener);
    assertTrue(listener.isDone());
    assertListenerThrows("non existent node should throw a NoNodeAvailableException", listener, NoNodeAvailableException.class);
    final IndexShardRoutingTable shardRoutings = state.routingTable().shardRoutingTable(shardId);
    final ShardRouting replica = randomFrom(shardRoutings.replicaShards().stream().filter(ShardRouting::assignedToNode).collect(Collectors.toList()));
    listener = new PlainActionFuture<>();
    proxy.performOn(replica, new Request(NO_SHARD_ID), primaryTerm, randomNonNegativeLong(), randomNonNegativeLong(), listener);
    assertFalse(listener.isDone());
    CapturingTransport.CapturedRequest[] captures = transport.getCapturedRequestsAndClear();
    assertThat(captures, arrayWithSize(1));
    if (randomBoolean()) {
        final TransportReplicationAction.ReplicaResponse response = new TransportReplicationAction.ReplicaResponse(randomLong(), randomLong());
        transport.handleResponse(captures[0].requestId, response);
        assertTrue(listener.isDone());
        assertThat(listener.get(), equalTo(response));
    } else if (randomBoolean()) {
        transport.handleRemoteError(captures[0].requestId, new OpenSearchException("simulated"));
        assertTrue(listener.isDone());
        assertListenerThrows("listener should reflect remote error", listener, OpenSearchException.class);
    } else {
        transport.handleError(captures[0].requestId, new TransportException("simulated"));
        assertTrue(listener.isDone());
        assertListenerThrows("listener should reflect remote error", listener, TransportException.class);
    }
    AtomicReference<Object> failure = new AtomicReference<>();
    AtomicBoolean success = new AtomicBoolean();
    proxy.failShardIfNeeded(replica, primaryTerm, "test", new OpenSearchException("simulated"), ActionListener.wrap(r -> success.set(true), failure::set));
    CapturingTransport.CapturedRequest[] shardFailedRequests = transport.getCapturedRequestsAndClear();
    // A replication action doesn't not fail the request
    assertEquals(0, shardFailedRequests.length);
}
Also used : Matchers.hasToString(org.hamcrest.Matchers.hasToString) Metadata(org.opensearch.cluster.metadata.Metadata) AllocationService(org.opensearch.cluster.routing.allocation.AllocationService) TestThreadPool(org.opensearch.threadpool.TestThreadPool) AlreadyClosedException(org.apache.lucene.store.AlreadyClosedException) ClusterStateChanges(org.opensearch.indices.cluster.ClusterStateChanges) Version(org.opensearch.Version) ClusterServiceUtils.setState(org.opensearch.test.ClusterServiceUtils.setState) NoNodeAvailableException(org.opensearch.client.transport.NoNodeAvailableException) Transport(org.opensearch.transport.Transport) Mockito.doThrow(org.mockito.Mockito.doThrow) ClusterBlock(org.opensearch.cluster.block.ClusterBlock) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) OpenSearchAllocationTestCase(org.opensearch.cluster.OpenSearchAllocationTestCase) Matchers.nullValue(org.hamcrest.Matchers.nullValue) Mockito.doAnswer(org.mockito.Mockito.doAnswer) Is.is(org.hamcrest.core.Is.is) ActionListener(org.opensearch.action.ActionListener) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest) ShardStateAction(org.opensearch.cluster.action.shard.ShardStateAction) EnumSet(java.util.EnumSet) AfterClass(org.junit.AfterClass) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) Index(org.opensearch.index.Index) Set(java.util.Set) Settings(org.opensearch.common.settings.Settings) TransportService(org.opensearch.transport.TransportService) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) ActionFilters(org.opensearch.action.support.ActionFilters) ActionTestUtils(org.opensearch.action.support.ActionTestUtils) RoutingNode(org.opensearch.cluster.routing.RoutingNode) TransportException(org.opensearch.transport.TransportException) Mockito.eq(org.mockito.Mockito.eq) Mockito.mock(org.mockito.Mockito.mock) Matchers.arrayWithSize(org.hamcrest.Matchers.arrayWithSize) IndexShardState(org.opensearch.index.shard.IndexShardState) ThreadPool(org.opensearch.threadpool.ThreadPool) Releasable(org.opensearch.common.lease.Releasable) MockTransportService(org.opensearch.test.transport.MockTransportService) ClusterState(org.opensearch.cluster.ClusterState) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) Mockito.anyLong(org.mockito.Mockito.anyLong) ClusterBlocks(org.opensearch.cluster.block.ClusterBlocks) Mockito.anyString(org.mockito.Mockito.anyString) Before(org.junit.Before) MockNioTransport(org.opensearch.transport.nio.MockNioTransport) IOException(java.io.IOException) ShardNotFoundException(org.opensearch.index.shard.ShardNotFoundException) IndexService(org.opensearch.index.IndexService) SETTING_WAIT_FOR_ACTIVE_SHARDS(org.opensearch.cluster.metadata.IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS) ExecutionException(java.util.concurrent.ExecutionException) Matcher(org.hamcrest.Matcher) ClusterService(org.opensearch.cluster.service.ClusterService) NetworkService(org.opensearch.common.network.NetworkService) Mockito.anyInt(org.mockito.Mockito.anyInt) ReplicationGroup(org.opensearch.index.shard.ReplicationGroup) Assert(org.junit.Assert) PageCacheRecycler(org.opensearch.common.util.PageCacheRecycler) NoneCircuitBreakerService(org.opensearch.indices.breaker.NoneCircuitBreakerService) OpenSearchException(org.opensearch.OpenSearchException) TestTransportChannel(org.opensearch.transport.TestTransportChannel) ReplicaResponse(org.opensearch.action.support.replication.ReplicationOperation.ReplicaResponse) Collections.singleton(java.util.Collections.singleton) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Locale(java.util.Locale) After(org.junit.After) UnavailableShardsException(org.opensearch.action.UnavailableShardsException) CloseIndexRequest(org.opensearch.action.admin.indices.close.CloseIndexRequest) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) TransportChannel(org.opensearch.transport.TransportChannel) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) IndicesService(org.opensearch.indices.IndicesService) ClusterBlockException(org.opensearch.cluster.block.ClusterBlockException) IndexShardClosedException(org.opensearch.index.shard.IndexShardClosedException) TransportResponse(org.opensearch.transport.TransportResponse) RestStatus(org.opensearch.rest.RestStatus) ClusterStateCreationUtils.state(org.opensearch.action.support.replication.ClusterStateCreationUtils.state) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) List(java.util.List) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Mockito.any(org.mockito.Mockito.any) DiscoveryNodes(org.opensearch.cluster.node.DiscoveryNodes) BeforeClass(org.junit.BeforeClass) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) ShardNotInPrimaryModeException(org.opensearch.index.shard.ShardNotInPrimaryModeException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) StreamOutput(org.opensearch.common.io.stream.StreamOutput) AtomicReference(java.util.concurrent.atomic.AtomicReference) NamedWriteableRegistry(org.opensearch.common.io.stream.NamedWriteableRegistry) HashSet(java.util.HashSet) IndexShard(org.opensearch.index.shard.IndexShard) IndexClosedException(org.opensearch.indices.IndexClosedException) StreamInput(org.opensearch.common.io.stream.StreamInput) ClusterServiceUtils.createClusterService(org.opensearch.test.ClusterServiceUtils.createClusterService) TransportRequest(org.opensearch.transport.TransportRequest) AllocationId(org.opensearch.cluster.routing.AllocationId) ClusterBlockLevel(org.opensearch.cluster.block.ClusterBlockLevel) IndexNotFoundException(org.opensearch.index.IndexNotFoundException) ClusterStateCreationUtils.stateWithActivePrimary(org.opensearch.action.support.replication.ClusterStateCreationUtils.stateWithActivePrimary) Mockito.when(org.mockito.Mockito.when) ActiveShardCount(org.opensearch.action.support.ActiveShardCount) Mockito.verify(org.mockito.Mockito.verify) ShardRouting(org.opensearch.cluster.routing.ShardRouting) ShardId(org.opensearch.index.shard.ShardId) TimeUnit(java.util.concurrent.TimeUnit) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting) CapturingTransport(org.opensearch.test.transport.CapturingTransport) Collections(java.util.Collections) IndexShardRoutingTable(org.opensearch.cluster.routing.IndexShardRoutingTable) Matchers.hasToString(org.hamcrest.Matchers.hasToString) CoreMatchers.containsString(org.hamcrest.CoreMatchers.containsString) Mockito.anyString(org.mockito.Mockito.anyString) ShardId(org.opensearch.index.shard.ShardId) ClusterState(org.opensearch.cluster.ClusterState) CreateIndexRequest(org.opensearch.action.admin.indices.create.CreateIndexRequest) CloseIndexRequest(org.opensearch.action.admin.indices.close.CloseIndexRequest) TransportRequest(org.opensearch.transport.TransportRequest) AtomicReference(java.util.concurrent.atomic.AtomicReference) ShardRoutingState(org.opensearch.cluster.routing.ShardRoutingState) TransportException(org.opensearch.transport.TransportException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) PlainActionFuture(org.opensearch.action.support.PlainActionFuture) ReplicaResponse(org.opensearch.action.support.replication.ReplicationOperation.ReplicaResponse) OpenSearchException(org.opensearch.OpenSearchException) ShardRouting(org.opensearch.cluster.routing.ShardRouting) TestShardRouting(org.opensearch.cluster.routing.TestShardRouting)

Example 4 with NoNodeAvailableException

use of org.opensearch.client.transport.NoNodeAvailableException in project OpenSearch by opensearch-project.

the class UpdateIT method testStressUpdateDeleteConcurrency.

public void testStressUpdateDeleteConcurrency() throws Exception {
    // We create an index with merging disabled so that deletes don't get merged away
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)));
    ensureGreen();
    Script fieldIncScript = new Script(ScriptType.INLINE, UPDATE_SCRIPTS, FIELD_INC_SCRIPT, Collections.singletonMap("field", "field"));
    final int numberOfThreads = scaledRandomIntBetween(3, 5);
    final int numberOfIdsPerThread = scaledRandomIntBetween(3, 10);
    final int numberOfUpdatesPerId = scaledRandomIntBetween(10, 100);
    final int retryOnConflict = randomIntBetween(0, 1);
    final CountDownLatch latch = new CountDownLatch(numberOfThreads);
    final CountDownLatch startLatch = new CountDownLatch(1);
    final List<Throwable> failures = new CopyOnWriteArrayList<>();
    final class UpdateThread extends Thread {

        final Map<Integer, Integer> failedMap = new HashMap<>();

        final int numberOfIds;

        final int maxUpdateRequests = numberOfIdsPerThread * numberOfUpdatesPerId;

        final int maxDeleteRequests = numberOfIdsPerThread * numberOfUpdatesPerId;

        private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);

        private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);

        UpdateThread(int numberOfIds) {
            this.numberOfIds = numberOfIds;
        }

        final class UpdateListener implements ActionListener<UpdateResponse> {

            int id;

            UpdateListener(int id) {
                this.id = id;
            }

            @Override
            public void onResponse(UpdateResponse updateResponse) {
                updateRequestsOutstanding.release(1);
            }

            @Override
            public void onFailure(Exception e) {
                synchronized (failedMap) {
                    incrementMapValue(id, failedMap);
                }
                updateRequestsOutstanding.release(1);
            }
        }

        final class DeleteListener implements ActionListener<DeleteResponse> {

            int id;

            DeleteListener(int id) {
                this.id = id;
            }

            @Override
            public void onResponse(DeleteResponse deleteResponse) {
                deleteRequestsOutstanding.release(1);
            }

            @Override
            public void onFailure(Exception e) {
                synchronized (failedMap) {
                    incrementMapValue(id, failedMap);
                }
                deleteRequestsOutstanding.release(1);
            }
        }

        @Override
        public void run() {
            try {
                startLatch.await();
                boolean hasWaitedForNoNode = false;
                for (int j = 0; j < numberOfIds; j++) {
                    for (int k = 0; k < numberOfUpdatesPerId; ++k) {
                        updateRequestsOutstanding.acquire();
                        try {
                            UpdateRequest ur = client().prepareUpdate("test", Integer.toString(j)).setScript(fieldIncScript).setRetryOnConflict(retryOnConflict).setUpsert(jsonBuilder().startObject().field("field", 1).endObject()).request();
                            if (randomBoolean()) {
                                client().update(ur, new UpdateListener(j));
                            } else {
                                client().prepareBulk().add(ur).execute(ActionListener.map(new UpdateListener(j), br -> {
                                    final BulkItemResponse ir = br.getItems()[0];
                                    if (ir.isFailed()) {
                                        throw ir.getFailure().getCause();
                                    } else {
                                        return ir.getResponse();
                                    }
                                }));
                            }
                        } catch (NoNodeAvailableException nne) {
                            updateRequestsOutstanding.release();
                            synchronized (failedMap) {
                                incrementMapValue(j, failedMap);
                            }
                            if (hasWaitedForNoNode) {
                                throw nne;
                            }
                            logger.warn("Got NoNodeException waiting for 1 second for things to recover.");
                            hasWaitedForNoNode = true;
                            Thread.sleep(1000);
                        }
                        try {
                            deleteRequestsOutstanding.acquire();
                            DeleteRequest dr = client().prepareDelete("test", Integer.toString(j)).request();
                            client().delete(dr, new DeleteListener(j));
                        } catch (NoNodeAvailableException nne) {
                            deleteRequestsOutstanding.release();
                            synchronized (failedMap) {
                                incrementMapValue(j, failedMap);
                            }
                            if (hasWaitedForNoNode) {
                                throw nne;
                            }
                            logger.warn("Got NoNodeException waiting for 1 second for things to recover.");
                            hasWaitedForNoNode = true;
                            // Wait for no-node to clear
                            Thread.sleep(1000);
                        }
                    }
                }
            } catch (Exception e) {
                logger.error("Something went wrong", e);
                failures.add(e);
            } finally {
                try {
                    waitForOutstandingRequests(TimeValue.timeValueSeconds(60), updateRequestsOutstanding, maxUpdateRequests, "Update");
                    waitForOutstandingRequests(TimeValue.timeValueSeconds(60), deleteRequestsOutstanding, maxDeleteRequests, "Delete");
                } catch (OpenSearchTimeoutException ete) {
                    failures.add(ete);
                }
                latch.countDown();
            }
        }

        private void incrementMapValue(int j, Map<Integer, Integer> map) {
            if (!map.containsKey(j)) {
                map.put(j, 0);
            }
            map.put(j, map.get(j) + 1);
        }

        private void waitForOutstandingRequests(TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
            long start = System.currentTimeMillis();
            do {
                long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
                logger.info("[{}] going to try and acquire [{}] in [{}]ms [{}] available to acquire right now", name, maxRequests, msRemaining, requestsOutstanding.availablePermits());
                try {
                    requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
                    return;
                } catch (InterruptedException ie) {
                // Just keep swimming
                }
            } while ((System.currentTimeMillis() - start) < timeOut.getMillis());
            throw new OpenSearchTimeoutException("Requests were still outstanding after the timeout [" + timeOut + "] for type [" + name + "]");
        }
    }
    final List<UpdateThread> threads = new ArrayList<>();
    for (int i = 0; i < numberOfThreads; i++) {
        UpdateThread ut = new UpdateThread(numberOfIdsPerThread);
        ut.start();
        threads.add(ut);
    }
    startLatch.countDown();
    latch.await();
    for (UpdateThread ut : threads) {
        // Threads should have finished because of the latch.await
        ut.join();
    }
    // aquiring the request outstanding semaphores.
    for (Throwable throwable : failures) {
        logger.info("Captured failure on concurrent update:", throwable);
    }
    assertThat(failures.size(), equalTo(0));
    // All the previous operations should be complete or failed at this point
    for (int i = 0; i < numberOfIdsPerThread; ++i) {
        client().prepareUpdate("test", Integer.toString(i)).setScript(fieldIncScript).setRetryOnConflict(Integer.MAX_VALUE).setUpsert(jsonBuilder().startObject().field("field", 1).endObject()).execute().actionGet();
    }
    refresh();
    for (int i = 0; i < numberOfIdsPerThread; ++i) {
        int totalFailures = 0;
        GetResponse response = client().prepareGet("test", Integer.toString(i)).execute().actionGet();
        if (response.isExists()) {
            assertThat(response.getId(), equalTo(Integer.toString(i)));
            int expectedVersion = (numberOfThreads * numberOfUpdatesPerId * 2) + 1;
            for (UpdateThread ut : threads) {
                if (ut.failedMap.containsKey(i)) {
                    totalFailures += ut.failedMap.get(i);
                }
            }
            expectedVersion -= totalFailures;
            logger.error("Actual version [{}] Expected version [{}] Total failures [{}]", response.getVersion(), expectedVersion, totalFailures);
            assertThat(response.getVersion(), equalTo((long) expectedVersion));
            assertThat(response.getVersion() + totalFailures, equalTo((long) ((numberOfUpdatesPerId * numberOfThreads * 2) + 1)));
        }
    }
}
Also used : XContentFactory.jsonBuilder(org.opensearch.common.xcontent.XContentFactory.jsonBuilder) Arrays(java.util.Arrays) IndexResponse(org.opensearch.action.index.IndexResponse) UpdateResponse(org.opensearch.action.update.UpdateResponse) HashMap(java.util.HashMap) Function(java.util.function.Function) ActionRequestValidationException(org.opensearch.action.ActionRequestValidationException) NoNodeAvailableException(org.opensearch.client.transport.NoNodeAvailableException) ArrayList(java.util.ArrayList) ScriptType(org.opensearch.script.ScriptType) InternalSettingsPlugin(org.opensearch.test.InternalSettingsPlugin) Alias(org.opensearch.action.admin.indices.alias.Alias) BulkItemResponse(org.opensearch.action.bulk.BulkItemResponse) Map(java.util.Map) Matchers.nullValue(org.hamcrest.Matchers.nullValue) DeleteResponse(org.opensearch.action.delete.DeleteResponse) XContentFactory(org.opensearch.common.xcontent.XContentFactory) ActionListener(org.opensearch.action.ActionListener) GetResponse(org.opensearch.action.get.GetResponse) DeleteRequest(org.opensearch.action.delete.DeleteRequest) OpenSearchAssertions.assertAcked(org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked) TimeValue(org.opensearch.common.unit.TimeValue) Matchers.notNullValue(org.hamcrest.Matchers.notNullValue) VersionConflictEngineException(org.opensearch.index.engine.VersionConflictEngineException) Semaphore(java.util.concurrent.Semaphore) Script(org.opensearch.script.Script) Collection(java.util.Collection) Settings(org.opensearch.common.settings.Settings) DocumentMissingException(org.opensearch.index.engine.DocumentMissingException) RestStatus(org.opensearch.rest.RestStatus) OpenSearchAssertions.assertFutureThrows(org.opensearch.test.hamcrest.OpenSearchAssertions.assertFutureThrows) UpdateRequestBuilder(org.opensearch.action.update.UpdateRequestBuilder) Plugin(org.opensearch.plugins.Plugin) TimeUnit(java.util.concurrent.TimeUnit) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Matchers.equalTo(org.hamcrest.Matchers.equalTo) OpenSearchTimeoutException(org.opensearch.OpenSearchTimeoutException) DocWriteResponse(org.opensearch.action.DocWriteResponse) UpdateRequest(org.opensearch.action.update.UpdateRequest) MockScriptPlugin(org.opensearch.script.MockScriptPlugin) MergePolicyConfig(org.opensearch.index.MergePolicyConfig) OpenSearchIntegTestCase(org.opensearch.test.OpenSearchIntegTestCase) Collections(java.util.Collections) Matchers.containsString(org.hamcrest.Matchers.containsString) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) OpenSearchTimeoutException(org.opensearch.OpenSearchTimeoutException) ArrayList(java.util.ArrayList) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) Semaphore(java.util.concurrent.Semaphore) Matchers.containsString(org.hamcrest.Matchers.containsString) UpdateResponse(org.opensearch.action.update.UpdateResponse) TimeValue(org.opensearch.common.unit.TimeValue) Script(org.opensearch.script.Script) UpdateRequest(org.opensearch.action.update.UpdateRequest) BulkItemResponse(org.opensearch.action.bulk.BulkItemResponse) CountDownLatch(java.util.concurrent.CountDownLatch) NoNodeAvailableException(org.opensearch.client.transport.NoNodeAvailableException) GetResponse(org.opensearch.action.get.GetResponse) ActionRequestValidationException(org.opensearch.action.ActionRequestValidationException) NoNodeAvailableException(org.opensearch.client.transport.NoNodeAvailableException) VersionConflictEngineException(org.opensearch.index.engine.VersionConflictEngineException) DocumentMissingException(org.opensearch.index.engine.DocumentMissingException) OpenSearchTimeoutException(org.opensearch.OpenSearchTimeoutException) DeleteResponse(org.opensearch.action.delete.DeleteResponse) HashMap(java.util.HashMap) Map(java.util.Map) DeleteRequest(org.opensearch.action.delete.DeleteRequest) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList)

Example 5 with NoNodeAvailableException

use of org.opensearch.client.transport.NoNodeAvailableException in project OpenSearch by opensearch-project.

the class BulkByScrollResponseTests method randomSearchFailures.

private List<ScrollableHitSource.SearchFailure> randomSearchFailures() {
    if (randomBoolean()) {
        return emptyList();
    }
    String index = null;
    Integer shardId = null;
    String nodeId = null;
    if (randomBoolean()) {
        index = randomAlphaOfLength(5);
        shardId = randomInt();
        nodeId = usually() ? randomAlphaOfLength(5) : null;
    }
    OpenSearchException exception = randomFrom(new ResourceNotFoundException("bar"), new OpenSearchException("foo"), new NoNodeAvailableException("baz"));
    return singletonList(new ScrollableHitSource.SearchFailure(exception, index, shardId, nodeId));
}
Also used : OpenSearchException(org.opensearch.OpenSearchException) TestUtil.randomSimpleString(org.apache.lucene.util.TestUtil.randomSimpleString) ResourceNotFoundException(org.opensearch.ResourceNotFoundException) NoNodeAvailableException(org.opensearch.client.transport.NoNodeAvailableException)

Aggregations

NoNodeAvailableException (org.opensearch.client.transport.NoNodeAvailableException)5 IOException (java.io.IOException)3 Collections (java.util.Collections)3 TimeUnit (java.util.concurrent.TimeUnit)3 Matchers.equalTo (org.hamcrest.Matchers.equalTo)3 OpenSearchException (org.opensearch.OpenSearchException)3 ActionListener (org.opensearch.action.ActionListener)3 Index (org.opensearch.index.Index)3 ShardId (org.opensearch.index.shard.ShardId)3 HashSet (java.util.HashSet)2 Locale (java.util.Locale)2 ExecutionException (java.util.concurrent.ExecutionException)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)2 AtomicReference (java.util.concurrent.atomic.AtomicReference)2 Collectors (java.util.stream.Collectors)2 Matchers.arrayWithSize (org.hamcrest.Matchers.arrayWithSize)2 Matchers.instanceOf (org.hamcrest.Matchers.instanceOf)2 After (org.junit.After)2 AfterClass (org.junit.AfterClass)2