use of org.opensearch.client.transport.NoNodeAvailableException in project OpenSearch by opensearch-project.
the class TransportWriteActionTests method testReplicaProxy.
public void testReplicaProxy() throws InterruptedException, ExecutionException {
CapturingTransport transport = new CapturingTransport();
TransportService transportService = transport.createTransportService(clusterService.getSettings(), threadPool, TransportService.NOOP_TRANSPORT_INTERCEPTOR, x -> clusterService.localNode(), null, Collections.emptySet());
transportService.start();
transportService.acceptIncomingRequests();
ShardStateAction shardStateAction = new ShardStateAction(clusterService, transportService, null, null, threadPool);
TestAction action = new TestAction(Settings.EMPTY, "internal:testAction", transportService, clusterService, shardStateAction, threadPool);
final String index = "test";
final ShardId shardId = new ShardId(index, "_na_", 0);
ClusterState state = ClusterStateCreationUtils.stateWithActivePrimary(index, true, 1 + randomInt(3), randomInt(2));
logger.info("using state: {}", state);
ClusterServiceUtils.setState(clusterService, state);
final long primaryTerm = state.metadata().index(index).primaryTerm(0);
ReplicationOperation.Replicas<TestRequest> proxy = action.newReplicasProxy();
// check that at unknown node fails
PlainActionFuture<ReplicaResponse> listener = new PlainActionFuture<>();
ShardRoutingState routingState = randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.STARTED, ShardRoutingState.RELOCATING);
proxy.performOn(TestShardRouting.newShardRouting(shardId, "NOT THERE", routingState == ShardRoutingState.RELOCATING ? state.nodes().iterator().next().getId() : null, false, routingState), new TestRequest(), primaryTerm, randomNonNegativeLong(), randomNonNegativeLong(), listener);
assertTrue(listener.isDone());
assertListenerThrows("non existent node should throw a NoNodeAvailableException", listener, NoNodeAvailableException.class);
final IndexShardRoutingTable shardRoutings = state.routingTable().shardRoutingTable(shardId);
final ShardRouting replica = randomFrom(shardRoutings.replicaShards().stream().filter(ShardRouting::assignedToNode).collect(Collectors.toList()));
listener = new PlainActionFuture<>();
proxy.performOn(replica, new TestRequest(), primaryTerm, randomNonNegativeLong(), randomNonNegativeLong(), listener);
assertFalse(listener.isDone());
CapturingTransport.CapturedRequest[] captures = transport.getCapturedRequestsAndClear();
assertThat(captures, arrayWithSize(1));
if (randomBoolean()) {
final TransportReplicationAction.ReplicaResponse response = new TransportReplicationAction.ReplicaResponse(randomLong(), randomLong());
transport.handleResponse(captures[0].requestId, response);
assertTrue(listener.isDone());
assertThat(listener.get(), equalTo(response));
} else if (randomBoolean()) {
transport.handleRemoteError(captures[0].requestId, new OpenSearchException("simulated"));
assertTrue(listener.isDone());
assertListenerThrows("listener should reflect remote error", listener, OpenSearchException.class);
} else {
transport.handleError(captures[0].requestId, new TransportException("simulated"));
assertTrue(listener.isDone());
assertListenerThrows("listener should reflect remote error", listener, TransportException.class);
}
AtomicReference<Object> failure = new AtomicReference<>();
AtomicBoolean success = new AtomicBoolean();
proxy.failShardIfNeeded(replica, primaryTerm, "test", new OpenSearchException("simulated"), ActionListener.wrap(r -> success.set(true), failure::set));
CapturingTransport.CapturedRequest[] shardFailedRequests = transport.getCapturedRequestsAndClear();
// A write replication action proxy should fail the shard
assertEquals(1, shardFailedRequests.length);
CapturingTransport.CapturedRequest shardFailedRequest = shardFailedRequests[0];
ShardStateAction.FailedShardEntry shardEntry = (ShardStateAction.FailedShardEntry) shardFailedRequest.request;
// the shard the request was sent to and the shard to be failed should be the same
assertEquals(shardEntry.getShardId(), replica.shardId());
assertEquals(shardEntry.getAllocationId(), replica.allocationId().getId());
if (randomBoolean()) {
// simulate success
transport.handleResponse(shardFailedRequest.requestId, TransportResponse.Empty.INSTANCE);
assertTrue(success.get());
assertNull(failure.get());
} else if (randomBoolean()) {
// simulate the primary has been demoted
transport.handleRemoteError(shardFailedRequest.requestId, new ShardStateAction.NoLongerPrimaryShardException(replica.shardId(), "shard-failed-test"));
assertFalse(success.get());
assertNotNull(failure.get());
} else {
// simulated a node closing exception
transport.handleRemoteError(shardFailedRequest.requestId, new NodeClosedException(state.nodes().getLocalNode()));
assertFalse(success.get());
assertNotNull(failure.get());
}
}
use of org.opensearch.client.transport.NoNodeAvailableException in project OpenSearch by opensearch-project.
the class OpenSearchExceptionTests method testFailureToAndFromXContentWithDetails.
public void testFailureToAndFromXContentWithDetails() throws IOException {
final XContent xContent = randomFrom(XContentType.values()).xContent();
Exception failure;
Throwable failureCause;
OpenSearchException expected;
OpenSearchException expectedCause;
OpenSearchException suppressed;
switch(randomIntBetween(0, 6)) {
case // Simple opensearch exception without cause
0:
failure = new NoNodeAvailableException("A");
expected = new OpenSearchException("OpenSearch exception [type=no_node_available_exception, reason=A]");
expected.addSuppressed(new OpenSearchException("OpenSearch exception [type=no_node_available_exception, reason=A]"));
break;
case // Simple opensearch exception with headers (other metadata of type number are not parsed)
1:
failure = new ParsingException(3, 2, "B", null);
((OpenSearchException) failure).addHeader("header_name", "0", "1");
expected = new OpenSearchException("OpenSearch exception [type=parsing_exception, reason=B]");
expected.addHeader("header_name", "0", "1");
suppressed = new OpenSearchException("OpenSearch exception [type=parsing_exception, reason=B]");
suppressed.addHeader("header_name", "0", "1");
expected.addSuppressed(suppressed);
break;
case // OpenSearch exception with a cause, headers and parsable metadata
2:
failureCause = new NullPointerException("var is null");
failure = new ScriptException("C", failureCause, singletonList("stack"), "test", "painless");
((OpenSearchException) failure).addHeader("script_name", "my_script");
expectedCause = new OpenSearchException("OpenSearch exception [type=null_pointer_exception, reason=var is null]");
expected = new OpenSearchException("OpenSearch exception [type=script_exception, reason=C]", expectedCause);
expected.addHeader("script_name", "my_script");
expected.addMetadata("opensearch.lang", "painless");
expected.addMetadata("opensearch.script", "test");
expected.addMetadata("opensearch.script_stack", "stack");
suppressed = new OpenSearchException("OpenSearch exception [type=script_exception, reason=C]");
suppressed.addHeader("script_name", "my_script");
suppressed.addMetadata("opensearch.lang", "painless");
suppressed.addMetadata("opensearch.script", "test");
suppressed.addMetadata("opensearch.script_stack", "stack");
expected.addSuppressed(suppressed);
break;
case // JDK exception without cause
3:
failure = new IllegalStateException("D");
expected = new OpenSearchException("OpenSearch exception [type=illegal_state_exception, reason=D]");
suppressed = new OpenSearchException("OpenSearch exception [type=illegal_state_exception, reason=D]");
expected.addSuppressed(suppressed);
break;
case // JDK exception with cause
4:
failureCause = new RoutingMissingException("idx", "id");
failure = new RuntimeException("E", failureCause);
expectedCause = new OpenSearchException("OpenSearch exception [type=routing_missing_exception, " + "reason=routing is required for [idx]/[id]]");
expectedCause.addMetadata("opensearch.index", "idx");
expectedCause.addMetadata("opensearch.index_uuid", "_na_");
expected = new OpenSearchException("OpenSearch exception [type=runtime_exception, reason=E]", expectedCause);
suppressed = new OpenSearchException("OpenSearch exception [type=runtime_exception, reason=E]");
expected.addSuppressed(suppressed);
break;
case // Wrapped exception with cause
5:
failureCause = new FileAlreadyExistsException("File exists");
failure = new BroadcastShardOperationFailedException(new ShardId("_index", "_uuid", 5), "F", failureCause);
expected = new OpenSearchException("OpenSearch exception [type=file_already_exists_exception, reason=File exists]");
suppressed = new OpenSearchException("OpenSearch exception [type=file_already_exists_exception, reason=File exists]");
expected.addSuppressed(suppressed);
break;
case // SearchPhaseExecutionException with cause and multiple failures
6:
DiscoveryNode node = new DiscoveryNode("node_g", buildNewFakeTransportAddress(), Version.CURRENT);
failureCause = new NodeClosedException(node);
failureCause = new NoShardAvailableActionException(new ShardId("_index_g", "_uuid_g", 6), "node_g", failureCause);
ShardSearchFailure[] shardFailures = new ShardSearchFailure[] { new ShardSearchFailure(new ParsingException(0, 0, "Parsing g", null), new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 61), null, OriginalIndices.NONE)), new ShardSearchFailure(new RepositoryException("repository_g", "Repo"), new SearchShardTarget("node_g", new ShardId(new Index("_index_g", "_uuid_g"), 62), null, OriginalIndices.NONE)), new ShardSearchFailure(new SearchContextMissingException(new ShardSearchContextId(UUIDs.randomBase64UUID(), 0L)), null) };
failure = new SearchPhaseExecutionException("phase_g", "G", failureCause, shardFailures);
expectedCause = new OpenSearchException("OpenSearch exception [type=node_closed_exception, " + "reason=node closed " + node + "]");
expectedCause = new OpenSearchException("OpenSearch exception [type=no_shard_available_action_exception, " + "reason=node_g]", expectedCause);
expectedCause.addMetadata("opensearch.index", "_index_g");
expectedCause.addMetadata("opensearch.index_uuid", "_uuid_g");
expectedCause.addMetadata("opensearch.shard", "6");
expected = new OpenSearchException("OpenSearch exception [type=search_phase_execution_exception, " + "reason=G]", expectedCause);
expected.addMetadata("opensearch.phase", "phase_g");
expected.addSuppressed(new OpenSearchException("OpenSearch exception [type=parsing_exception, reason=Parsing g]"));
expected.addSuppressed(new OpenSearchException("OpenSearch exception [type=repository_exception, " + "reason=[repository_g] Repo]"));
expected.addSuppressed(new OpenSearchException("OpenSearch exception [type=search_context_missing_exception, " + "reason=No search context found for id [0]]"));
break;
default:
throw new UnsupportedOperationException("Failed to generate randomized failure");
}
Exception finalFailure = failure;
BytesReference failureBytes = toShuffledXContent((builder, params) -> {
OpenSearchException.generateFailureXContent(builder, params, finalFailure, true);
return builder;
}, xContent.type(), ToXContent.EMPTY_PARAMS, randomBoolean());
try (XContentParser parser = createParser(xContent, failureBytes)) {
failureBytes = BytesReference.bytes(shuffleXContent(parser, randomBoolean()));
}
OpenSearchException parsedFailure;
try (XContentParser parser = createParser(xContent, failureBytes)) {
assertEquals(XContentParser.Token.START_OBJECT, parser.nextToken());
assertEquals(XContentParser.Token.FIELD_NAME, parser.nextToken());
parsedFailure = OpenSearchException.failureFromXContent(parser);
assertEquals(XContentParser.Token.END_OBJECT, parser.nextToken());
assertNull(parser.nextToken());
}
assertDeepEquals(expected, parsedFailure);
}
use of org.opensearch.client.transport.NoNodeAvailableException in project OpenSearch by opensearch-project.
the class TransportReplicationActionTests method testReplicaProxy.
public void testReplicaProxy() throws InterruptedException, ExecutionException {
final String index = "test";
final ShardId shardId = new ShardId(index, "_na_", 0);
ClusterState state = stateWithActivePrimary(index, true, 1 + randomInt(3), randomInt(2));
logger.info("using state: {}", state);
setState(clusterService, state);
final long primaryTerm = state.metadata().index(index).primaryTerm(0);
ReplicationOperation.Replicas<Request> proxy = action.newReplicasProxy();
// check that at unknown node fails
PlainActionFuture<ReplicaResponse> listener = new PlainActionFuture<>();
ShardRoutingState routingState = randomFrom(ShardRoutingState.INITIALIZING, ShardRoutingState.STARTED, ShardRoutingState.RELOCATING);
proxy.performOn(TestShardRouting.newShardRouting(shardId, "NOT THERE", routingState == ShardRoutingState.RELOCATING ? state.nodes().iterator().next().getId() : null, false, routingState), new Request(NO_SHARD_ID), primaryTerm, randomNonNegativeLong(), randomNonNegativeLong(), listener);
assertTrue(listener.isDone());
assertListenerThrows("non existent node should throw a NoNodeAvailableException", listener, NoNodeAvailableException.class);
final IndexShardRoutingTable shardRoutings = state.routingTable().shardRoutingTable(shardId);
final ShardRouting replica = randomFrom(shardRoutings.replicaShards().stream().filter(ShardRouting::assignedToNode).collect(Collectors.toList()));
listener = new PlainActionFuture<>();
proxy.performOn(replica, new Request(NO_SHARD_ID), primaryTerm, randomNonNegativeLong(), randomNonNegativeLong(), listener);
assertFalse(listener.isDone());
CapturingTransport.CapturedRequest[] captures = transport.getCapturedRequestsAndClear();
assertThat(captures, arrayWithSize(1));
if (randomBoolean()) {
final TransportReplicationAction.ReplicaResponse response = new TransportReplicationAction.ReplicaResponse(randomLong(), randomLong());
transport.handleResponse(captures[0].requestId, response);
assertTrue(listener.isDone());
assertThat(listener.get(), equalTo(response));
} else if (randomBoolean()) {
transport.handleRemoteError(captures[0].requestId, new OpenSearchException("simulated"));
assertTrue(listener.isDone());
assertListenerThrows("listener should reflect remote error", listener, OpenSearchException.class);
} else {
transport.handleError(captures[0].requestId, new TransportException("simulated"));
assertTrue(listener.isDone());
assertListenerThrows("listener should reflect remote error", listener, TransportException.class);
}
AtomicReference<Object> failure = new AtomicReference<>();
AtomicBoolean success = new AtomicBoolean();
proxy.failShardIfNeeded(replica, primaryTerm, "test", new OpenSearchException("simulated"), ActionListener.wrap(r -> success.set(true), failure::set));
CapturingTransport.CapturedRequest[] shardFailedRequests = transport.getCapturedRequestsAndClear();
// A replication action doesn't not fail the request
assertEquals(0, shardFailedRequests.length);
}
use of org.opensearch.client.transport.NoNodeAvailableException in project OpenSearch by opensearch-project.
the class UpdateIT method testStressUpdateDeleteConcurrency.
public void testStressUpdateDeleteConcurrency() throws Exception {
// We create an index with merging disabled so that deletes don't get merged away
assertAcked(prepareCreate("test").setSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)));
ensureGreen();
Script fieldIncScript = new Script(ScriptType.INLINE, UPDATE_SCRIPTS, FIELD_INC_SCRIPT, Collections.singletonMap("field", "field"));
final int numberOfThreads = scaledRandomIntBetween(3, 5);
final int numberOfIdsPerThread = scaledRandomIntBetween(3, 10);
final int numberOfUpdatesPerId = scaledRandomIntBetween(10, 100);
final int retryOnConflict = randomIntBetween(0, 1);
final CountDownLatch latch = new CountDownLatch(numberOfThreads);
final CountDownLatch startLatch = new CountDownLatch(1);
final List<Throwable> failures = new CopyOnWriteArrayList<>();
final class UpdateThread extends Thread {
final Map<Integer, Integer> failedMap = new HashMap<>();
final int numberOfIds;
final int maxUpdateRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
final int maxDeleteRequests = numberOfIdsPerThread * numberOfUpdatesPerId;
private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);
private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);
UpdateThread(int numberOfIds) {
this.numberOfIds = numberOfIds;
}
final class UpdateListener implements ActionListener<UpdateResponse> {
int id;
UpdateListener(int id) {
this.id = id;
}
@Override
public void onResponse(UpdateResponse updateResponse) {
updateRequestsOutstanding.release(1);
}
@Override
public void onFailure(Exception e) {
synchronized (failedMap) {
incrementMapValue(id, failedMap);
}
updateRequestsOutstanding.release(1);
}
}
final class DeleteListener implements ActionListener<DeleteResponse> {
int id;
DeleteListener(int id) {
this.id = id;
}
@Override
public void onResponse(DeleteResponse deleteResponse) {
deleteRequestsOutstanding.release(1);
}
@Override
public void onFailure(Exception e) {
synchronized (failedMap) {
incrementMapValue(id, failedMap);
}
deleteRequestsOutstanding.release(1);
}
}
@Override
public void run() {
try {
startLatch.await();
boolean hasWaitedForNoNode = false;
for (int j = 0; j < numberOfIds; j++) {
for (int k = 0; k < numberOfUpdatesPerId; ++k) {
updateRequestsOutstanding.acquire();
try {
UpdateRequest ur = client().prepareUpdate("test", Integer.toString(j)).setScript(fieldIncScript).setRetryOnConflict(retryOnConflict).setUpsert(jsonBuilder().startObject().field("field", 1).endObject()).request();
if (randomBoolean()) {
client().update(ur, new UpdateListener(j));
} else {
client().prepareBulk().add(ur).execute(ActionListener.map(new UpdateListener(j), br -> {
final BulkItemResponse ir = br.getItems()[0];
if (ir.isFailed()) {
throw ir.getFailure().getCause();
} else {
return ir.getResponse();
}
}));
}
} catch (NoNodeAvailableException nne) {
updateRequestsOutstanding.release();
synchronized (failedMap) {
incrementMapValue(j, failedMap);
}
if (hasWaitedForNoNode) {
throw nne;
}
logger.warn("Got NoNodeException waiting for 1 second for things to recover.");
hasWaitedForNoNode = true;
Thread.sleep(1000);
}
try {
deleteRequestsOutstanding.acquire();
DeleteRequest dr = client().prepareDelete("test", Integer.toString(j)).request();
client().delete(dr, new DeleteListener(j));
} catch (NoNodeAvailableException nne) {
deleteRequestsOutstanding.release();
synchronized (failedMap) {
incrementMapValue(j, failedMap);
}
if (hasWaitedForNoNode) {
throw nne;
}
logger.warn("Got NoNodeException waiting for 1 second for things to recover.");
hasWaitedForNoNode = true;
// Wait for no-node to clear
Thread.sleep(1000);
}
}
}
} catch (Exception e) {
logger.error("Something went wrong", e);
failures.add(e);
} finally {
try {
waitForOutstandingRequests(TimeValue.timeValueSeconds(60), updateRequestsOutstanding, maxUpdateRequests, "Update");
waitForOutstandingRequests(TimeValue.timeValueSeconds(60), deleteRequestsOutstanding, maxDeleteRequests, "Delete");
} catch (OpenSearchTimeoutException ete) {
failures.add(ete);
}
latch.countDown();
}
}
private void incrementMapValue(int j, Map<Integer, Integer> map) {
if (!map.containsKey(j)) {
map.put(j, 0);
}
map.put(j, map.get(j) + 1);
}
private void waitForOutstandingRequests(TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
long start = System.currentTimeMillis();
do {
long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
logger.info("[{}] going to try and acquire [{}] in [{}]ms [{}] available to acquire right now", name, maxRequests, msRemaining, requestsOutstanding.availablePermits());
try {
requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
return;
} catch (InterruptedException ie) {
// Just keep swimming
}
} while ((System.currentTimeMillis() - start) < timeOut.getMillis());
throw new OpenSearchTimeoutException("Requests were still outstanding after the timeout [" + timeOut + "] for type [" + name + "]");
}
}
final List<UpdateThread> threads = new ArrayList<>();
for (int i = 0; i < numberOfThreads; i++) {
UpdateThread ut = new UpdateThread(numberOfIdsPerThread);
ut.start();
threads.add(ut);
}
startLatch.countDown();
latch.await();
for (UpdateThread ut : threads) {
// Threads should have finished because of the latch.await
ut.join();
}
// aquiring the request outstanding semaphores.
for (Throwable throwable : failures) {
logger.info("Captured failure on concurrent update:", throwable);
}
assertThat(failures.size(), equalTo(0));
// All the previous operations should be complete or failed at this point
for (int i = 0; i < numberOfIdsPerThread; ++i) {
client().prepareUpdate("test", Integer.toString(i)).setScript(fieldIncScript).setRetryOnConflict(Integer.MAX_VALUE).setUpsert(jsonBuilder().startObject().field("field", 1).endObject()).execute().actionGet();
}
refresh();
for (int i = 0; i < numberOfIdsPerThread; ++i) {
int totalFailures = 0;
GetResponse response = client().prepareGet("test", Integer.toString(i)).execute().actionGet();
if (response.isExists()) {
assertThat(response.getId(), equalTo(Integer.toString(i)));
int expectedVersion = (numberOfThreads * numberOfUpdatesPerId * 2) + 1;
for (UpdateThread ut : threads) {
if (ut.failedMap.containsKey(i)) {
totalFailures += ut.failedMap.get(i);
}
}
expectedVersion -= totalFailures;
logger.error("Actual version [{}] Expected version [{}] Total failures [{}]", response.getVersion(), expectedVersion, totalFailures);
assertThat(response.getVersion(), equalTo((long) expectedVersion));
assertThat(response.getVersion() + totalFailures, equalTo((long) ((numberOfUpdatesPerId * numberOfThreads * 2) + 1)));
}
}
}
use of org.opensearch.client.transport.NoNodeAvailableException in project OpenSearch by opensearch-project.
the class BulkByScrollResponseTests method randomSearchFailures.
private List<ScrollableHitSource.SearchFailure> randomSearchFailures() {
if (randomBoolean()) {
return emptyList();
}
String index = null;
Integer shardId = null;
String nodeId = null;
if (randomBoolean()) {
index = randomAlphaOfLength(5);
shardId = randomInt();
nodeId = usually() ? randomAlphaOfLength(5) : null;
}
OpenSearchException exception = randomFrom(new ResourceNotFoundException("bar"), new OpenSearchException("foo"), new NoNodeAvailableException("baz"));
return singletonList(new ScrollableHitSource.SearchFailure(exception, index, shardId, nodeId));
}
Aggregations