use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.
the class RetryOnFailureResultReceiver method fail.
@Override
public void fail(@Nonnull Throwable wrappedError) {
final Throwable error = SQLExceptions.unwrap(wrappedError);
if (attempt <= Constants.MAX_SHARD_MISSING_RETRIES && (SQLExceptions.isShardFailure(error) || error instanceof ConnectTransportException || indexWasTemporaryUnavailable(error))) {
if (clusterService.state().blocks().hasGlobalBlockWithStatus(RestStatus.SERVICE_UNAVAILABLE)) {
delegate.fail(error);
} else {
ClusterStateObserver clusterStateObserver = new ClusterStateObserver(initialState, clusterService, null, LOGGER);
clusterStateObserver.waitForNextChange(new ClusterStateObserver.Listener() {
@Override
public void onNewClusterState(ClusterState state) {
attempt += 1;
retry();
}
@Override
public void onClusterServiceClose() {
delegate.fail(error);
}
@Override
public void onTimeout(TimeValue timeout) {
delegate.fail(error);
}
});
}
} else {
delegate.fail(error);
}
}
use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.
the class LeaderCheckerTests method testFollowerFailsImmediatelyOnDisconnection.
public void testFollowerFailsImmediatelyOnDisconnection() {
final DiscoveryNode localNode = new DiscoveryNode("local-node", buildNewFakeTransportAddress(), Version.CURRENT);
final DiscoveryNode leader = new DiscoveryNode("leader", buildNewFakeTransportAddress(), Version.CURRENT);
final Response[] responseHolder = new Response[] { Response.SUCCESS };
final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), localNode.getId()).build();
final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
final MockTransport mockTransport = new MockTransport() {
@Override
protected void onSendRequest(long requestId, String action, TransportRequest request, DiscoveryNode node) {
if (action.equals(HANDSHAKE_ACTION_NAME)) {
handleResponse(requestId, new TransportService.HandshakeResponse(node, ClusterName.DEFAULT, Version.CURRENT));
return;
}
assertThat(action, equalTo(LEADER_CHECK_ACTION_NAME));
assertTrue(node.equals(leader));
final Response response = responseHolder[0];
deterministicTaskQueue.scheduleNow(new Runnable() {
@Override
public void run() {
switch(response) {
case SUCCESS:
handleResponse(requestId, Empty.INSTANCE);
break;
case REMOTE_ERROR:
handleRemoteError(requestId, new ConnectTransportException(leader, "simulated error"));
break;
case DIRECT_ERROR:
handleError(requestId, new ConnectTransportException(leader, "simulated error"));
}
}
@Override
public String toString() {
return response + " response to request " + requestId;
}
});
}
};
final TransportService transportService = mockTransport.createTransportService(settings, deterministicTaskQueue.getThreadPool(), boundTransportAddress -> localNode, null);
transportService.start();
transportService.acceptIncomingRequests();
final AtomicBoolean leaderFailed = new AtomicBoolean();
final LeaderChecker leaderChecker = new LeaderChecker(settings, transportService, e -> {
assertThat(e.getMessage(), anyOf(endsWith("disconnected"), endsWith("disconnected during check")));
assertTrue(leaderFailed.compareAndSet(false, true));
});
leaderChecker.updateLeader(leader);
{
while (deterministicTaskQueue.getCurrentTimeMillis() < 10 * LEADER_CHECK_INTERVAL_SETTING.get(Settings.EMPTY).millis()) {
deterministicTaskQueue.runAllRunnableTasks();
deterministicTaskQueue.advanceTime();
}
deterministicTaskQueue.runAllRunnableTasks();
assertFalse(leaderFailed.get());
responseHolder[0] = Response.REMOTE_ERROR;
deterministicTaskQueue.advanceTime();
deterministicTaskQueue.runAllRunnableTasks();
assertTrue(leaderFailed.get());
}
leaderChecker.updateLeader(null);
deterministicTaskQueue.runAllTasks();
leaderFailed.set(false);
responseHolder[0] = Response.SUCCESS;
leaderChecker.updateLeader(leader);
{
while (deterministicTaskQueue.getCurrentTimeMillis() < 10 * LEADER_CHECK_INTERVAL_SETTING.get(Settings.EMPTY).millis()) {
deterministicTaskQueue.runAllRunnableTasks();
deterministicTaskQueue.advanceTime();
}
deterministicTaskQueue.runAllRunnableTasks();
assertFalse(leaderFailed.get());
responseHolder[0] = Response.DIRECT_ERROR;
deterministicTaskQueue.advanceTime();
deterministicTaskQueue.runAllRunnableTasks();
assertTrue(leaderFailed.get());
}
deterministicTaskQueue.runAllTasks();
leaderFailed.set(false);
responseHolder[0] = Response.SUCCESS;
leaderChecker.updateLeader(leader);
{
// need to connect first for disconnect to have any effect
AbstractSimpleTransportTestCase.connectToNode(transportService, leader);
transportService.disconnectFromNode(leader);
deterministicTaskQueue.runAllRunnableTasks();
assertTrue(leaderFailed.get());
}
}
use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.
the class RetryOnFailureResultReceiverTest method testRetryOnNodeConnectionError.
@Test
public void testRetryOnNodeConnectionError() throws Exception {
AtomicInteger numRetries = new AtomicInteger(0);
BaseResultReceiver baseResultReceiver = new BaseResultReceiver();
ClusterState initialState = clusterService.state();
RetryOnFailureResultReceiver retryOnFailureResultReceiver = new RetryOnFailureResultReceiver(clusterService, initialState, indexName -> true, baseResultReceiver, UUID.randomUUID(), (newJobId, receiver) -> numRetries.incrementAndGet());
// Must have a different cluster state then the initial state to trigger a retry
clusterService.submitStateUpdateTask("dummy", new DummyUpdate());
assertBusy(() -> assertThat(initialState, Matchers.not(sameInstance(clusterService.state()))));
retryOnFailureResultReceiver.fail(new ConnectTransportException(null, "node not connected"));
assertThat(numRetries.get(), is(1));
}
use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.
the class DisruptableMockTransport method openConnection.
@Override
public void openConnection(DiscoveryNode node, ConnectionProfile profile, ActionListener<Connection> listener) {
final Optional<DisruptableMockTransport> optionalMatchingTransport = getDisruptableMockTransport(node.getAddress());
if (optionalMatchingTransport.isPresent()) {
final DisruptableMockTransport matchingTransport = optionalMatchingTransport.get();
final ConnectionStatus connectionStatus = getConnectionStatus(matchingTransport.getLocalNode());
if (connectionStatus != ConnectionStatus.CONNECTED) {
listener.onFailure(new ConnectTransportException(node, "node [" + node + "] is [" + connectionStatus + "] not [CONNECTED]"));
} else {
listener.onResponse(new CloseableConnection() {
@Override
public DiscoveryNode getNode() {
return node;
}
@Override
public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) throws TransportException {
onSendRequest(requestId, action, request, matchingTransport);
}
});
}
} else {
listener.onFailure(new ConnectTransportException(node, "node " + node + " does not exist"));
}
}
use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.
the class HandshakingTransportAddressConnector method connectToRemoteMasterNode.
@Override
public void connectToRemoteMasterNode(TransportAddress transportAddress, ActionListener<DiscoveryNode> listener) {
transportService.getThreadPool().generic().execute(new ActionRunnable<>(listener) {
@Override
protected void doRun() {
// TODO if transportService is already connected to this address then skip the handshaking
final DiscoveryNode targetNode = new DiscoveryNode("", transportAddress.toString(), // generated deterministically for reproducible tests
UUIDs.randomBase64UUID(Randomness.get()), transportAddress.address().getHostString(), transportAddress.getAddress(), transportAddress, emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion());
LOGGER.trace("[{}] opening probe connection", this);
transportService.openConnection(targetNode, ConnectionProfile.buildSingleChannelProfile(Type.REG, probeConnectTimeout, probeHandshakeTimeout, TimeValue.MINUS_ONE, null), new ActionListener<>() {
@Override
public void onResponse(Connection connection) {
LOGGER.trace("[{}] opened probe connection", this);
// use NotifyOnceListener to make sure the following line does not result in onFailure being called when
// the connection is closed in the onResponse handler
transportService.handshake(connection, probeHandshakeTimeout.millis(), new NotifyOnceListener<DiscoveryNode>() {
@Override
protected void innerOnResponse(DiscoveryNode remoteNode) {
try {
// success means (amongst other things) that the cluster names match
LOGGER.trace("[{}] handshake successful: {}", this, remoteNode);
IOUtils.closeWhileHandlingException(connection);
if (remoteNode.equals(transportService.getLocalNode())) {
// TODO cache this result for some time? forever?
listener.onFailure(new ConnectTransportException(remoteNode, "local node found"));
} else if (remoteNode.isMasterEligibleNode() == false) {
// TODO cache this result for some time?
listener.onFailure(new ConnectTransportException(remoteNode, "non-master-eligible node found"));
} else {
transportService.connectToNode(remoteNode, new ActionListener<Void>() {
@Override
public void onResponse(Void ignored) {
LOGGER.trace("[{}] full connection successful: {}", this, remoteNode);
listener.onResponse(remoteNode);
}
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
});
}
} catch (Exception e) {
listener.onFailure(e);
}
}
@Override
protected void innerOnFailure(Exception e) {
// we opened a connection and successfully performed a low-level handshake, so we were definitely
// talking to an Elasticsearch node, but the high-level handshake failed indicating some kind of
// mismatched configurations (e.g. cluster name) that the user should address
LOGGER.warn(new ParameterizedMessage("handshake failed for [{}]", this), e);
IOUtils.closeWhileHandlingException(connection);
listener.onFailure(e);
}
});
}
@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
});
}
@Override
public String toString() {
return "connectToRemoteMasterNode[" + transportAddress + "]";
}
});
}
Aggregations