Search in sources :

Example 6 with ConnectTransportException

use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.

the class RetryOnFailureResultReceiver method fail.

@Override
public void fail(@Nonnull Throwable wrappedError) {
    final Throwable error = SQLExceptions.unwrap(wrappedError);
    if (attempt <= Constants.MAX_SHARD_MISSING_RETRIES && (SQLExceptions.isShardFailure(error) || error instanceof ConnectTransportException || indexWasTemporaryUnavailable(error))) {
        if (clusterService.state().blocks().hasGlobalBlockWithStatus(RestStatus.SERVICE_UNAVAILABLE)) {
            delegate.fail(error);
        } else {
            ClusterStateObserver clusterStateObserver = new ClusterStateObserver(initialState, clusterService, null, LOGGER);
            clusterStateObserver.waitForNextChange(new ClusterStateObserver.Listener() {

                @Override
                public void onNewClusterState(ClusterState state) {
                    attempt += 1;
                    retry();
                }

                @Override
                public void onClusterServiceClose() {
                    delegate.fail(error);
                }

                @Override
                public void onTimeout(TimeValue timeout) {
                    delegate.fail(error);
                }
            });
        }
    } else {
        delegate.fail(error);
    }
}
Also used : ClusterState(org.elasticsearch.cluster.ClusterState) ClusterStateObserver(org.elasticsearch.cluster.ClusterStateObserver) ConnectTransportException(org.elasticsearch.transport.ConnectTransportException) TimeValue(io.crate.common.unit.TimeValue)

Example 7 with ConnectTransportException

use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.

the class LeaderCheckerTests method testFollowerFailsImmediatelyOnDisconnection.

public void testFollowerFailsImmediatelyOnDisconnection() {
    final DiscoveryNode localNode = new DiscoveryNode("local-node", buildNewFakeTransportAddress(), Version.CURRENT);
    final DiscoveryNode leader = new DiscoveryNode("leader", buildNewFakeTransportAddress(), Version.CURRENT);
    final Response[] responseHolder = new Response[] { Response.SUCCESS };
    final Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), localNode.getId()).build();
    final DeterministicTaskQueue deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
    final MockTransport mockTransport = new MockTransport() {

        @Override
        protected void onSendRequest(long requestId, String action, TransportRequest request, DiscoveryNode node) {
            if (action.equals(HANDSHAKE_ACTION_NAME)) {
                handleResponse(requestId, new TransportService.HandshakeResponse(node, ClusterName.DEFAULT, Version.CURRENT));
                return;
            }
            assertThat(action, equalTo(LEADER_CHECK_ACTION_NAME));
            assertTrue(node.equals(leader));
            final Response response = responseHolder[0];
            deterministicTaskQueue.scheduleNow(new Runnable() {

                @Override
                public void run() {
                    switch(response) {
                        case SUCCESS:
                            handleResponse(requestId, Empty.INSTANCE);
                            break;
                        case REMOTE_ERROR:
                            handleRemoteError(requestId, new ConnectTransportException(leader, "simulated error"));
                            break;
                        case DIRECT_ERROR:
                            handleError(requestId, new ConnectTransportException(leader, "simulated error"));
                    }
                }

                @Override
                public String toString() {
                    return response + " response to request " + requestId;
                }
            });
        }
    };
    final TransportService transportService = mockTransport.createTransportService(settings, deterministicTaskQueue.getThreadPool(), boundTransportAddress -> localNode, null);
    transportService.start();
    transportService.acceptIncomingRequests();
    final AtomicBoolean leaderFailed = new AtomicBoolean();
    final LeaderChecker leaderChecker = new LeaderChecker(settings, transportService, e -> {
        assertThat(e.getMessage(), anyOf(endsWith("disconnected"), endsWith("disconnected during check")));
        assertTrue(leaderFailed.compareAndSet(false, true));
    });
    leaderChecker.updateLeader(leader);
    {
        while (deterministicTaskQueue.getCurrentTimeMillis() < 10 * LEADER_CHECK_INTERVAL_SETTING.get(Settings.EMPTY).millis()) {
            deterministicTaskQueue.runAllRunnableTasks();
            deterministicTaskQueue.advanceTime();
        }
        deterministicTaskQueue.runAllRunnableTasks();
        assertFalse(leaderFailed.get());
        responseHolder[0] = Response.REMOTE_ERROR;
        deterministicTaskQueue.advanceTime();
        deterministicTaskQueue.runAllRunnableTasks();
        assertTrue(leaderFailed.get());
    }
    leaderChecker.updateLeader(null);
    deterministicTaskQueue.runAllTasks();
    leaderFailed.set(false);
    responseHolder[0] = Response.SUCCESS;
    leaderChecker.updateLeader(leader);
    {
        while (deterministicTaskQueue.getCurrentTimeMillis() < 10 * LEADER_CHECK_INTERVAL_SETTING.get(Settings.EMPTY).millis()) {
            deterministicTaskQueue.runAllRunnableTasks();
            deterministicTaskQueue.advanceTime();
        }
        deterministicTaskQueue.runAllRunnableTasks();
        assertFalse(leaderFailed.get());
        responseHolder[0] = Response.DIRECT_ERROR;
        deterministicTaskQueue.advanceTime();
        deterministicTaskQueue.runAllRunnableTasks();
        assertTrue(leaderFailed.get());
    }
    deterministicTaskQueue.runAllTasks();
    leaderFailed.set(false);
    responseHolder[0] = Response.SUCCESS;
    leaderChecker.updateLeader(leader);
    {
        // need to connect first for disconnect to have any effect
        AbstractSimpleTransportTestCase.connectToNode(transportService, leader);
        transportService.disconnectFromNode(leader);
        deterministicTaskQueue.runAllRunnableTasks();
        assertTrue(leaderFailed.get());
    }
}
Also used : DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) TransportRequest(org.elasticsearch.transport.TransportRequest) TransportResponse(org.elasticsearch.transport.TransportResponse) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TransportService(org.elasticsearch.transport.TransportService) ConnectTransportException(org.elasticsearch.transport.ConnectTransportException) MockTransport(org.elasticsearch.test.transport.MockTransport) Settings(org.elasticsearch.common.settings.Settings)

Example 8 with ConnectTransportException

use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.

the class RetryOnFailureResultReceiverTest method testRetryOnNodeConnectionError.

@Test
public void testRetryOnNodeConnectionError() throws Exception {
    AtomicInteger numRetries = new AtomicInteger(0);
    BaseResultReceiver baseResultReceiver = new BaseResultReceiver();
    ClusterState initialState = clusterService.state();
    RetryOnFailureResultReceiver retryOnFailureResultReceiver = new RetryOnFailureResultReceiver(clusterService, initialState, indexName -> true, baseResultReceiver, UUID.randomUUID(), (newJobId, receiver) -> numRetries.incrementAndGet());
    // Must have a different cluster state then the initial state to trigger a retry
    clusterService.submitStateUpdateTask("dummy", new DummyUpdate());
    assertBusy(() -> assertThat(initialState, Matchers.not(sameInstance(clusterService.state()))));
    retryOnFailureResultReceiver.fail(new ConnectTransportException(null, "node not connected"));
    assertThat(numRetries.get(), is(1));
}
Also used : ClusterState(org.elasticsearch.cluster.ClusterState) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ConnectTransportException(org.elasticsearch.transport.ConnectTransportException) BaseResultReceiver(io.crate.action.sql.BaseResultReceiver) Test(org.junit.Test) CrateDummyClusterServiceUnitTest(io.crate.test.integration.CrateDummyClusterServiceUnitTest)

Example 9 with ConnectTransportException

use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.

the class DisruptableMockTransport method openConnection.

@Override
public void openConnection(DiscoveryNode node, ConnectionProfile profile, ActionListener<Connection> listener) {
    final Optional<DisruptableMockTransport> optionalMatchingTransport = getDisruptableMockTransport(node.getAddress());
    if (optionalMatchingTransport.isPresent()) {
        final DisruptableMockTransport matchingTransport = optionalMatchingTransport.get();
        final ConnectionStatus connectionStatus = getConnectionStatus(matchingTransport.getLocalNode());
        if (connectionStatus != ConnectionStatus.CONNECTED) {
            listener.onFailure(new ConnectTransportException(node, "node [" + node + "] is [" + connectionStatus + "] not [CONNECTED]"));
        } else {
            listener.onResponse(new CloseableConnection() {

                @Override
                public DiscoveryNode getNode() {
                    return node;
                }

                @Override
                public void sendRequest(long requestId, String action, TransportRequest request, TransportRequestOptions options) throws TransportException {
                    onSendRequest(requestId, action, request, matchingTransport);
                }
            });
        }
    } else {
        listener.onFailure(new ConnectTransportException(node, "node " + node + " does not exist"));
    }
}
Also used : DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) TransportRequest(org.elasticsearch.transport.TransportRequest) ConnectTransportException(org.elasticsearch.transport.ConnectTransportException) CloseableConnection(org.elasticsearch.transport.CloseableConnection) TransportRequestOptions(org.elasticsearch.transport.TransportRequestOptions) ConnectTransportException(org.elasticsearch.transport.ConnectTransportException) TransportException(org.elasticsearch.transport.TransportException)

Example 10 with ConnectTransportException

use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.

the class HandshakingTransportAddressConnector method connectToRemoteMasterNode.

@Override
public void connectToRemoteMasterNode(TransportAddress transportAddress, ActionListener<DiscoveryNode> listener) {
    transportService.getThreadPool().generic().execute(new ActionRunnable<>(listener) {

        @Override
        protected void doRun() {
            // TODO if transportService is already connected to this address then skip the handshaking
            final DiscoveryNode targetNode = new DiscoveryNode("", transportAddress.toString(), // generated deterministically for reproducible tests
            UUIDs.randomBase64UUID(Randomness.get()), transportAddress.address().getHostString(), transportAddress.getAddress(), transportAddress, emptyMap(), emptySet(), Version.CURRENT.minimumCompatibilityVersion());
            LOGGER.trace("[{}] opening probe connection", this);
            transportService.openConnection(targetNode, ConnectionProfile.buildSingleChannelProfile(Type.REG, probeConnectTimeout, probeHandshakeTimeout, TimeValue.MINUS_ONE, null), new ActionListener<>() {

                @Override
                public void onResponse(Connection connection) {
                    LOGGER.trace("[{}] opened probe connection", this);
                    // use NotifyOnceListener to make sure the following line does not result in onFailure being called when
                    // the connection is closed in the onResponse handler
                    transportService.handshake(connection, probeHandshakeTimeout.millis(), new NotifyOnceListener<DiscoveryNode>() {

                        @Override
                        protected void innerOnResponse(DiscoveryNode remoteNode) {
                            try {
                                // success means (amongst other things) that the cluster names match
                                LOGGER.trace("[{}] handshake successful: {}", this, remoteNode);
                                IOUtils.closeWhileHandlingException(connection);
                                if (remoteNode.equals(transportService.getLocalNode())) {
                                    // TODO cache this result for some time? forever?
                                    listener.onFailure(new ConnectTransportException(remoteNode, "local node found"));
                                } else if (remoteNode.isMasterEligibleNode() == false) {
                                    // TODO cache this result for some time?
                                    listener.onFailure(new ConnectTransportException(remoteNode, "non-master-eligible node found"));
                                } else {
                                    transportService.connectToNode(remoteNode, new ActionListener<Void>() {

                                        @Override
                                        public void onResponse(Void ignored) {
                                            LOGGER.trace("[{}] full connection successful: {}", this, remoteNode);
                                            listener.onResponse(remoteNode);
                                        }

                                        @Override
                                        public void onFailure(Exception e) {
                                            listener.onFailure(e);
                                        }
                                    });
                                }
                            } catch (Exception e) {
                                listener.onFailure(e);
                            }
                        }

                        @Override
                        protected void innerOnFailure(Exception e) {
                            // we opened a connection and successfully performed a low-level handshake, so we were definitely
                            // talking to an Elasticsearch node, but the high-level handshake failed indicating some kind of
                            // mismatched configurations (e.g. cluster name) that the user should address
                            LOGGER.warn(new ParameterizedMessage("handshake failed for [{}]", this), e);
                            IOUtils.closeWhileHandlingException(connection);
                            listener.onFailure(e);
                        }
                    });
                }

                @Override
                public void onFailure(Exception e) {
                    listener.onFailure(e);
                }
            });
        }

        @Override
        public String toString() {
            return "connectToRemoteMasterNode[" + transportAddress + "]";
        }
    });
}
Also used : DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) ActionListener(org.elasticsearch.action.ActionListener) ConnectTransportException(org.elasticsearch.transport.ConnectTransportException) Connection(org.elasticsearch.transport.Transport.Connection) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) ConnectTransportException(org.elasticsearch.transport.ConnectTransportException)

Aggregations

ConnectTransportException (org.elasticsearch.transport.ConnectTransportException)22 DiscoveryNode (org.elasticsearch.cluster.node.DiscoveryNode)12 IOException (java.io.IOException)11 TransportRequest (org.elasticsearch.transport.TransportRequest)10 TransportRequestOptions (org.elasticsearch.transport.TransportRequestOptions)9 TransportService (org.elasticsearch.transport.TransportService)8 ClusterState (org.elasticsearch.cluster.ClusterState)7 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)6 Settings (org.elasticsearch.common.settings.Settings)6 TimeValue (org.elasticsearch.common.unit.TimeValue)6 MockTransportService (org.elasticsearch.test.transport.MockTransportService)5 TransportException (org.elasticsearch.transport.TransportException)5 TimeValue (io.crate.common.unit.TimeValue)4 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)4 ElasticsearchException (org.elasticsearch.ElasticsearchException)4 AbstractRunnable (org.elasticsearch.common.util.concurrent.AbstractRunnable)4 TransportResponseHandler (org.elasticsearch.transport.TransportResponseHandler)4 ArrayList (java.util.ArrayList)3 HashSet (java.util.HashSet)3 CountDownLatch (java.util.concurrent.CountDownLatch)3