Examples with ConnectTransportException org.elasticsearch.transport.ConnectTransportException used on opensource projects

Example 16 with ConnectTransportException

use of org.elasticsearch.transport.ConnectTransportException in project elasticsearch by elastic.

the class TransportInstanceSingleOperationActionTests method testRetryOfAnAlreadyTimedOutRequest.

public void testRetryOfAnAlreadyTimedOutRequest() throws Exception {
    Request request = new Request().index("test").timeout(new TimeValue(0, TimeUnit.MILLISECONDS));
    request.shardId = new ShardId("test", "_na_", 0);
    PlainActionFuture<Response> listener = new PlainActionFuture<>();
    setState(clusterService, ClusterStateCreationUtils.state("test", randomBoolean(), ShardRoutingState.STARTED));
    action.new AsyncSingleAction(request, listener).start();
    assertThat(transport.capturedRequests().length, equalTo(1));
    long requestId = transport.capturedRequests()[0].requestId;
    transport.clear();
    DiscoveryNode node = clusterService.state().getNodes().getLocalNode();
    transport.handleLocalError(requestId, new ConnectTransportException(node, "test exception"));
    // wait until the timeout was triggered and we actually tried to send for the second time
    assertBusy(new Runnable() {

        @Override
        public void run() {
            assertThat(transport.capturedRequests().length, equalTo(1));
        }
    });
    // let it fail the second time too
    requestId = transport.capturedRequests()[0].requestId;
    transport.handleLocalError(requestId, new ConnectTransportException(node, "test exception"));
    try {
        // result should return immediately
        assertTrue(listener.isDone());
        listener.get();
        fail("this should fail with a transport exception");
    } catch (ExecutionException t) {
        if (ExceptionsHelper.unwrap(t, ConnectTransportException.class) == null) {
            logger.info("expected ConnectTransportException  but got ", t);
            fail("expected and ConnectTransportException");
        }
    }
}

Example 17 with ConnectTransportException

use of org.elasticsearch.transport.ConnectTransportException in project elasticsearch by elastic.

the class ExceptionSerializationTests method testConnectTransportException.

public void testConnectTransportException() throws IOException {
    TransportAddress transportAddress = buildNewFakeTransportAddress();
    DiscoveryNode node = new DiscoveryNode("thenode", transportAddress, emptyMap(), emptySet(), Version.CURRENT);
    ConnectTransportException ex = serialize(new ConnectTransportException(node, "msg", "action", null));
    assertEquals("[][" + transportAddress.toString() + "][action] msg", ex.getMessage());
    assertEquals(node, ex.node());
    assertEquals("action", ex.action());
    assertNull(ex.getCause());
    ex = serialize(new ConnectTransportException(node, "msg", "action", new NullPointerException()));
    assertEquals("[][" + transportAddress + "][action] msg", ex.getMessage());
    assertEquals(node, ex.node());
    assertEquals("action", ex.action());
    assertTrue(ex.getCause() instanceof NullPointerException);
}

Example 18 with ConnectTransportException

use of org.elasticsearch.transport.ConnectTransportException in project elasticsearch by elastic.

the class MockTransportService method addUnresponsiveRule.

/**
     * Adds a rule that will cause ignores each send request, simulating an unresponsive node
     * and failing to connect once the rule was added.
     *
     * @param duration the amount of time to delay sending and connecting.
     */
public void addUnresponsiveRule(TransportAddress transportAddress, final TimeValue duration) {
    final long startTime = System.currentTimeMillis();
    addDelegate(transportAddress, new ClearableTransport(original) {

        private final Queue<Runnable> requestsToSendWhenCleared = new LinkedBlockingDeque<Runnable>();

        private boolean cleared = false;

        TimeValue getDelay() {
            return new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime));
        }

        @Override
        public void connectToNode(DiscoveryNode node, ConnectionProfile connectionProfile, CheckedBiConsumer<Connection, ConnectionProfile, IOException> connectionValidator) throws ConnectTransportException {
            if (original.nodeConnected(node)) {
                // connecting to an already connected node is a no-op
                return;
            }
            TimeValue delay = getDelay();
            if (delay.millis() <= 0) {
                original.connectToNode(node, connectionProfile, connectionValidator);
                return;
            }
            // TODO: Replace with proper setting
            TimeValue connectingTimeout = NetworkService.TcpSettings.TCP_CONNECT_TIMEOUT.getDefault(Settings.EMPTY);
            try {
                if (delay.millis() < connectingTimeout.millis()) {
                    Thread.sleep(delay.millis());
                    original.connectToNode(node, connectionProfile, connectionValidator);
                } else {
                    Thread.sleep(connectingTimeout.millis());
                    throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
                }
            } catch (InterruptedException e) {
                throw new ConnectTransportException(node, "UNRESPONSIVE: simulated");
            }
        }

        @Override
        protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
            // delayed sending - even if larger then the request timeout to simulated a potential late response from target node
            TimeValue delay = getDelay();
            if (delay.millis() <= 0) {
                connection.sendRequest(requestId, action, request, options);
                return;
            }
            // poor mans request cloning...
            RequestHandlerRegistry reg = MockTransportService.this.getRequestHandler(action);
            BytesStreamOutput bStream = new BytesStreamOutput();
            request.writeTo(bStream);
            final TransportRequest clonedRequest = reg.newRequest();
            clonedRequest.readFrom(bStream.bytes().streamInput());
            Runnable runnable = new AbstractRunnable() {

                AtomicBoolean requestSent = new AtomicBoolean();

                @Override
                public void onFailure(Exception e) {
                    logger.debug("failed to send delayed request", e);
                }

                @Override
                protected void doRun() throws IOException {
                    if (requestSent.compareAndSet(false, true)) {
                        connection.sendRequest(requestId, action, clonedRequest, options);
                    }
                }
            };
            // store the request to send it once the rule is cleared.
            synchronized (this) {
                if (cleared) {
                    runnable.run();
                } else {
                    requestsToSendWhenCleared.add(runnable);
                    threadPool.schedule(delay, ThreadPool.Names.GENERIC, runnable);
                }
            }
        }

        @Override
        public void clearRule() {
            synchronized (this) {
                assert cleared == false;
                cleared = true;
                requestsToSendWhenCleared.forEach(Runnable::run);
            }
        }
    });
}

Example 19 with ConnectTransportException

use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.

the class PeerRecoveryTargetService method doRecovery.

private void doRecovery(final long recoveryId) {
    final StartRecoveryRequest request;
    final RecoveryState.Timer timer;
    CancellableThreads cancellableThreads;
    try (RecoveryRef recoveryRef = onGoingRecoveries.getRecovery(recoveryId)) {
        if (recoveryRef == null) {
            LOGGER.trace("not running recovery with id [{}] - can not find it (probably finished)", recoveryId);
            return;
        }
        final RecoveryTarget recoveryTarget = recoveryRef.target();
        timer = recoveryTarget.state().getTimer();
        cancellableThreads = recoveryTarget.cancellableThreads();
        try {
            assert recoveryTarget.sourceNode() != null : "can not do a recovery without a source node";
            LOGGER.trace("{} preparing shard for peer recovery", recoveryTarget.shardId());
            recoveryTarget.indexShard().prepareForIndexRecovery();
            final long startingSeqNo = recoveryTarget.indexShard().recoverLocallyUpToGlobalCheckpoint();
            assert startingSeqNo == UNASSIGNED_SEQ_NO || recoveryTarget.state().getStage() == RecoveryState.Stage.TRANSLOG : "unexpected recovery stage [" + recoveryTarget.state().getStage() + "] starting seqno [ " + startingSeqNo + "]";
            request = getStartRecoveryRequest(LOGGER, clusterService.localNode(), recoveryTarget, startingSeqNo);
        } catch (final Exception e) {
            // this will be logged as warning later on...
            LOGGER.trace("unexpected error while preparing shard for peer recovery, failing recovery", e);
            onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(recoveryTarget.state(), "failed to prepare shard for recovery", e), true);
            return;
        }
    }
    Consumer<Exception> handleException = e -> {
        if (LOGGER.isTraceEnabled()) {
            LOGGER.trace(() -> new ParameterizedMessage("[{}][{}] Got exception on recovery", request.shardId().getIndex().getName(), request.shardId().id()), e);
        }
        Throwable cause = SQLExceptions.unwrap(e);
        if (cause instanceof CancellableThreads.ExecutionCancelledException) {
            // this can also come from the source wrapped in a RemoteTransportException
            onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, "source has canceled the recovery", cause), false);
            return;
        }
        if (cause instanceof RecoveryEngineException) {
            // unwrap an exception that was thrown as part of the recovery
            cause = cause.getCause();
        }
        // do it twice, in case we have double transport exception
        cause = SQLExceptions.unwrap(cause);
        if (cause instanceof RecoveryEngineException) {
            // unwrap an exception that was thrown as part of the recovery
            cause = cause.getCause();
        }
        if (cause instanceof IllegalIndexShardStateException || cause instanceof IndexNotFoundException || cause instanceof ShardNotFoundException) {
            // if the target is not ready yet, retry
            retryRecovery(recoveryId, "remote shard not ready", recoverySettings.retryDelayStateSync(), recoverySettings.activityTimeout());
            return;
        }
        if (cause instanceof DelayRecoveryException) {
            retryRecovery(recoveryId, cause, recoverySettings.retryDelayStateSync(), recoverySettings.activityTimeout());
            return;
        }
        if (cause instanceof ConnectTransportException) {
            LOGGER.debug("delaying recovery of {} for [{}] due to networking error [{}]", request.shardId(), recoverySettings.retryDelayNetwork(), cause.getMessage());
            retryRecovery(recoveryId, cause.getMessage(), recoverySettings.retryDelayNetwork(), recoverySettings.activityTimeout());
            return;
        }
        if (cause instanceof AlreadyClosedException) {
            onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, "source shard is closed", cause), false);
            return;
        }
        onGoingRecoveries.failRecovery(recoveryId, new RecoveryFailedException(request, e), true);
    };
    try {
        LOGGER.trace("{} starting recovery from {}", request.shardId(), request.sourceNode());
        cancellableThreads.executeIO(() -> transportService.sendRequest(request.sourceNode(), PeerRecoverySourceService.Actions.START_RECOVERY, request, new TransportResponseHandler<RecoveryResponse>() {

            @Override
            public void handleResponse(RecoveryResponse recoveryResponse) {
                final TimeValue recoveryTime = new TimeValue(timer.time());
                // do this through ongoing recoveries to remove it from the collection
                onGoingRecoveries.markRecoveryAsDone(recoveryId);
                if (LOGGER.isTraceEnabled()) {
                    StringBuilder sb = new StringBuilder();
                    sb.append('[').append(request.shardId().getIndex().getName()).append(']').append('[').append(request.shardId().id()).append("] ");
                    sb.append("recovery completed from ").append(request.sourceNode()).append(", took[").append(recoveryTime).append("]\n");
                    sb.append("   phase1: recovered_files [").append(recoveryResponse.phase1FileNames.size()).append("]").append(" with total_size of [").append(new ByteSizeValue(recoveryResponse.phase1TotalSize)).append("]").append(", took [").append(timeValueMillis(recoveryResponse.phase1Time)).append("], throttling_wait [").append(timeValueMillis(recoveryResponse.phase1ThrottlingWaitTime)).append(']').append("\n");
                    sb.append("         : reusing_files   [").append(recoveryResponse.phase1ExistingFileNames.size()).append("] with total_size of [").append(new ByteSizeValue(recoveryResponse.phase1ExistingTotalSize)).append("]\n");
                    sb.append("   phase2: start took [").append(timeValueMillis(recoveryResponse.startTime)).append("]\n");
                    sb.append("         : recovered [").append(recoveryResponse.phase2Operations).append("]").append(" transaction log operations").append(", took [").append(timeValueMillis(recoveryResponse.phase2Time)).append("]").append("\n");
                    LOGGER.trace("{}", sb);
                } else {
                    LOGGER.debug("{} recovery done from [{}], took [{}]", request.shardId(), request.sourceNode(), recoveryTime);
                }
            }

            @Override
            public void handleException(TransportException e) {
                handleException.accept(e);
            }

            @Override
            public String executor() {
                // we do some heavy work like refreshes in the response so fork off to the generic threadpool
                return ThreadPool.Names.GENERIC;
            }

            @Override
            public RecoveryResponse read(StreamInput in) throws IOException {
                return new RecoveryResponse(in);
            }
        }));
    } catch (CancellableThreads.ExecutionCancelledException e) {
        LOGGER.trace("recovery cancelled", e);
    } catch (Exception e) {
        handleException.accept(e);
    }
}

Example 20 with ConnectTransportException

use of org.elasticsearch.transport.ConnectTransportException in project crate by crate.

the class PreVoteCollectorTests method createObjects.

@Before
public void createObjects() {
    Settings settings = Settings.builder().put(NODE_NAME_SETTING.getKey(), "node").build();
    deterministicTaskQueue = new DeterministicTaskQueue(settings, random());
    final MockTransport mockTransport = new MockTransport() {

        @Override
        protected void onSendRequest(final long requestId, final String action, final TransportRequest request, final DiscoveryNode node) {
            super.onSendRequest(requestId, action, request, node);
            assertThat(action, is(REQUEST_PRE_VOTE_ACTION_NAME));
            assertThat(request, instanceOf(PreVoteRequest.class));
            assertThat(node, not(equalTo(localNode)));
            PreVoteRequest preVoteRequest = (PreVoteRequest) request;
            assertThat(preVoteRequest.getSourceNode(), equalTo(localNode));
            deterministicTaskQueue.scheduleNow(new Runnable() {

                @Override
                public void run() {
                    final PreVoteResponse response = responsesByNode.get(node);
                    if (response == null) {
                        handleRemoteError(requestId, new ConnectTransportException(node, "no response"));
                    } else {
                        handleResponse(requestId, response);
                    }
                }

                @Override
                public String toString() {
                    return "response to " + request + " from " + node;
                }
            });
        }
    };
    lastAcceptedTerm = randomNonNegativeLong();
    currentTerm = randomLongBetween(lastAcceptedTerm, Long.MAX_VALUE);
    lastAcceptedVersion = randomNonNegativeLong();
    localNode = new DiscoveryNode("local-node", buildNewFakeTransportAddress(), Version.CURRENT);
    responsesByNode.put(localNode, new PreVoteResponse(currentTerm, lastAcceptedTerm, lastAcceptedVersion));
    transportService = mockTransport.createTransportService(settings, deterministicTaskQueue.getThreadPool(), boundTransportAddress -> localNode, null);
    transportService.start();
    transportService.acceptIncomingRequests();
    preVoteCollector = new PreVoteCollector(transportService, () -> {
        assert electionOccurred == false;
        electionOccurred = true;
    }, l -> {
    });
    // TODO need tests that check that the max term seen is updated
    preVoteCollector.update(getLocalPreVoteResponse(), null);
}