Examples with ReceiveTimeoutTransportException org.elasticsearch.transport.ReceiveTimeoutTransportException used on opensource projects

Example 11 with ReceiveTimeoutTransportException

use of org.elasticsearch.transport.ReceiveTimeoutTransportException in project nifi by apache.

the class TestFetchElasticsearch method testFetchElasticsearchOnTriggerWithExceptions.

@Test
public void testFetchElasticsearchOnTriggerWithExceptions() throws IOException {
    FetchElasticsearchTestProcessor processor = new FetchElasticsearchTestProcessor(true);
    runner = TestRunners.newTestRunner(processor);
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.CLUSTER_NAME, "elasticsearch");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.HOSTS, "127.0.0.1:9300");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.PING_TIMEOUT, "5s");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.SAMPLER_INTERVAL, "5s");
    runner.setProperty(FetchElasticsearch.INDEX, "doc");
    runner.setProperty(FetchElasticsearch.TYPE, "status");
    runner.setValidateExpressionUsage(true);
    runner.setProperty(FetchElasticsearch.DOC_ID, "${doc_id}");
    // No Node Available exception
    processor.setExceptionToThrow(new NoNodeAvailableException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652140");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();
    // Elasticsearch Timeout exception
    processor.setExceptionToThrow(new ElasticsearchTimeoutException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652141");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();
    // Receive Timeout Transport exception
    processor.setExceptionToThrow(new ReceiveTimeoutTransportException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652141");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();
    // Node Closed exception
    processor.setExceptionToThrow(new NodeClosedException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652141");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();
    // Elasticsearch Parse exception
    processor.setExceptionToThrow(new ElasticsearchParseException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652141");
        }
    });
    runner.run(1, true, true);
    // This test generates an exception on execute(),routes to failure
    runner.assertTransferCount(FetchElasticsearch.REL_FAILURE, 1);
}

Example 12 with ReceiveTimeoutTransportException

use of org.elasticsearch.transport.ReceiveTimeoutTransportException in project nifi by apache.

the class TestPutElasticsearch method testPutElasticsearchOnTriggerWithExceptions.

@Test
public void testPutElasticsearchOnTriggerWithExceptions() throws IOException {
    PutElasticsearchTestProcessor processor = new PutElasticsearchTestProcessor(false);
    runner = TestRunners.newTestRunner(processor);
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.CLUSTER_NAME, "elasticsearch");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.HOSTS, "127.0.0.1:9300");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.PING_TIMEOUT, "5s");
    runner.setProperty(AbstractElasticsearchTransportClientProcessor.SAMPLER_INTERVAL, "5s");
    runner.setProperty(PutElasticsearch.INDEX, "doc");
    runner.setProperty(PutElasticsearch.TYPE, "status");
    runner.setValidateExpressionUsage(true);
    runner.setProperty(PutElasticsearch.ID_ATTRIBUTE, "doc_id");
    // No Node Available exception
    processor.setExceptionToThrow(new NoNodeAvailableException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652140");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();
    // Elasticsearch Timeout exception
    processor.setExceptionToThrow(new ElasticsearchTimeoutException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652141");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();
    // Receive Timeout Transport exception
    processor.setExceptionToThrow(new ReceiveTimeoutTransportException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652142");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();
    // Node Closed exception
    processor.setExceptionToThrow(new NodeClosedException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652143");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch.REL_RETRY, 1);
    runner.clearTransferState();
    // Elasticsearch Parse exception
    processor.setExceptionToThrow(new ElasticsearchParseException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652144");
        }
    });
    runner.run(1, true, true);
    // This test generates an exception on execute(),routes to failure
    runner.assertTransferCount(PutElasticsearch.REL_FAILURE, 1);
}

Example 13 with ReceiveTimeoutTransportException

use of org.elasticsearch.transport.ReceiveTimeoutTransportException in project crate by crate.

the class InternalClusterInfoService method refresh.

/**
 * Refreshes the ClusterInfo in a blocking fashion
 */
public final ClusterInfo refresh() {
    if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("Performing ClusterInfoUpdateJob");
    }
    final CountDownLatch nodeLatch = updateNodeStats(new ActionListener<>() {

        @Override
        public void onResponse(NodesStatsResponse nodesStatsResponse) {
            ImmutableOpenMap.Builder<String, DiskUsage> leastAvailableUsagesBuilder = ImmutableOpenMap.builder();
            ImmutableOpenMap.Builder<String, DiskUsage> mostAvailableUsagesBuilder = ImmutableOpenMap.builder();
            fillDiskUsagePerNode(LOGGER, nodesStatsResponse.getNodes(), leastAvailableUsagesBuilder, mostAvailableUsagesBuilder);
            leastAvailableSpaceUsages = leastAvailableUsagesBuilder.build();
            mostAvailableSpaceUsages = mostAvailableUsagesBuilder.build();
        }

        @Override
        public void onFailure(Exception e) {
            if (e instanceof ReceiveTimeoutTransportException) {
                LOGGER.error("NodeStatsAction timed out for ClusterInfoUpdateJob", e);
            } else {
                if (e instanceof ClusterBlockException) {
                    if (LOGGER.isTraceEnabled()) {
                        LOGGER.trace("Failed to execute NodeStatsAction for ClusterInfoUpdateJob", e);
                    }
                } else {
                    LOGGER.warn("Failed to execute NodeStatsAction for ClusterInfoUpdateJob", e);
                }
                // we empty the usages list, to be safe - we don't know what's going on.
                leastAvailableSpaceUsages = ImmutableOpenMap.of();
                mostAvailableSpaceUsages = ImmutableOpenMap.of();
            }
        }
    });
    final CountDownLatch indicesLatch = updateIndicesStats(new ActionListener<>() {

        @Override
        public void onResponse(IndicesStatsResponse indicesStatsResponse) {
            ShardStats[] stats = indicesStatsResponse.getShards();
            ImmutableOpenMap.Builder<String, Long> newShardSizes = ImmutableOpenMap.builder();
            ImmutableOpenMap.Builder<ShardRouting, String> newShardRoutingToDataPath = ImmutableOpenMap.builder();
            buildShardLevelInfo(LOGGER, stats, newShardSizes, newShardRoutingToDataPath);
            shardSizes = newShardSizes.build();
            shardRoutingToDataPath = newShardRoutingToDataPath.build();
        }

        @Override
        public void onFailure(Exception e) {
            if (e instanceof ReceiveTimeoutTransportException) {
                LOGGER.error("IndicesStatsAction timed out for ClusterInfoUpdateJob", e);
            } else {
                if (e instanceof ClusterBlockException) {
                    if (LOGGER.isTraceEnabled()) {
                        LOGGER.trace("Failed to execute IndicesStatsAction for ClusterInfoUpdateJob", e);
                    }
                } else {
                    LOGGER.warn("Failed to execute IndicesStatsAction for ClusterInfoUpdateJob", e);
                }
                // we empty the usages list, to be safe - we don't know what's going on.
                shardSizes = ImmutableOpenMap.of();
                shardRoutingToDataPath = ImmutableOpenMap.of();
            }
        }
    });
    try {
        nodeLatch.await(fetchTimeout.getMillis(), TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        // restore interrupt status
        Thread.currentThread().interrupt();
        LOGGER.warn("Failed to update node information for ClusterInfoUpdateJob within {} timeout", fetchTimeout);
    }
    try {
        indicesLatch.await(fetchTimeout.getMillis(), TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        // restore interrupt status
        Thread.currentThread().interrupt();
        LOGGER.warn("Failed to update shard information for ClusterInfoUpdateJob within {} timeout", fetchTimeout);
    }
    ClusterInfo clusterInfo = getClusterInfo();
    boolean anyListeners = false;
    for (final Consumer<ClusterInfo> listener : listeners) {
        anyListeners = true;
        try {
            LOGGER.trace("notifying [{}] of new cluster info", listener);
            listener.accept(clusterInfo);
        } catch (Exception e) {
            LOGGER.info(new ParameterizedMessage("failed to notify [{}] of new cluster info", listener), e);
        }
    }
    assert anyListeners : "expected to notify at least one listener";
    return clusterInfo;
}

Example 14 with ReceiveTimeoutTransportException

use of org.elasticsearch.transport.ReceiveTimeoutTransportException in project crate by crate.

the class AsyncShardFetch method processAsyncFetch.

/**
 * Called by the response handler of the async action to fetch data. Verifies that its still working
 * on the same cache generation, otherwise the results are discarded. It then goes and fills the relevant data for
 * the shard (response + failures), issuing a reroute at the end of it to make sure there will be another round
 * of allocations taking this new data into account.
 */
protected synchronized void processAsyncFetch(List<T> responses, List<FailedNodeException> failures, long fetchingRound) {
    if (closed) {
        // we are closed, no need to process this async fetch at all
        logger.trace("{} ignoring fetched [{}] results, already closed", shardId, type);
        return;
    }
    logger.trace("{} processing fetched [{}] results", shardId, type);
    if (responses != null) {
        for (T response : responses) {
            NodeEntry<T> nodeEntry = cache.get(response.getNode().getId());
            if (nodeEntry != null) {
                if (nodeEntry.getFetchingRound() != fetchingRound) {
                    assert nodeEntry.getFetchingRound() > fetchingRound : "node entries only replaced by newer rounds";
                    logger.trace("{} received response for [{}] from node {} for an older fetching round (expected: {} but was: {})", shardId, nodeEntry.getNodeId(), type, nodeEntry.getFetchingRound(), fetchingRound);
                } else if (nodeEntry.isFailed()) {
                    logger.trace("{} node {} has failed for [{}] (failure [{}])", shardId, nodeEntry.getNodeId(), type, nodeEntry.getFailure());
                } else {
                    // if the entry is there, for the right fetching round and not marked as failed already, process it
                    logger.trace("{} marking {} as done for [{}], result is [{}]", shardId, nodeEntry.getNodeId(), type, response);
                    nodeEntry.doneFetching(response);
                }
            }
        }
    }
    if (failures != null) {
        for (FailedNodeException failure : failures) {
            logger.trace("{} processing failure {} for [{}]", shardId, failure, type);
            NodeEntry<T> nodeEntry = cache.get(failure.nodeId());
            if (nodeEntry != null) {
                if (nodeEntry.getFetchingRound() != fetchingRound) {
                    assert nodeEntry.getFetchingRound() > fetchingRound : "node entries only replaced by newer rounds";
                    logger.trace("{} received failure for [{}] from node {} for an older fetching round (expected: {} but was: {})", shardId, nodeEntry.getNodeId(), type, nodeEntry.getFetchingRound(), fetchingRound);
                } else if (nodeEntry.isFailed() == false) {
                    // if the entry is there, for the right fetching round and not marked as failed already, process it
                    Throwable unwrappedCause = SQLExceptions.unwrap(failure.getCause());
                    // if the request got rejected or timed out, we need to try it again next time...
                    if (unwrappedCause instanceof EsRejectedExecutionException || unwrappedCause instanceof ReceiveTimeoutTransportException || unwrappedCause instanceof ElasticsearchTimeoutException) {
                        nodeEntry.restartFetching();
                    } else {
                        logger.warn(() -> new ParameterizedMessage("{}: failed to list shard for {} on node [{}]", shardId, type, failure.nodeId()), failure);
                        nodeEntry.doneFetching(failure.getCause());
                    }
                }
            }
        }
    }
    reroute(shardId, "post_response");
}