Examples with ElasticsearchTimeoutException org.elasticsearch.ElasticsearchTimeoutException used on opensource projects

Example 11 with ElasticsearchTimeoutException

use of org.elasticsearch.ElasticsearchTimeoutException in project elasticsearch by elastic.

the class UpdateIT method testStressUpdateDeleteConcurrency.

public void testStressUpdateDeleteConcurrency() throws Exception {
    //We create an index with merging disabled so that deletes don't get merged away
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)));
    ensureGreen();
    final int numberOfThreads = scaledRandomIntBetween(3, 5);
    final int numberOfIdsPerThread = scaledRandomIntBetween(3, 10);
    final int numberOfUpdatesPerId = scaledRandomIntBetween(10, 100);
    final int retryOnConflict = randomIntBetween(0, 1);
    final CountDownLatch latch = new CountDownLatch(numberOfThreads);
    final CountDownLatch startLatch = new CountDownLatch(1);
    final List<Throwable> failures = new CopyOnWriteArrayList<>();
    final class UpdateThread extends Thread {

        final Map<Integer, Integer> failedMap = new HashMap<>();

        final int numberOfIds;

        final int updatesPerId;

        final int maxUpdateRequests = numberOfIdsPerThread * numberOfUpdatesPerId;

        final int maxDeleteRequests = numberOfIdsPerThread * numberOfUpdatesPerId;

        private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);

        private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);

        UpdateThread(int numberOfIds, int updatesPerId) {
            this.numberOfIds = numberOfIds;
            this.updatesPerId = updatesPerId;
        }

        final class UpdateListener implements ActionListener<UpdateResponse> {

            int id;

            UpdateListener(int id) {
                this.id = id;
            }

            @Override
            public void onResponse(UpdateResponse updateResponse) {
                updateRequestsOutstanding.release(1);
            }

            @Override
            public void onFailure(Exception e) {
                synchronized (failedMap) {
                    incrementMapValue(id, failedMap);
                }
                updateRequestsOutstanding.release(1);
            }
        }

        final class DeleteListener implements ActionListener<DeleteResponse> {

            int id;

            DeleteListener(int id) {
                this.id = id;
            }

            @Override
            public void onResponse(DeleteResponse deleteResponse) {
                deleteRequestsOutstanding.release(1);
            }

            @Override
            public void onFailure(Exception e) {
                synchronized (failedMap) {
                    incrementMapValue(id, failedMap);
                }
                deleteRequestsOutstanding.release(1);
            }
        }

        @Override
        public void run() {
            try {
                startLatch.await();
                boolean hasWaitedForNoNode = false;
                for (int j = 0; j < numberOfIds; j++) {
                    for (int k = 0; k < numberOfUpdatesPerId; ++k) {
                        updateRequestsOutstanding.acquire();
                        try {
                            UpdateRequest ur = client().prepareUpdate("test", "type1", Integer.toString(j)).setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).setRetryOnConflict(retryOnConflict).setUpsert(jsonBuilder().startObject().field("field", 1).endObject()).request();
                            client().update(ur, new UpdateListener(j));
                        } catch (NoNodeAvailableException nne) {
                            updateRequestsOutstanding.release();
                            synchronized (failedMap) {
                                incrementMapValue(j, failedMap);
                            }
                            if (hasWaitedForNoNode) {
                                throw nne;
                            }
                            logger.warn("Got NoNodeException waiting for 1 second for things to recover.");
                            hasWaitedForNoNode = true;
                            Thread.sleep(1000);
                        }
                        try {
                            deleteRequestsOutstanding.acquire();
                            DeleteRequest dr = client().prepareDelete("test", "type1", Integer.toString(j)).request();
                            client().delete(dr, new DeleteListener(j));
                        } catch (NoNodeAvailableException nne) {
                            deleteRequestsOutstanding.release();
                            synchronized (failedMap) {
                                incrementMapValue(j, failedMap);
                            }
                            if (hasWaitedForNoNode) {
                                throw nne;
                            }
                            logger.warn("Got NoNodeException waiting for 1 second for things to recover.");
                            hasWaitedForNoNode = true;
                            //Wait for no-node to clear
                            Thread.sleep(1000);
                        }
                    }
                }
            } catch (Exception e) {
                logger.error("Something went wrong", e);
                failures.add(e);
            } finally {
                try {
                    waitForOutstandingRequests(TimeValue.timeValueSeconds(60), updateRequestsOutstanding, maxUpdateRequests, "Update");
                    waitForOutstandingRequests(TimeValue.timeValueSeconds(60), deleteRequestsOutstanding, maxDeleteRequests, "Delete");
                } catch (ElasticsearchTimeoutException ete) {
                    failures.add(ete);
                }
                latch.countDown();
            }
        }

        private void incrementMapValue(int j, Map<Integer, Integer> map) {
            if (!map.containsKey(j)) {
                map.put(j, 0);
            }
            map.put(j, map.get(j) + 1);
        }

        private void waitForOutstandingRequests(TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
            long start = System.currentTimeMillis();
            do {
                long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
                logger.info("[{}] going to try and acquire [{}] in [{}]ms [{}] available to acquire right now", name, maxRequests, msRemaining, requestsOutstanding.availablePermits());
                try {
                    requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
                    return;
                } catch (InterruptedException ie) {
                //Just keep swimming
                }
            } while ((System.currentTimeMillis() - start) < timeOut.getMillis());
            throw new ElasticsearchTimeoutException("Requests were still outstanding after the timeout [" + timeOut + "] for type [" + name + "]");
        }
    }
    final List<UpdateThread> threads = new ArrayList<>();
    for (int i = 0; i < numberOfThreads; i++) {
        UpdateThread ut = new UpdateThread(numberOfIdsPerThread, numberOfUpdatesPerId);
        ut.start();
        threads.add(ut);
    }
    startLatch.countDown();
    latch.await();
    for (UpdateThread ut : threads) {
        //Threads should have finished because of the latch.await
        ut.join();
    }
    //aquiring the request outstanding semaphores.
    for (Throwable throwable : failures) {
        logger.info("Captured failure on concurrent update:", throwable);
    }
    assertThat(failures.size(), equalTo(0));
    //All the previous operations should be complete or failed at this point
    for (int i = 0; i < numberOfIdsPerThread; ++i) {
        UpdateResponse ur = client().prepareUpdate("test", "type1", Integer.toString(i)).setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).setRetryOnConflict(Integer.MAX_VALUE).setUpsert(jsonBuilder().startObject().field("field", 1).endObject()).execute().actionGet();
    }
    refresh();
    for (int i = 0; i < numberOfIdsPerThread; ++i) {
        int totalFailures = 0;
        GetResponse response = client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet();
        if (response.isExists()) {
            assertThat(response.getId(), equalTo(Integer.toString(i)));
            int expectedVersion = (numberOfThreads * numberOfUpdatesPerId * 2) + 1;
            for (UpdateThread ut : threads) {
                if (ut.failedMap.containsKey(i)) {
                    totalFailures += ut.failedMap.get(i);
                }
            }
            expectedVersion -= totalFailures;
            logger.error("Actual version [{}] Expected version [{}] Total failures [{}]", response.getVersion(), expectedVersion, totalFailures);
            assertThat(response.getVersion(), equalTo((long) expectedVersion));
            assertThat(response.getVersion() + totalFailures, equalTo((long) ((numberOfUpdatesPerId * numberOfThreads * 2) + 1)));
        }
    }
}

Example 12 with ElasticsearchTimeoutException

use of org.elasticsearch.ElasticsearchTimeoutException in project elasticsearch by elastic.

the class AsyncShardFetch method processAsyncFetch.

/**
     * Called by the response handler of the async action to fetch data. Verifies that its still working
     * on the same cache generation, otherwise the results are discarded. It then goes and fills the relevant data for
     * the shard (response + failures), issuing a reroute at the end of it to make sure there will be another round
     * of allocations taking this new data into account.
     */
protected synchronized void processAsyncFetch(ShardId shardId, List<T> responses, List<FailedNodeException> failures) {
    if (closed) {
        // we are closed, no need to process this async fetch at all
        logger.trace("{} ignoring fetched [{}] results, already closed", shardId, type);
        return;
    }
    logger.trace("{} processing fetched [{}] results", shardId, type);
    if (responses != null) {
        for (T response : responses) {
            NodeEntry<T> nodeEntry = cache.get(response.getNode().getId());
            // if the entry is there, and not marked as failed already, process it
            if (nodeEntry == null) {
                continue;
            }
            if (nodeEntry.isFailed()) {
                logger.trace("{} node {} has failed for [{}] (failure [{}])", shardId, nodeEntry.getNodeId(), type, nodeEntry.getFailure());
            } else {
                logger.trace("{} marking {} as done for [{}], result is [{}]", shardId, nodeEntry.getNodeId(), type, response);
                nodeEntry.doneFetching(response);
            }
        }
    }
    if (failures != null) {
        for (FailedNodeException failure : failures) {
            logger.trace("{} processing failure {} for [{}]", shardId, failure, type);
            NodeEntry<T> nodeEntry = cache.get(failure.nodeId());
            // if the entry is there, and not marked as failed already, process it
            if (nodeEntry != null && nodeEntry.isFailed() == false) {
                Throwable unwrappedCause = ExceptionsHelper.unwrapCause(failure.getCause());
                // if the request got rejected or timed out, we need to try it again next time...
                if (unwrappedCause instanceof EsRejectedExecutionException || unwrappedCause instanceof ReceiveTimeoutTransportException || unwrappedCause instanceof ElasticsearchTimeoutException) {
                    nodeEntry.restartFetching();
                } else {
                    logger.warn((Supplier<?>) () -> new ParameterizedMessage("{}: failed to list shard for {} on node [{}]", shardId, type, failure.nodeId()), failure);
                    nodeEntry.doneFetching(failure.getCause());
                }
            }
        }
    }
    reroute(shardId, "post_response");
}

Example 13 with ElasticsearchTimeoutException

use of org.elasticsearch.ElasticsearchTimeoutException in project graylog2-server by Graylog2.

the class IndexerSetupService method startUp.

@Override
protected void startUp() throws Exception {
    Tools.silenceUncaughtExceptionsInThisThread();
    LOG.debug("Starting indexer");
    node.start();
    final Client client = node.client();
    try {
        /* try to determine the cluster health. if this times out we could not connect and try to determine if there's
                   anything listening at all. if that happens this usually has these reasons:
                    1. cluster.name is different
                    2. network.publish_host is not reachable
                    3. wrong address configured
                    4. multicast in use but broken in this environment
                   we handle a red cluster state differently because if we can get that result it means the cluster itself
                   is reachable, which is a completely different problem from not being able to join at all.
                 */
        final ClusterHealthRequest atLeastRed = client.admin().cluster().prepareHealth().setWaitForStatus(ClusterHealthStatus.RED).request();
        final ClusterHealthResponse health = client.admin().cluster().health(atLeastRed).actionGet(configuration.getClusterDiscoveryTimeout(), MILLISECONDS);
        // we don't get here if we couldn't join the cluster. just check for red cluster state
        if (ClusterHealthStatus.RED.equals(health.getStatus())) {
            final Notification notification = notificationService.buildNow().addSeverity(Notification.Severity.URGENT).addType(Notification.Type.ES_CLUSTER_RED);
            notificationService.publishIfFirst(notification);
            LOG.warn("The Elasticsearch cluster state is RED which means shards are unassigned.");
            LOG.info("This usually indicates a crashed and corrupt cluster and needs to be investigated. Graylog will write into the local disk journal.");
            LOG.info("See {} for details.", DocsHelper.PAGE_ES_CONFIGURATION);
        }
        if (ClusterHealthStatus.GREEN.equals(health.getStatus())) {
            notificationService.fixed(Notification.Type.ES_CLUSTER_RED);
        }
        notificationService.fixed(Notification.Type.ES_UNAVAILABLE);
    } catch (ElasticsearchTimeoutException e) {
        final String hosts = node.settings().get("discovery.zen.ping.unicast.hosts");
        if (!isNullOrEmpty(hosts)) {
            final Iterable<String> hostList = Splitter.on(',').omitEmptyStrings().trimResults().split(hosts);
            for (String host : hostList) {
                final URI esUri = URI.create("http://" + HostAndPort.fromString(host).getHostText() + ":9200/");
                LOG.info("Checking Elasticsearch HTTP API at {}", esUri);
                // Try the HTTP API endpoint
                final Request request = new Request.Builder().get().url(esUri.resolve("/_nodes").toString()).build();
                try (final Response response = httpClient.newCall(request).execute()) {
                    if (response.isSuccessful()) {
                        final JsonNode resultTree = objectMapper.readTree(response.body().byteStream());
                        final JsonNode nodesList = resultTree.get("nodes");
                        if (!configuration.isDisableVersionCheck()) {
                            final Iterator<String> nodes = nodesList.fieldNames();
                            while (nodes.hasNext()) {
                                final String id = nodes.next();
                                final Version clusterVersion = Version.fromString(nodesList.get(id).get("version").textValue());
                                checkClusterVersion(clusterVersion);
                            }
                        }
                        final String clusterName = resultTree.get("cluster_name").textValue();
                        checkClusterName(clusterName);
                    } else {
                        LOG.error("Could not connect to Elasticsearch at " + esUri + ". Is it running?");
                    }
                } catch (IOException ioException) {
                    LOG.error("Could not connect to Elasticsearch: {}", ioException.getMessage());
                }
            }
        }
        final Notification notification = notificationService.buildNow().addSeverity(Notification.Severity.URGENT).addType(Notification.Type.ES_UNAVAILABLE);
        notificationService.publishIfFirst(notification);
        LOG.warn("Could not connect to Elasticsearch");
        LOG.info("If you're using multicast, check that it is working in your network and that Elasticsearch is accessible. Also check that the cluster name setting is correct.");
        LOG.info("See {} for details.", DocsHelper.PAGE_ES_CONFIGURATION);
    }
}

Example 14 with ElasticsearchTimeoutException

use of org.elasticsearch.ElasticsearchTimeoutException in project nifi by apache.

the class DeleteElasticsearch5 method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    synchronized (esClient) {
        if (esClient.get() == null) {
            setup(context);
        }
    }
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final String index = context.getProperty(INDEX).evaluateAttributeExpressions(flowFile).getValue();
    final String documentId = context.getProperty(DOCUMENT_ID).evaluateAttributeExpressions(flowFile).getValue();
    final String documentType = context.getProperty(TYPE).evaluateAttributeExpressions(flowFile).getValue();
    final ComponentLog logger = getLogger();
    if (StringUtils.isBlank(index)) {
        logger.debug("Index is required but was empty {}", new Object[] { index });
        flowFile = session.putAttribute(flowFile, ES_ERROR_MESSAGE, "Index is required but was empty");
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    if (StringUtils.isBlank(documentType)) {
        logger.debug("Document type is required but was empty {}", new Object[] { documentType });
        flowFile = session.putAttribute(flowFile, ES_ERROR_MESSAGE, "Document type is required but was empty");
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    if (StringUtils.isBlank(documentId)) {
        logger.debug("Document id is required but was empty {}", new Object[] { documentId });
        flowFile = session.putAttribute(flowFile, ES_ERROR_MESSAGE, "Document id is required but was empty");
        session.transfer(flowFile, REL_FAILURE);
        return;
    }
    flowFile = session.putAllAttributes(flowFile, new HashMap<String, String>() {

        {
            put(ES_FILENAME, documentId);
            put(ES_INDEX, index);
            put(ES_TYPE, documentType);
        }
    });
    try {
        logger.debug("Deleting document {}/{}/{} from Elasticsearch", new Object[] { index, documentType, documentId });
        DeleteRequestBuilder requestBuilder = prepareDeleteRequest(index, documentId, documentType);
        final DeleteResponse response = doDelete(requestBuilder);
        if (response.status() != RestStatus.OK) {
            logger.warn("Failed to delete document {}/{}/{} from Elasticsearch: Status {}", new Object[] { index, documentType, documentId, response.status() });
            flowFile = session.putAttribute(flowFile, ES_ERROR_MESSAGE, UNABLE_TO_DELETE_DOCUMENT_MESSAGE);
            flowFile = session.putAttribute(flowFile, ES_REST_STATUS, response.status().toString());
            context.yield();
            if (response.status() == RestStatus.NOT_FOUND) {
                session.transfer(flowFile, REL_NOT_FOUND);
            } else {
                session.transfer(flowFile, REL_FAILURE);
            }
        } else {
            logger.debug("Elasticsearch document " + documentId + " deleted");
            session.transfer(flowFile, REL_SUCCESS);
        }
    } catch (ElasticsearchTimeoutException | ReceiveTimeoutTransportException exception) {
        logger.error("Failed to delete document {} from Elasticsearch due to {}", new Object[] { documentId, exception.getLocalizedMessage() }, exception);
        flowFile = session.putAttribute(flowFile, ES_ERROR_MESSAGE, exception.getLocalizedMessage());
        session.transfer(flowFile, REL_RETRY);
        context.yield();
    } catch (Exception e) {
        logger.error("Failed to delete document {} from Elasticsearch due to {}", new Object[] { documentId, e.getLocalizedMessage() }, e);
        flowFile = session.putAttribute(flowFile, ES_ERROR_MESSAGE, e.getLocalizedMessage());
        session.transfer(flowFile, REL_FAILURE);
        context.yield();
    }
}

Example 15 with ElasticsearchTimeoutException

use of org.elasticsearch.ElasticsearchTimeoutException in project nifi by apache.

the class TestPutElasticsearch5 method testPutElasticsearch5OnTriggerWithExceptions.

@Test
public void testPutElasticsearch5OnTriggerWithExceptions() throws IOException {
    PutElasticsearch5TestProcessor processor = new PutElasticsearch5TestProcessor(false);
    runner = TestRunners.newTestRunner(processor);
    runner.setProperty(AbstractElasticsearch5TransportClientProcessor.CLUSTER_NAME, "elasticsearch");
    runner.setProperty(AbstractElasticsearch5TransportClientProcessor.HOSTS, "127.0.0.1:9300");
    runner.setProperty(AbstractElasticsearch5TransportClientProcessor.PING_TIMEOUT, "5s");
    runner.setProperty(AbstractElasticsearch5TransportClientProcessor.SAMPLER_INTERVAL, "5s");
    runner.setProperty(PutElasticsearch5.INDEX, "doc");
    runner.setProperty(PutElasticsearch5.TYPE, "status");
    runner.setValidateExpressionUsage(true);
    runner.setProperty(PutElasticsearch5.ID_ATTRIBUTE, "doc_id");
    // No Node Available exception
    processor.setExceptionToThrow(new NoNodeAvailableException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652140");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch5.REL_RETRY, 1);
    runner.clearTransferState();
    // Elasticsearch5 Timeout exception
    processor.setExceptionToThrow(new ElasticsearchTimeoutException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652141");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch5.REL_RETRY, 1);
    runner.clearTransferState();
    // Receive Timeout Transport exception
    processor.setExceptionToThrow(new ReceiveTimeoutTransportException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652142");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch5.REL_RETRY, 1);
    runner.clearTransferState();
    // Node Closed exception
    processor.setExceptionToThrow(new NodeClosedException(mock(StreamInput.class)));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652143");
        }
    });
    runner.run(1, true, true);
    runner.assertAllFlowFilesTransferred(FetchElasticsearch5.REL_RETRY, 1);
    runner.clearTransferState();
    // Elasticsearch5 Parse exception
    processor.setExceptionToThrow(new ElasticsearchParseException("test"));
    runner.enqueue(docExample, new HashMap<String, String>() {

        {
            put("doc_id", "28039652144");
        }
    });
    runner.run(1, true, true);
    // This test generates an exception on execute(),routes to failure
    runner.assertTransferCount(PutElasticsearch5.REL_FAILURE, 1);
}