Examples with NodesStatsResponse org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse used on opensource projects

Example 11 with NodesStatsResponse

use of org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse in project elasticsearch by elastic.

the class ExceptionRetryIT method testRetryDueToExceptionOnNetworkLayer.

/**
     * Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally
     * failing. If auto generated ids are used this must not lead to duplicate ids
     * see https://github.com/elastic/elasticsearch/issues/8788
     */
public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {
    final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
    int numDocs = scaledRandomIntBetween(100, 1000);
    Client client = internalCluster().coordOnlyNodeClient();
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    NodeStats unluckyNode = randomFrom(nodeStats.getNodes().stream().filter((s) -> s.getNode().isDataNode()).collect(Collectors.toList()));
    assertAcked(client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 1).put("index.number_of_shards", 5)));
    ensureGreen("index");
    logger.info("unlucky node: {}", unluckyNode.getNode());
    //create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.
    for (NodeStats dataNode : nodeStats.getNodes()) {
        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
        mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {

            @Override
            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                super.sendRequest(connection, requestId, action, request, options);
                if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
                    logger.debug("Throw ConnectTransportException");
                    throw new ConnectTransportException(connection.getNode(), action);
                }
            }
        });
    }
    BulkRequestBuilder bulkBuilder = client.prepareBulk();
    for (int i = 0; i < numDocs; i++) {
        XContentBuilder doc = null;
        doc = jsonBuilder().startObject().field("foo", "bar").endObject();
        bulkBuilder.add(client.prepareIndex("index", "type").setSource(doc));
    }
    BulkResponse response = bulkBuilder.get();
    if (response.hasFailures()) {
        for (BulkItemResponse singleIndexRespons : response.getItems()) {
            if (singleIndexRespons.isFailed()) {
                fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage());
            }
        }
    }
    refresh();
    SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
    Set<String> uniqueIds = new HashSet();
    long dupCounter = 0;
    boolean found_duplicate_already = false;
    for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
        if (!uniqueIds.add(searchResponse.getHits().getHits()[i].getId())) {
            if (!found_duplicate_already) {
                SearchResponse dupIdResponse = client().prepareSearch("index").setQuery(termQuery("_id", searchResponse.getHits().getHits()[i].getId())).setExplain(true).get();
                assertThat(dupIdResponse.getHits().getTotalHits(), greaterThan(1L));
                logger.info("found a duplicate id:");
                for (SearchHit hit : dupIdResponse.getHits()) {
                    logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
                }
                logger.info("will not print anymore in case more duplicates are found.");
                found_duplicate_already = true;
            }
            dupCounter++;
        }
    }
    assertSearchResponse(searchResponse);
    assertThat(dupCounter, equalTo(0L));
    assertHitCount(searchResponse, numDocs);
    IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setSegments(true).get();
    IndexStats indexStats = index.getIndex("index");
    long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
    for (IndexShardStats indexShardStats : indexStats) {
        for (ShardStats shardStats : indexShardStats) {
            SegmentsStats segments = shardStats.getStats().getSegments();
            maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp());
        }
    }
    assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get());
    assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1);
}

Example 12 with NodesStatsResponse

use of org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse in project elasticsearch by elastic.

the class CorruptedFileIT method listShardFiles.

public List<Path> listShardFiles(ShardRouting routing) throws IOException {
    NodesStatsResponse nodeStatses = client().admin().cluster().prepareNodesStats(routing.currentNodeId()).setFs(true).get();
    ClusterState state = client().admin().cluster().prepareState().get().getState();
    final Index test = state.metaData().index("test").getIndex();
    assertThat(routing.toString(), nodeStatses.getNodes().size(), equalTo(1));
    List<Path> files = new ArrayList<>();
    for (FsInfo.Path info : nodeStatses.getNodes().get(0).getFs()) {
        String path = info.getPath();
        Path file = PathUtils.get(path).resolve("indices/" + test.getUUID() + "/" + Integer.toString(routing.getId()) + "/index");
        if (Files.exists(file)) {
            // multi data path might only have one path in use
            try (DirectoryStream<Path> stream = Files.newDirectoryStream(file)) {
                for (Path item : stream) {
                    files.add(item);
                }
            }
        }
    }
    return files;
}

Example 13 with NodesStatsResponse

use of org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse in project elasticsearch by elastic.

the class SuggestStatsIT method testSimpleStats.

public void testSimpleStats() throws Exception {
    // clear all stats first
    client().admin().indices().prepareStats().clear().execute().actionGet();
    final int numNodes = cluster().numDataNodes();
    assertThat(numNodes, greaterThanOrEqualTo(2));
    // we make sure each node gets at least a single shard...
    final int shardsIdx1 = randomIntBetween(1, 10);
    final int shardsIdx2 = Math.max(numNodes - shardsIdx1, randomIntBetween(1, 10));
    final int totalShards = shardsIdx1 + shardsIdx2;
    assertThat(numNodes, lessThanOrEqualTo(totalShards));
    assertAcked(prepareCreate("test1").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, shardsIdx1).put(SETTING_NUMBER_OF_REPLICAS, 0)).addMapping("type", "f", "type=text"));
    assertAcked(prepareCreate("test2").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, shardsIdx2).put(SETTING_NUMBER_OF_REPLICAS, 0)).addMapping("type", "f", "type=text"));
    assertThat(shardsIdx1 + shardsIdx2, equalTo(numAssignedShards("test1", "test2")));
    assertThat(numAssignedShards("test1", "test2"), greaterThanOrEqualTo(2));
    ensureGreen();
    for (int i = 0; i < randomIntBetween(20, 100); i++) {
        index("test" + ((i % 2) + 1), "type", "" + i, "f", "test" + i);
    }
    refresh();
    int suggestAllIdx = scaledRandomIntBetween(20, 50);
    int suggestIdx1 = scaledRandomIntBetween(20, 50);
    int suggestIdx2 = scaledRandomIntBetween(20, 50);
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < suggestAllIdx; i++) {
        SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch(), i).get();
        assertAllSuccessful(suggestResponse);
    }
    for (int i = 0; i < suggestIdx1; i++) {
        SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch("test1"), i).get();
        assertAllSuccessful(suggestResponse);
    }
    for (int i = 0; i < suggestIdx2; i++) {
        SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch("test2"), i).get();
        assertAllSuccessful(suggestResponse);
    }
    long endTime = System.currentTimeMillis();
    IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().execute().actionGet();
    final SearchStats.Stats suggest = indicesStats.getTotal().getSearch().getTotal();
    // check current
    assertThat(suggest.getSuggestCurrent(), equalTo(0L));
    // check suggest count
    assertThat(suggest.getSuggestCount(), equalTo((long) (suggestAllIdx * totalShards + suggestIdx1 * shardsIdx1 + suggestIdx2 * shardsIdx2)));
    assertThat(indicesStats.getIndices().get("test1").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx1) * shardsIdx1)));
    assertThat(indicesStats.getIndices().get("test2").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx2) * shardsIdx2)));
    logger.info("iter {}, iter1 {}, iter2 {}, {}", suggestAllIdx, suggestIdx1, suggestIdx2, endTime - startTime);
    // check suggest time
    assertThat(suggest.getSuggestTimeInMillis(), greaterThan(0L));
    // the upperbound is num shards * total time since we do searches in parallel
    assertThat(suggest.getSuggestTimeInMillis(), lessThanOrEqualTo(totalShards * (endTime - startTime)));
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet();
    Set<String> nodeIdsWithIndex = nodeIdsWithIndex("test1", "test2");
    int num = 0;
    for (NodeStats stat : nodeStats.getNodes()) {
        SearchStats.Stats suggestStats = stat.getIndices().getSearch().getTotal();
        logger.info("evaluating {}", stat.getNode());
        if (nodeIdsWithIndex.contains(stat.getNode().getId())) {
            assertThat(suggestStats.getSuggestCount(), greaterThan(0L));
            assertThat(suggestStats.getSuggestTimeInMillis(), greaterThan(0L));
            num++;
        } else {
            assertThat(suggestStats.getSuggestCount(), equalTo(0L));
            assertThat(suggestStats.getSuggestTimeInMillis(), equalTo(0L));
        }
    }
    assertThat(num, greaterThan(0));
}

Example 14 with NodesStatsResponse

use of org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse in project elasticsearch by elastic.

the class IndexStatsIT method testClearAllCaches.

public void testClearAllCaches() throws Exception {
    assertAcked(client().admin().indices().prepareCreate("test").setSettings(settingsBuilder().put("index.number_of_replicas", 0).put("index.number_of_shards", 2)).addMapping("type", "field", "type=text,fielddata=true").get());
    ensureGreen();
    client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
    client().prepareIndex("test", "type", "1").setSource("field", "value1").execute().actionGet();
    client().prepareIndex("test", "type", "2").setSource("field", "value2").execute().actionGet();
    client().admin().indices().prepareRefresh().execute().actionGet();
    NodesStatsResponse nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
    assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
    assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
    IndicesStatsResponse indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).setQueryCache(true).execute().actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));
    assertThat(indicesStats.getTotal().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
    // sort to load it to field data and filter to load filter cache
    client().prepareSearch().setPostFilter(QueryBuilders.termQuery("field", "value1")).addSort("field", SortOrder.ASC).execute().actionGet();
    client().prepareSearch().setPostFilter(QueryBuilders.termQuery("field", "value2")).addSort("field", SortOrder.ASC).execute().actionGet();
    nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
    assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
    assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), greaterThan(0L));
    indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).setQueryCache(true).execute().actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), greaterThan(0L));
    assertThat(indicesStats.getTotal().getQueryCache().getMemorySizeInBytes(), greaterThan(0L));
    client().admin().indices().prepareClearCache().execute().actionGet();
    // Make sure the filter cache entries have been removed...
    Thread.sleep(100);
    nodesStats = client().admin().cluster().prepareNodesStats("data:true").setIndices(true).execute().actionGet();
    assertThat(nodesStats.getNodes().get(0).getIndices().getFieldData().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
    assertThat(nodesStats.getNodes().get(0).getIndices().getQueryCache().getMemorySizeInBytes() + nodesStats.getNodes().get(1).getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
    indicesStats = client().admin().indices().prepareStats("test").clear().setFieldData(true).setQueryCache(true).execute().actionGet();
    assertThat(indicesStats.getTotal().getFieldData().getMemorySizeInBytes(), equalTo(0L));
    assertThat(indicesStats.getTotal().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
}

Example 15 with NodesStatsResponse

use of org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse in project sonarqube by SonarSource.

the class EsMonitor method nodeAttributes.

/**
   * map of {node name -> node attributes}
   */
private LinkedHashMap<String, LinkedHashMap<String, Object>> nodeAttributes() {
    LinkedHashMap<String, LinkedHashMap<String, Object>> nodes = new LinkedHashMap<>();
    NodesStatsResponse nodesStats = esClient.prepareNodesStats().all().get();
    for (Map.Entry<String, NodeStats> entry : nodesStats.getNodesMap().entrySet()) {
        LinkedHashMap<String, Object> nodeAttributes = new LinkedHashMap<>();
        NodeStats stats = entry.getValue();
        DiscoveryNode node = stats.getNode();
        nodes.put(node.getName(), nodeAttributes);
        nodeAttributes.put("Address", node.getAddress().toString());
        nodeAttributes.put("Type", node.isMasterNode() ? "Master" : "Slave");
        nodeAttributes.put("Disk Available", byteCountToDisplaySize(stats.getFs().getTotal().getAvailable().bytes()));
        nodeAttributes.put("Store Size", byteCountToDisplaySize(stats.getIndices().getStore().getSizeInBytes()));
        nodeAttributes.put("Open Files", stats.getProcess().getOpenFileDescriptors());
        nodeAttributes.put("JVM Heap Usage", formatPercent(stats.getJvm().getMem().getHeapUsedPercent()));
        nodeAttributes.put("JVM Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getHeapUsed().bytes()));
        nodeAttributes.put("JVM Heap Max", byteCountToDisplaySize(stats.getJvm().getMem().getHeapMax().bytes()));
        nodeAttributes.put("JVM Non Heap Used", byteCountToDisplaySize(stats.getJvm().getMem().getNonHeapUsed().bytes()));
        nodeAttributes.put("JVM Threads", stats.getJvm().getThreads().getCount());
        nodeAttributes.put("Field Data Memory", byteCountToDisplaySize(stats.getIndices().getFieldData().getMemorySizeInBytes()));
        nodeAttributes.put("Field Data Circuit Breaker Limit", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getLimit()));
        nodeAttributes.put("Field Data Circuit Breaker Estimation", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated()));
        nodeAttributes.put("Request Circuit Breaker Limit", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.REQUEST).getLimit()));
        nodeAttributes.put("Request Circuit Breaker Estimation", byteCountToDisplaySize(stats.getBreaker().getStats(CircuitBreaker.REQUEST).getEstimated()));
        nodeAttributes.put("Query Cache Memory", byteCountToDisplaySize(stats.getIndices().getQueryCache().getMemorySizeInBytes()));
        nodeAttributes.put("Request Cache Memory", byteCountToDisplaySize(stats.getIndices().getRequestCache().getMemorySizeInBytes()));
        nodeAttributes.put("Query Cache Memory", byteCountToDisplaySize(stats.getIndices().getQueryCache().getMemorySizeInBytes()));
    }
    return nodes;
}