Examples with NodeStats org.elasticsearch.action.admin.cluster.node.stats.NodeStats used on opensource projects

Example 1 with NodeStats

use of org.elasticsearch.action.admin.cluster.node.stats.NodeStats in project crate by crate.

the class NodeStatsContextFieldResolverTest method testPSQLPortResolution.

@Test
public void testPSQLPortResolution() throws IOException {
    NodeInfo nodeInfo = mock(NodeInfo.class);
    when(nodeService.info()).thenReturn(nodeInfo);
    NodeStats stats = mock(NodeStats.class);
    when(nodeService.stats()).thenReturn(stats);
    when(stats.getNode()).thenReturn(mock(DiscoveryNode.class));
    InetSocketTransportAddress inetAddress = new InetSocketTransportAddress(Inet4Address.getLocalHost(), 5432);
    BoundTransportAddress boundAddress = new BoundTransportAddress(new TransportAddress[] { inetAddress }, inetAddress);
    when(postgresNetty.boundAddress()).thenReturn(boundAddress);
    NodeStatsContext context = resolver.forTopColumnIdents(ImmutableSet.of(SysNodesTableInfo.Columns.PORT));
    assertThat(context.isComplete(), is(true));
    assertThat(context.port().get("psql"), is(5432));
}

Example 2 with NodeStats

use of org.elasticsearch.action.admin.cluster.node.stats.NodeStats in project elasticsearch by elastic.

the class SearchPreferenceIT method testNodesOnlyRandom.

public void testNodesOnlyRandom() throws Exception {
    assertAcked(prepareCreate("test").setSettings(//this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data
    Settings.builder().put(indexSettings()).put(SETTING_NUMBER_OF_REPLICAS, between(1, maximumNumberOfReplicas()))));
    ensureGreen();
    client().prepareIndex("test", "type1").setSource("field1", "value1").execute().actionGet();
    refresh();
    final Client client = internalCluster().smartClient();
    SearchRequestBuilder request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference(// multiple wildchar  to cover multi-param usecase
    "_only_nodes:*,nodes*");
    assertSearchOnRandomNodes(request);
    request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference("_only_nodes:*");
    assertSearchOnRandomNodes(request);
    ArrayList<String> allNodeIds = new ArrayList<>();
    ArrayList<String> allNodeNames = new ArrayList<>();
    ArrayList<String> allNodeHosts = new ArrayList<>();
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet();
    for (NodeStats node : nodeStats.getNodes()) {
        allNodeIds.add(node.getNode().getId());
        allNodeNames.add(node.getNode().getName());
        allNodeHosts.add(node.getHostname());
    }
    String node_expr = "_only_nodes:" + Strings.arrayToCommaDelimitedString(allNodeIds.toArray());
    request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference(node_expr);
    assertSearchOnRandomNodes(request);
    node_expr = "_only_nodes:" + Strings.arrayToCommaDelimitedString(allNodeNames.toArray());
    request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference(node_expr);
    assertSearchOnRandomNodes(request);
    node_expr = "_only_nodes:" + Strings.arrayToCommaDelimitedString(allNodeHosts.toArray());
    request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference(node_expr);
    assertSearchOnRandomNodes(request);
    node_expr = "_only_nodes:" + Strings.arrayToCommaDelimitedString(allNodeHosts.toArray());
    request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference(node_expr);
    assertSearchOnRandomNodes(request);
    // Mix of valid and invalid nodes
    node_expr = "_only_nodes:*,invalidnode";
    request = client.prepareSearch("test").setQuery(matchAllQuery()).setPreference(node_expr);
    assertSearchOnRandomNodes(request);
}

Example 3 with NodeStats

use of org.elasticsearch.action.admin.cluster.node.stats.NodeStats in project elasticsearch by elastic.

the class ExceptionRetryIT method testRetryDueToExceptionOnNetworkLayer.

/**
     * Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally
     * failing. If auto generated ids are used this must not lead to duplicate ids
     * see https://github.com/elastic/elasticsearch/issues/8788
     */
public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {
    final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
    int numDocs = scaledRandomIntBetween(100, 1000);
    Client client = internalCluster().coordOnlyNodeClient();
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    NodeStats unluckyNode = randomFrom(nodeStats.getNodes().stream().filter((s) -> s.getNode().isDataNode()).collect(Collectors.toList()));
    assertAcked(client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 1).put("index.number_of_shards", 5)));
    ensureGreen("index");
    logger.info("unlucky node: {}", unluckyNode.getNode());
    //create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.
    for (NodeStats dataNode : nodeStats.getNodes()) {
        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
        mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {

            @Override
            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                super.sendRequest(connection, requestId, action, request, options);
                if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
                    logger.debug("Throw ConnectTransportException");
                    throw new ConnectTransportException(connection.getNode(), action);
                }
            }
        });
    }
    BulkRequestBuilder bulkBuilder = client.prepareBulk();
    for (int i = 0; i < numDocs; i++) {
        XContentBuilder doc = null;
        doc = jsonBuilder().startObject().field("foo", "bar").endObject();
        bulkBuilder.add(client.prepareIndex("index", "type").setSource(doc));
    }
    BulkResponse response = bulkBuilder.get();
    if (response.hasFailures()) {
        for (BulkItemResponse singleIndexRespons : response.getItems()) {
            if (singleIndexRespons.isFailed()) {
                fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage());
            }
        }
    }
    refresh();
    SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
    Set<String> uniqueIds = new HashSet();
    long dupCounter = 0;
    boolean found_duplicate_already = false;
    for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
        if (!uniqueIds.add(searchResponse.getHits().getHits()[i].getId())) {
            if (!found_duplicate_already) {
                SearchResponse dupIdResponse = client().prepareSearch("index").setQuery(termQuery("_id", searchResponse.getHits().getHits()[i].getId())).setExplain(true).get();
                assertThat(dupIdResponse.getHits().getTotalHits(), greaterThan(1L));
                logger.info("found a duplicate id:");
                for (SearchHit hit : dupIdResponse.getHits()) {
                    logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
                }
                logger.info("will not print anymore in case more duplicates are found.");
                found_duplicate_already = true;
            }
            dupCounter++;
        }
    }
    assertSearchResponse(searchResponse);
    assertThat(dupCounter, equalTo(0L));
    assertHitCount(searchResponse, numDocs);
    IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setSegments(true).get();
    IndexStats indexStats = index.getIndex("index");
    long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
    for (IndexShardStats indexShardStats : indexStats) {
        for (ShardStats shardStats : indexShardStats) {
            SegmentsStats segments = shardStats.getStats().getSegments();
            maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp());
        }
    }
    assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get());
    assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1);
}

Example 4 with NodeStats

use of org.elasticsearch.action.admin.cluster.node.stats.NodeStats in project elasticsearch by elastic.

the class SuggestStatsIT method testSimpleStats.

public void testSimpleStats() throws Exception {
    // clear all stats first
    client().admin().indices().prepareStats().clear().execute().actionGet();
    final int numNodes = cluster().numDataNodes();
    assertThat(numNodes, greaterThanOrEqualTo(2));
    // we make sure each node gets at least a single shard...
    final int shardsIdx1 = randomIntBetween(1, 10);
    final int shardsIdx2 = Math.max(numNodes - shardsIdx1, randomIntBetween(1, 10));
    final int totalShards = shardsIdx1 + shardsIdx2;
    assertThat(numNodes, lessThanOrEqualTo(totalShards));
    assertAcked(prepareCreate("test1").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, shardsIdx1).put(SETTING_NUMBER_OF_REPLICAS, 0)).addMapping("type", "f", "type=text"));
    assertAcked(prepareCreate("test2").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, shardsIdx2).put(SETTING_NUMBER_OF_REPLICAS, 0)).addMapping("type", "f", "type=text"));
    assertThat(shardsIdx1 + shardsIdx2, equalTo(numAssignedShards("test1", "test2")));
    assertThat(numAssignedShards("test1", "test2"), greaterThanOrEqualTo(2));
    ensureGreen();
    for (int i = 0; i < randomIntBetween(20, 100); i++) {
        index("test" + ((i % 2) + 1), "type", "" + i, "f", "test" + i);
    }
    refresh();
    int suggestAllIdx = scaledRandomIntBetween(20, 50);
    int suggestIdx1 = scaledRandomIntBetween(20, 50);
    int suggestIdx2 = scaledRandomIntBetween(20, 50);
    long startTime = System.currentTimeMillis();
    for (int i = 0; i < suggestAllIdx; i++) {
        SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch(), i).get();
        assertAllSuccessful(suggestResponse);
    }
    for (int i = 0; i < suggestIdx1; i++) {
        SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch("test1"), i).get();
        assertAllSuccessful(suggestResponse);
    }
    for (int i = 0; i < suggestIdx2; i++) {
        SearchResponse suggestResponse = addSuggestions(internalCluster().coordOnlyNodeClient().prepareSearch("test2"), i).get();
        assertAllSuccessful(suggestResponse);
    }
    long endTime = System.currentTimeMillis();
    IndicesStatsResponse indicesStats = client().admin().indices().prepareStats().execute().actionGet();
    final SearchStats.Stats suggest = indicesStats.getTotal().getSearch().getTotal();
    // check current
    assertThat(suggest.getSuggestCurrent(), equalTo(0L));
    // check suggest count
    assertThat(suggest.getSuggestCount(), equalTo((long) (suggestAllIdx * totalShards + suggestIdx1 * shardsIdx1 + suggestIdx2 * shardsIdx2)));
    assertThat(indicesStats.getIndices().get("test1").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx1) * shardsIdx1)));
    assertThat(indicesStats.getIndices().get("test2").getTotal().getSearch().getTotal().getSuggestCount(), equalTo((long) ((suggestAllIdx + suggestIdx2) * shardsIdx2)));
    logger.info("iter {}, iter1 {}, iter2 {}, {}", suggestAllIdx, suggestIdx1, suggestIdx2, endTime - startTime);
    // check suggest time
    assertThat(suggest.getSuggestTimeInMillis(), greaterThan(0L));
    // the upperbound is num shards * total time since we do searches in parallel
    assertThat(suggest.getSuggestTimeInMillis(), lessThanOrEqualTo(totalShards * (endTime - startTime)));
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().execute().actionGet();
    Set<String> nodeIdsWithIndex = nodeIdsWithIndex("test1", "test2");
    int num = 0;
    for (NodeStats stat : nodeStats.getNodes()) {
        SearchStats.Stats suggestStats = stat.getIndices().getSearch().getTotal();
        logger.info("evaluating {}", stat.getNode());
        if (nodeIdsWithIndex.contains(stat.getNode().getId())) {
            assertThat(suggestStats.getSuggestCount(), greaterThan(0L));
            assertThat(suggestStats.getSuggestTimeInMillis(), greaterThan(0L));
            num++;
        } else {
            assertThat(suggestStats.getSuggestCount(), equalTo(0L));
            assertThat(suggestStats.getSuggestTimeInMillis(), equalTo(0L));
        }
    }
    assertThat(num, greaterThan(0));
}

Example 5 with NodeStats

use of org.elasticsearch.action.admin.cluster.node.stats.NodeStats in project elasticsearch by elastic.

the class ClusterStatsIT method testValuesSmokeScreen.

public void testValuesSmokeScreen() throws IOException, ExecutionException, InterruptedException {
    internalCluster().startNodes(randomIntBetween(1, 3));
    index("test1", "type", "1", "f", "f");
    ClusterStatsResponse response = client().admin().cluster().prepareClusterStats().get();
    String msg = response.toString();
    // 1 Jan 2000
    assertThat(msg, response.getTimestamp(), Matchers.greaterThan(946681200000L));
    assertThat(msg, response.indicesStats.getStore().getSizeInBytes(), Matchers.greaterThan(0L));
    assertThat(msg, response.nodesStats.getFs().getTotal().getBytes(), Matchers.greaterThan(0L));
    assertThat(msg, response.nodesStats.getJvm().getVersions().size(), Matchers.greaterThan(0));
    assertThat(msg, response.nodesStats.getVersions().size(), Matchers.greaterThan(0));
    assertThat(msg, response.nodesStats.getVersions().contains(Version.CURRENT), Matchers.equalTo(true));
    assertThat(msg, response.nodesStats.getPlugins().size(), Matchers.greaterThanOrEqualTo(0));
    assertThat(msg, response.nodesStats.getProcess().count, Matchers.greaterThan(0));
    // 0 happens when not supported on platform
    assertThat(msg, response.nodesStats.getProcess().getAvgOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(0L));
    // these can be -1 if not supported on platform
    assertThat(msg, response.nodesStats.getProcess().getMinOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L));
    assertThat(msg, response.nodesStats.getProcess().getMaxOpenFileDescriptors(), Matchers.greaterThanOrEqualTo(-1L));
    NodesStatsResponse nodesStatsResponse = client().admin().cluster().prepareNodesStats().setOs(true).get();
    long total = 0;
    long free = 0;
    long used = 0;
    for (NodeStats nodeStats : nodesStatsResponse.getNodes()) {
        total += nodeStats.getOs().getMem().getTotal().getBytes();
        free += nodeStats.getOs().getMem().getFree().getBytes();
        used += nodeStats.getOs().getMem().getUsed().getBytes();
    }
    assertEquals(msg, free, response.nodesStats.getOs().getMem().getFree().getBytes());
    assertEquals(msg, total, response.nodesStats.getOs().getMem().getTotal().getBytes());
    assertEquals(msg, used, response.nodesStats.getOs().getMem().getUsed().getBytes());
    assertEquals(msg, OsStats.calculatePercentage(used, total), response.nodesStats.getOs().getMem().getUsedPercent());
    assertEquals(msg, OsStats.calculatePercentage(free, total), response.nodesStats.getOs().getMem().getFreePercent());
}