Examples with NodesStatsResponse org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse used on opensource projects

Example 31 with NodesStatsResponse

use of org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse in project elasticsearch by elastic.

the class MockDiskUsagesIT method testRerouteOccursOnDiskPassingHighWatermark.

public void testRerouteOccursOnDiskPassingHighWatermark() throws Exception {
    List<String> nodes = internalCluster().startNodes(3);
    // Wait for all 3 nodes to be up
    assertBusy(new Runnable() {

        @Override
        public void run() {
            NodesStatsResponse resp = client().admin().cluster().prepareNodesStats().get();
            assertThat(resp.getNodes().size(), equalTo(3));
        }
    });
    // Start with all nodes at 50% usage
    final MockInternalClusterInfoService cis = (MockInternalClusterInfoService) internalCluster().getInstance(ClusterInfoService.class, internalCluster().getMasterName());
    cis.setUpdateFrequency(TimeValue.timeValueMillis(200));
    cis.onMaster();
    cis.setN1Usage(nodes.get(0), new DiskUsage(nodes.get(0), "n1", "/dev/null", 100, 50));
    cis.setN2Usage(nodes.get(1), new DiskUsage(nodes.get(1), "n2", "/dev/null", 100, 50));
    cis.setN3Usage(nodes.get(2), new DiskUsage(nodes.get(2), "n3", "/dev/null", 100, 50));
    client().admin().cluster().prepareUpdateSettings().setTransientSettings(Settings.builder().put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_LOW_DISK_WATERMARK_SETTING.getKey(), randomFrom("20b", "80%")).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_HIGH_DISK_WATERMARK_SETTING.getKey(), randomFrom("10b", "90%")).put(DiskThresholdSettings.CLUSTER_ROUTING_ALLOCATION_REROUTE_INTERVAL_SETTING.getKey(), "1ms")).get();
    // Create an index with 10 shards so we can check allocation for it
    prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 10).put("number_of_replicas", 0).put("index.routing.allocation.exclude._name", "")).get();
    ensureGreen("test");
    // Block until the "fake" cluster info is retrieved at least once
    assertBusy(new Runnable() {

        @Override
        public void run() {
            ClusterInfo info = cis.getClusterInfo();
            logger.info("--> got: {} nodes", info.getNodeLeastAvailableDiskUsages().size());
            assertThat(info.getNodeLeastAvailableDiskUsages().size(), greaterThan(0));
        }
    });
    final List<String> realNodeNames = new ArrayList<>();
    ClusterStateResponse resp = client().admin().cluster().prepareState().get();
    Iterator<RoutingNode> iter = resp.getState().getRoutingNodes().iterator();
    while (iter.hasNext()) {
        RoutingNode node = iter.next();
        realNodeNames.add(node.nodeId());
        logger.info("--> node {} has {} shards", node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
    }
    // Update the disk usages so one node has now passed the high watermark
    cis.setN1Usage(realNodeNames.get(0), new DiskUsage(nodes.get(0), "n1", "_na_", 100, 50));
    cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", "_na_", 100, 50));
    // nothing free on node3
    cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", "_na_", 100, 0));
    // Retrieve the count of shards on each node
    final Map<String, Integer> nodesToShardCount = new HashMap<>();
    assertBusy(new Runnable() {

        @Override
        public void run() {
            ClusterStateResponse resp = client().admin().cluster().prepareState().get();
            Iterator<RoutingNode> iter = resp.getState().getRoutingNodes().iterator();
            while (iter.hasNext()) {
                RoutingNode node = iter.next();
                logger.info("--> node {} has {} shards", node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
                nodesToShardCount.put(node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
            }
            assertThat("node1 has 5 shards", nodesToShardCount.get(realNodeNames.get(0)), equalTo(5));
            assertThat("node2 has 5 shards", nodesToShardCount.get(realNodeNames.get(1)), equalTo(5));
            assertThat("node3 has 0 shards", nodesToShardCount.get(realNodeNames.get(2)), equalTo(0));
        }
    });
    // Update the disk usages so one node is now back under the high watermark
    cis.setN1Usage(realNodeNames.get(0), new DiskUsage(nodes.get(0), "n1", "_na_", 100, 50));
    cis.setN2Usage(realNodeNames.get(1), new DiskUsage(nodes.get(1), "n2", "_na_", 100, 50));
    // node3 has free space now
    cis.setN3Usage(realNodeNames.get(2), new DiskUsage(nodes.get(2), "n3", "_na_", 100, 50));
    // Retrieve the count of shards on each node
    nodesToShardCount.clear();
    assertBusy(new Runnable() {

        @Override
        public void run() {
            ClusterStateResponse resp = client().admin().cluster().prepareState().get();
            Iterator<RoutingNode> iter = resp.getState().getRoutingNodes().iterator();
            while (iter.hasNext()) {
                RoutingNode node = iter.next();
                logger.info("--> node {} has {} shards", node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
                nodesToShardCount.put(node.nodeId(), resp.getState().getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
            }
            assertThat("node1 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(0)), greaterThanOrEqualTo(3));
            assertThat("node2 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(1)), greaterThanOrEqualTo(3));
            assertThat("node3 has at least 3 shards", nodesToShardCount.get(realNodeNames.get(2)), greaterThanOrEqualTo(3));
        }
    });
}

Example 32 with NodesStatsResponse

use of org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse in project elasticsearch by elastic.

the class ExternalTestCluster method ensureEstimatedStats.

@Override
public void ensureEstimatedStats() {
    if (size() > 0) {
        NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().clear().setBreaker(true).setIndices(true).execute().actionGet();
        for (NodeStats stats : nodeStats.getNodes()) {
            assertThat("Fielddata breaker not reset to 0 on node: " + stats.getNode(), stats.getBreaker().getStats(CircuitBreaker.FIELDDATA).getEstimated(), equalTo(0L));
            // ExternalTestCluster does not check the request breaker,
            // because checking it requires a network request, which in
            // turn increments the breaker, making it non-0
            assertThat("Fielddata size must be 0 on node: " + stats.getNode(), stats.getIndices().getFieldData().getMemorySizeInBytes(), equalTo(0L));
            assertThat("Query cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getQueryCache().getMemorySizeInBytes(), equalTo(0L));
            assertThat("FixedBitSet cache size must be 0 on node: " + stats.getNode(), stats.getIndices().getSegments().getBitsetMemoryInBytes(), equalTo(0L));
        }
    }
}

Example 33 with NodesStatsResponse

use of org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse in project MSEC by Tencent.

the class ESHelper method ClusterStatus.

public void ClusterStatus(ArrayList<String> ips, String cluster_name, QueryESClusterDetailResponse response) {
    Logger logger = Logger.getLogger(ESHelper.class);
    TransportClient client = null;
    HashSet<String> total_ips = new HashSet<>(ips);
    try {
        Settings settings = Settings.builder().put("cluster.name", cluster_name).put("client.transport.sniff", true).build();
        for (String ip : ips) {
            if (client == null) {
                client = new PreBuiltTransportClient(settings).addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(ip), default_client_port));
            } else
                client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(ip), default_client_port));
        }
        ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth().setTimeout(TimeValue.timeValueSeconds(5)).execute().actionGet();
        if (healthResponse != null && !healthResponse.isTimedOut()) {
            response.setActive_shards(healthResponse.getActiveShards());
            response.setTotal_shards(healthResponse.getActiveShards() + healthResponse.getUnassignedShards());
            response.setHealth_status(healthResponse.getStatus().toString().toLowerCase());
            response.setServer_port(default_port);
            logger.info(healthResponse);
            if (healthResponse.getNumberOfNodes() > 0) {
                NodesStatsResponse nodeStats = client.admin().cluster().prepareNodesStats().all().get();
                if (nodeStats != null) {
                    for (NodeStats stats : nodeStats.getNodes()) {
                        QueryESClusterDetailResponse.RTInfo info = new QueryESClusterDetailResponse().new RTInfo();
                        info.setOK(true);
                        info.setAvail_disk_size(stats.getFs().getTotal().getAvailable().getBytes());
                        info.setDoc_count(stats.getIndices().getDocs().getCount());
                        info.setDoc_disk_size(stats.getIndices().getStore().getSizeInBytes());
                        response.getInfo_map().put(stats.getNode().getAddress().getHost(), info);
                        total_ips.add(stats.getNode().getAddress().getHost());
                    }
                }
            }
        }
        // update Zen settings
        client.admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", total_ips.size() / 2 + 1)).setTransientSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", total_ips.size() / 2 + 1)).get();
        // update template settings
        PutIndexTemplateResponse tpl_resp = client.admin().indices().preparePutTemplate("template_msec").setTemplate("msec_*").setSettings(Settings.builder().put("number_of_replicas", 1).put("refresh_interval", "30s")).addMapping("logs", MAPPING).execute().get();
        logger.info("Create mapping: " + tpl_resp.isAcknowledged());
    } catch (UnknownHostException e) {
        logger.error(e);
    } catch (Exception e) {
        logger.error(e);
    } finally {
        if (client != null)
            client.close();
    }
}

Example 34 with NodesStatsResponse

use of org.elasticsearch.action.admin.cluster.node.stats.NodesStatsResponse in project crate by crate.

the class InternalClusterInfoService method refresh.

/**
 * Refreshes the ClusterInfo in a blocking fashion
 */
public final ClusterInfo refresh() {
    if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("Performing ClusterInfoUpdateJob");
    }
    final CountDownLatch nodeLatch = updateNodeStats(new ActionListener<>() {

        @Override
        public void onResponse(NodesStatsResponse nodesStatsResponse) {
            ImmutableOpenMap.Builder<String, DiskUsage> leastAvailableUsagesBuilder = ImmutableOpenMap.builder();
            ImmutableOpenMap.Builder<String, DiskUsage> mostAvailableUsagesBuilder = ImmutableOpenMap.builder();
            fillDiskUsagePerNode(LOGGER, nodesStatsResponse.getNodes(), leastAvailableUsagesBuilder, mostAvailableUsagesBuilder);
            leastAvailableSpaceUsages = leastAvailableUsagesBuilder.build();
            mostAvailableSpaceUsages = mostAvailableUsagesBuilder.build();
        }

        @Override
        public void onFailure(Exception e) {
            if (e instanceof ReceiveTimeoutTransportException) {
                LOGGER.error("NodeStatsAction timed out for ClusterInfoUpdateJob", e);
            } else {
                if (e instanceof ClusterBlockException) {
                    if (LOGGER.isTraceEnabled()) {
                        LOGGER.trace("Failed to execute NodeStatsAction for ClusterInfoUpdateJob", e);
                    }
                } else {
                    LOGGER.warn("Failed to execute NodeStatsAction for ClusterInfoUpdateJob", e);
                }
                // we empty the usages list, to be safe - we don't know what's going on.
                leastAvailableSpaceUsages = ImmutableOpenMap.of();
                mostAvailableSpaceUsages = ImmutableOpenMap.of();
            }
        }
    });
    final CountDownLatch indicesLatch = updateIndicesStats(new ActionListener<>() {

        @Override
        public void onResponse(IndicesStatsResponse indicesStatsResponse) {
            ShardStats[] stats = indicesStatsResponse.getShards();
            ImmutableOpenMap.Builder<String, Long> newShardSizes = ImmutableOpenMap.builder();
            ImmutableOpenMap.Builder<ShardRouting, String> newShardRoutingToDataPath = ImmutableOpenMap.builder();
            buildShardLevelInfo(LOGGER, stats, newShardSizes, newShardRoutingToDataPath);
            shardSizes = newShardSizes.build();
            shardRoutingToDataPath = newShardRoutingToDataPath.build();
        }

        @Override
        public void onFailure(Exception e) {
            if (e instanceof ReceiveTimeoutTransportException) {
                LOGGER.error("IndicesStatsAction timed out for ClusterInfoUpdateJob", e);
            } else {
                if (e instanceof ClusterBlockException) {
                    if (LOGGER.isTraceEnabled()) {
                        LOGGER.trace("Failed to execute IndicesStatsAction for ClusterInfoUpdateJob", e);
                    }
                } else {
                    LOGGER.warn("Failed to execute IndicesStatsAction for ClusterInfoUpdateJob", e);
                }
                // we empty the usages list, to be safe - we don't know what's going on.
                shardSizes = ImmutableOpenMap.of();
                shardRoutingToDataPath = ImmutableOpenMap.of();
            }
        }
    });
    try {
        nodeLatch.await(fetchTimeout.getMillis(), TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        // restore interrupt status
        Thread.currentThread().interrupt();
        LOGGER.warn("Failed to update node information for ClusterInfoUpdateJob within {} timeout", fetchTimeout);
    }
    try {
        indicesLatch.await(fetchTimeout.getMillis(), TimeUnit.MILLISECONDS);
    } catch (InterruptedException e) {
        // restore interrupt status
        Thread.currentThread().interrupt();
        LOGGER.warn("Failed to update shard information for ClusterInfoUpdateJob within {} timeout", fetchTimeout);
    }
    ClusterInfo clusterInfo = getClusterInfo();
    boolean anyListeners = false;
    for (final Consumer<ClusterInfo> listener : listeners) {
        anyListeners = true;
        try {
            LOGGER.trace("notifying [{}] of new cluster info", listener);
            listener.accept(clusterInfo);
        } catch (Exception e) {
            LOGGER.info(new ParameterizedMessage("failed to notify [{}] of new cluster info", listener), e);
        }
    }
    assert anyListeners : "expected to notify at least one listener";
    return clusterInfo;
}