Examples with NodeStats org.elasticsearch.action.admin.cluster.node.stats.NodeStats used on opensource projects

Example 26 with NodeStats

use of org.elasticsearch.action.admin.cluster.node.stats.NodeStats in project elasticsearch by elastic.

the class CircuitBreakerServiceIT method testLimitsRequestSize.

public void testLimitsRequestSize() throws Exception {
    ByteSizeValue inFlightRequestsLimit = new ByteSizeValue(8, ByteSizeUnit.KB);
    if (noopBreakerUsed()) {
        logger.info("--> noop breakers used, skipping test");
        return;
    }
    internalCluster().ensureAtLeastNumDataNodes(2);
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    List<NodeStats> dataNodeStats = new ArrayList<>();
    for (NodeStats stat : nodeStats.getNodes()) {
        if (stat.getNode().isDataNode()) {
            dataNodeStats.add(stat);
        }
    }
    assertThat(dataNodeStats.size(), greaterThanOrEqualTo(2));
    Collections.shuffle(dataNodeStats, random());
    // send bulk request from source node to target node later. The sole shard is bound to the target node.
    NodeStats targetNode = dataNodeStats.get(0);
    NodeStats sourceNode = dataNodeStats.get(1);
    assertAcked(prepareCreate("index").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).put("index.routing.allocation.include._name", targetNode.getNode().getName()).put(EnableAllocationDecider.INDEX_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), EnableAllocationDecider.Rebalance.NONE)));
    Client client = client(sourceNode.getNode().getName());
    // we use the limit size as a (very) rough indication on how many requests we should sent to hit the limit
    int numRequests = inFlightRequestsLimit.bytesAsInt();
    BulkRequest bulkRequest = new BulkRequest();
    for (int i = 0; i < numRequests; i++) {
        IndexRequest indexRequest = new IndexRequest("index", "type", Integer.toString(i));
        indexRequest.source(Requests.INDEX_CONTENT_TYPE, "field", "value", "num", i);
        bulkRequest.add(indexRequest);
    }
    Settings limitSettings = Settings.builder().put(IN_FLIGHT_REQUESTS_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), inFlightRequestsLimit).build();
    assertAcked(client().admin().cluster().prepareUpdateSettings().setTransientSettings(limitSettings));
    // can either fail directly with an exception or the response contains exceptions (depending on client)
    try {
        BulkResponse response = client.bulk(bulkRequest).actionGet();
        if (!response.hasFailures()) {
            fail("Should have thrown CircuitBreakingException");
        } else {
            // each item must have failed with CircuitBreakingException
            for (BulkItemResponse bulkItemResponse : response) {
                Throwable cause = ExceptionsHelper.unwrapCause(bulkItemResponse.getFailure().getCause());
                assertThat(cause, instanceOf(CircuitBreakingException.class));
                assertEquals(((CircuitBreakingException) cause).getByteLimit(), inFlightRequestsLimit.getBytes());
            }
        }
    } catch (CircuitBreakingException ex) {
        assertEquals(ex.getByteLimit(), inFlightRequestsLimit.getBytes());
    }
}

Example 27 with NodeStats

use of org.elasticsearch.action.admin.cluster.node.stats.NodeStats in project elasticsearch by elastic.

the class CircuitBreakerServiceIT method testRamAccountingTermsEnum.

public void testRamAccountingTermsEnum() throws Exception {
    if (noopBreakerUsed()) {
        logger.info("--> noop breakers used, skipping test");
        return;
    }
    final Client client = client();
    // Create an index where the mappings have a field data filter
    assertAcked(prepareCreate("ramtest").setSource("{\"mappings\": {\"type\": {\"properties\": {\"test\": " + "{\"type\": \"text\",\"fielddata\": true,\"fielddata_frequency_filter\": {\"max\": 10000}}}}}}", XContentType.JSON));
    ensureGreen("ramtest");
    // index some different terms so we have some field data for loading
    int docCount = scaledRandomIntBetween(300, 1000);
    List<IndexRequestBuilder> reqs = new ArrayList<>();
    for (long id = 0; id < docCount; id++) {
        reqs.add(client.prepareIndex("ramtest", "type", Long.toString(id)).setSource("test", "value" + id));
    }
    indexRandom(true, false, true, reqs);
    // execute a search that loads field data (sorting on the "test" field)
    client.prepareSearch("ramtest").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC).get();
    // clear field data cache (thus setting the loaded field data back to 0)
    clearFieldData();
    // Update circuit breaker settings
    Settings settings = Settings.builder().put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_LIMIT_SETTING.getKey(), "100b").put(HierarchyCircuitBreakerService.FIELDDATA_CIRCUIT_BREAKER_OVERHEAD_SETTING.getKey(), 1.05).build();
    assertAcked(client.admin().cluster().prepareUpdateSettings().setTransientSettings(settings));
    // execute a search that loads field data (sorting on the "test" field)
    // again, this time it should trip the breaker
    SearchRequestBuilder searchRequest = client.prepareSearch("ramtest").setQuery(matchAllQuery()).addSort("test", SortOrder.DESC);
    String errMsg = "Data too large, data for [test] would be";
    assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
    errMsg = "which is larger than the limit of [100/100b]";
    assertFailures(searchRequest, RestStatus.INTERNAL_SERVER_ERROR, containsString(errMsg));
    NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().setBreaker(true).get();
    int breaks = 0;
    for (NodeStats stat : stats.getNodes()) {
        CircuitBreakerStats breakerStats = stat.getBreaker().getStats(CircuitBreaker.FIELDDATA);
        breaks += breakerStats.getTrippedCount();
    }
    assertThat(breaks, greaterThanOrEqualTo(1));
}

Example 28 with NodeStats

use of org.elasticsearch.action.admin.cluster.node.stats.NodeStats in project elasticsearch by elastic.

the class MockInternalClusterInfoService method makeStats.

/** Create a fake NodeStats for the given node and usage */
public static NodeStats makeStats(String nodeName, DiskUsage usage) {
    FsInfo.Path[] paths = new FsInfo.Path[1];
    FsInfo.Path path = new FsInfo.Path("/dev/null", null, usage.getTotalBytes(), usage.getFreeBytes(), usage.getFreeBytes());
    paths[0] = path;
    FsInfo fsInfo = new FsInfo(System.currentTimeMillis(), null, paths);
    return new NodeStats(new DiscoveryNode(nodeName, ESTestCase.buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT), System.currentTimeMillis(), null, null, null, null, null, fsInfo, null, null, null, null, null, null);
}

Example 29 with NodeStats

use of org.elasticsearch.action.admin.cluster.node.stats.NodeStats in project MSEC by Tencent.

the class ESHelper method ClusterStatus.

public void ClusterStatus(ArrayList<String> ips, String cluster_name, QueryESClusterDetailResponse response) {
    Logger logger = Logger.getLogger(ESHelper.class);
    TransportClient client = null;
    HashSet<String> total_ips = new HashSet<>(ips);
    try {
        Settings settings = Settings.builder().put("cluster.name", cluster_name).put("client.transport.sniff", true).build();
        for (String ip : ips) {
            if (client == null) {
                client = new PreBuiltTransportClient(settings).addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(ip), default_client_port));
            } else
                client.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(ip), default_client_port));
        }
        ClusterHealthResponse healthResponse = client.admin().cluster().prepareHealth().setTimeout(TimeValue.timeValueSeconds(5)).execute().actionGet();
        if (healthResponse != null && !healthResponse.isTimedOut()) {
            response.setActive_shards(healthResponse.getActiveShards());
            response.setTotal_shards(healthResponse.getActiveShards() + healthResponse.getUnassignedShards());
            response.setHealth_status(healthResponse.getStatus().toString().toLowerCase());
            response.setServer_port(default_port);
            logger.info(healthResponse);
            if (healthResponse.getNumberOfNodes() > 0) {
                NodesStatsResponse nodeStats = client.admin().cluster().prepareNodesStats().all().get();
                if (nodeStats != null) {
                    for (NodeStats stats : nodeStats.getNodes()) {
                        QueryESClusterDetailResponse.RTInfo info = new QueryESClusterDetailResponse().new RTInfo();
                        info.setOK(true);
                        info.setAvail_disk_size(stats.getFs().getTotal().getAvailable().getBytes());
                        info.setDoc_count(stats.getIndices().getDocs().getCount());
                        info.setDoc_disk_size(stats.getIndices().getStore().getSizeInBytes());
                        response.getInfo_map().put(stats.getNode().getAddress().getHost(), info);
                        total_ips.add(stats.getNode().getAddress().getHost());
                    }
                }
            }
        }
        //update Zen settings
        client.admin().cluster().prepareUpdateSettings().setPersistentSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", total_ips.size() / 2 + 1)).setTransientSettings(Settings.builder().put("discovery.zen.minimum_master_nodes", total_ips.size() / 2 + 1)).get();
        //update template settings
        PutIndexTemplateResponse tpl_resp = client.admin().indices().preparePutTemplate("template_msec").setTemplate("msec_*").setSettings(Settings.builder().put("number_of_replicas", 1).put("refresh_interval", "30s")).addMapping("logs", MAPPING).execute().get();
        logger.info("Create mapping: " + tpl_resp.isAcknowledged());
    } catch (UnknownHostException e) {
        logger.error(e);
    } catch (Exception e) {
        logger.error(e);
    } finally {
        if (client != null)
            client.close();
    }
}

Example 30 with NodeStats

use of org.elasticsearch.action.admin.cluster.node.stats.NodeStats in project graylog2-server by Graylog2.

the class IndexerClusterCheckerThread method doRun.

@Override
public void doRun() {
    if (!notificationService.isFirst(Notification.Type.ES_OPEN_FILES)) {
        return;
    }
    try {
        cluster.health().getStatus();
    } catch (Exception e) {
        LOG.info("Indexer not fully initialized yet. Skipping periodic cluster check.");
        return;
    }
    boolean allHigher = true;
    final Map<String, NodeInfo> nodesInfos = cluster.getDataNodes();
    final Map<String, NodeStats> nodesStats = cluster.getNodesStats(nodesInfos.keySet().toArray(new String[nodesInfos.size()]));
    for (Map.Entry<String, NodeStats> entry : nodesStats.entrySet()) {
        final String nodeId = entry.getKey();
        final NodeStats nodeStats = entry.getValue();
        final NodeInfo nodeInfo = nodesInfos.get(nodeId);
        final String nodeName = nodeInfo.getNode().getName();
        // Check number of maximum open files.
        final ProcessStats processStats = nodeStats.getProcess();
        if (processStats == null) {
            LOG.debug("Couldn't read process stats of Elasticsearch node {}", nodeName);
            return;
        }
        final long maxFileDescriptors = processStats.getMaxFileDescriptors();
        final JvmInfo jvmInfo = nodeInfo.getJvm();
        if (jvmInfo == null) {
            LOG.debug("Couldn't read JVM info of Elasticsearch node {}", nodeName);
            return;
        }
        final String osName = jvmInfo.getSystemProperties().getOrDefault("os.name", "");
        if (osName.startsWith("Windows")) {
            LOG.debug("Skipping open file limit check for Indexer node <{}> on Windows", nodeName);
        } else if (maxFileDescriptors != -1 && maxFileDescriptors < MINIMUM_OPEN_FILES_LIMIT) {
            // Write notification.
            final Notification notification = notificationService.buildNow().addType(Notification.Type.ES_OPEN_FILES).addSeverity(Notification.Severity.URGENT).addDetail("hostname", nodeInfo.getHostname()).addDetail("max_file_descriptors", maxFileDescriptors);
            if (notificationService.publishIfFirst(notification)) {
                LOG.warn("Indexer node <{}> open file limit is too low: [{}]. Set it to at least {}.", nodeName, maxFileDescriptors, MINIMUM_OPEN_FILES_LIMIT);
            }
            allHigher = false;
        }
    }
    if (allHigher) {
        Notification notification = notificationService.build().addType(Notification.Type.ES_OPEN_FILES);
        notificationService.fixed(notification);
    }
}