Examples with ProcessStats org.elasticsearch.monitor.process.ProcessStats used on opensource projects

Example 1 with ProcessStats

use of org.elasticsearch.monitor.process.ProcessStats in project crate by crate.

the class SysNodesExpressionsOnHandlerTest method setup.

@BeforeClass
public static void setup() throws IOException {
    // jvm
    JvmStats jvmStats = mock(JvmStats.class);
    JvmStats.Mem jvmStatsMem = mock(JvmStats.Mem.class);
    ByteSizeValue heapByteSizeValueMax = new ByteSizeValue(123456L);
    when(jvmStatsMem.getHeapMax()).thenReturn(heapByteSizeValueMax);
    when(jvmStatsMem.getHeapUsed()).thenReturn(heapByteSizeValueMax);
    when(jvmStats.getMem()).thenReturn(jvmStatsMem);
    // mem
    ByteSizeValue byteSizeValue = new ByteSizeValue(12345342234L);
    // os service
    OsService osService = mock(OsService.class);
    OsStats osStats = mock(OsStats.class);
    when(osService.stats()).thenReturn(osStats);
    OsStats.Mem mem = mock(OsStats.Mem.class);
    when(osStats.getMem()).thenReturn(mem);
    when(mem.getFree()).thenReturn(byteSizeValue);
    when(mem.getUsed()).thenReturn(byteSizeValue);
    when(mem.getUsedPercent()).thenReturn((short) 22);
    when(mem.getFreePercent()).thenReturn((short) 78);
    // os info
    OsInfo osInfo = mock(OsInfo.class);
    when(osService.info()).thenReturn(osInfo);
    when(osInfo.getAvailableProcessors()).thenReturn(4);
    // node info
    NodeEnvironment nodeEnv = mock(NodeEnvironment.class);
    Path[] dataLocations = new Path[] { new File("/foo").toPath(), new File("/bar").toPath() };
    when(nodeEnv.hasNodeFile()).then(invocation -> true);
    when(nodeEnv.nodeDataPaths()).thenReturn(dataLocations);
    ExtendedNodeInfo extendedNodeInfo = new DummyExtendedNodeInfo(nodeEnv);
    // process stats
    ProcessStats processStats = mock(ProcessStats.class);
    when(processStats.getOpenFileDescriptors()).thenReturn(42L);
    when(processStats.getMaxFileDescriptors()).thenReturn(1000L);
    CONTEXT.id(BytesRefs.toBytesRef("93c7ff92-52fa-11e6-aad8-3c15c2d3ad18"));
    CONTEXT.name(BytesRefs.toBytesRef("crate1"));
    CONTEXT.hostname(BytesRefs.toBytesRef("crate1.example.com"));
    CONTEXT.version(Version.CURRENT);
    CONTEXT.build(Build.CURRENT);
    CONTEXT.timestamp(100L);
    CONTEXT.restUrl(BytesRefs.toBytesRef("10.0.0.1:4200"));
    CONTEXT.port(new HashMap<String, Integer>(2) {

        {
            put("http", 4200);
            put("transport", 4300);
        }
    });
    CONTEXT.jvmStats(jvmStats);
    CONTEXT.osInfo(osInfo);
    CONTEXT.processStats(processStats);
    CONTEXT.osStats(osStats);
    CONTEXT.extendedOsStats(extendedNodeInfo.osStats());
    CONTEXT.networkStats(extendedNodeInfo.networkStats());
    CONTEXT.extendedProcessCpuStats(extendedNodeInfo.processCpuStats());
    CONTEXT.extendedFsStats(extendedNodeInfo.fsStats());
}

Example 2 with ProcessStats

use of org.elasticsearch.monitor.process.ProcessStats in project elasticsearch by elastic.

the class RestNodesAction method buildTable.

private Table buildTable(boolean fullId, RestRequest req, ClusterStateResponse state, NodesInfoResponse nodesInfo, NodesStatsResponse nodesStats) {
    DiscoveryNodes nodes = state.getState().nodes();
    String masterId = nodes.getMasterNodeId();
    Table table = getTableWithHeader(req);
    for (DiscoveryNode node : nodes) {
        NodeInfo info = nodesInfo.getNodesMap().get(node.getId());
        NodeStats stats = nodesStats.getNodesMap().get(node.getId());
        JvmInfo jvmInfo = info == null ? null : info.getJvm();
        JvmStats jvmStats = stats == null ? null : stats.getJvm();
        FsInfo fsInfo = stats == null ? null : stats.getFs();
        OsStats osStats = stats == null ? null : stats.getOs();
        ProcessStats processStats = stats == null ? null : stats.getProcess();
        NodeIndicesStats indicesStats = stats == null ? null : stats.getIndices();
        table.startRow();
        table.addCell(fullId ? node.getId() : Strings.substring(node.getId(), 0, 4));
        table.addCell(info == null ? null : info.getProcess().getId());
        table.addCell(node.getHostAddress());
        table.addCell(node.getAddress().address().getPort());
        final HttpInfo httpInfo = info == null ? null : info.getHttp();
        if (httpInfo != null) {
            TransportAddress transportAddress = httpInfo.getAddress().publishAddress();
            table.addCell(NetworkAddress.format(transportAddress.address()));
        } else {
            table.addCell("-");
        }
        table.addCell(node.getVersion().toString());
        table.addCell(info == null ? null : info.getBuild().shortHash());
        table.addCell(jvmInfo == null ? null : jvmInfo.version());
        table.addCell(fsInfo == null ? null : fsInfo.getTotal().getAvailable());
        table.addCell(jvmStats == null ? null : jvmStats.getMem().getHeapUsed());
        table.addCell(jvmStats == null ? null : jvmStats.getMem().getHeapUsedPercent());
        table.addCell(jvmInfo == null ? null : jvmInfo.getMem().getHeapMax());
        table.addCell(osStats == null ? null : osStats.getMem() == null ? null : osStats.getMem().getUsed());
        table.addCell(osStats == null ? null : osStats.getMem() == null ? null : osStats.getMem().getUsedPercent());
        table.addCell(osStats == null ? null : osStats.getMem() == null ? null : osStats.getMem().getTotal());
        table.addCell(processStats == null ? null : processStats.getOpenFileDescriptors());
        table.addCell(processStats == null ? null : calculatePercentage(processStats.getOpenFileDescriptors(), processStats.getMaxFileDescriptors()));
        table.addCell(processStats == null ? null : processStats.getMaxFileDescriptors());
        table.addCell(osStats == null ? null : Short.toString(osStats.getCpu().getPercent()));
        boolean hasLoadAverage = osStats != null && osStats.getCpu().getLoadAverage() != null;
        table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[0] == -1 ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[0]));
        table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[1] == -1 ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[1]));
        table.addCell(!hasLoadAverage || osStats.getCpu().getLoadAverage()[2] == -1 ? null : String.format(Locale.ROOT, "%.2f", osStats.getCpu().getLoadAverage()[2]));
        table.addCell(jvmStats == null ? null : jvmStats.getUptime());
        final String roles;
        if (node.getRoles().isEmpty()) {
            roles = "-";
        } else {
            roles = node.getRoles().stream().map(DiscoveryNode.Role::getAbbreviation).collect(Collectors.joining());
        }
        table.addCell(roles);
        table.addCell(masterId == null ? "x" : masterId.equals(node.getId()) ? "*" : "-");
        table.addCell(node.getName());
        CompletionStats completionStats = indicesStats == null ? null : stats.getIndices().getCompletion();
        table.addCell(completionStats == null ? null : completionStats.getSize());
        FieldDataStats fdStats = indicesStats == null ? null : stats.getIndices().getFieldData();
        table.addCell(fdStats == null ? null : fdStats.getMemorySize());
        table.addCell(fdStats == null ? null : fdStats.getEvictions());
        QueryCacheStats fcStats = indicesStats == null ? null : indicesStats.getQueryCache();
        table.addCell(fcStats == null ? null : fcStats.getMemorySize());
        table.addCell(fcStats == null ? null : fcStats.getEvictions());
        RequestCacheStats qcStats = indicesStats == null ? null : indicesStats.getRequestCache();
        table.addCell(qcStats == null ? null : qcStats.getMemorySize());
        table.addCell(qcStats == null ? null : qcStats.getEvictions());
        table.addCell(qcStats == null ? null : qcStats.getHitCount());
        table.addCell(qcStats == null ? null : qcStats.getMissCount());
        FlushStats flushStats = indicesStats == null ? null : indicesStats.getFlush();
        table.addCell(flushStats == null ? null : flushStats.getTotal());
        table.addCell(flushStats == null ? null : flushStats.getTotalTime());
        GetStats getStats = indicesStats == null ? null : indicesStats.getGet();
        table.addCell(getStats == null ? null : getStats.current());
        table.addCell(getStats == null ? null : getStats.getTime());
        table.addCell(getStats == null ? null : getStats.getCount());
        table.addCell(getStats == null ? null : getStats.getExistsTime());
        table.addCell(getStats == null ? null : getStats.getExistsCount());
        table.addCell(getStats == null ? null : getStats.getMissingTime());
        table.addCell(getStats == null ? null : getStats.getMissingCount());
        IndexingStats indexingStats = indicesStats == null ? null : indicesStats.getIndexing();
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getDeleteCurrent());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getDeleteTime());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getDeleteCount());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexCurrent());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexTime());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexCount());
        table.addCell(indexingStats == null ? null : indexingStats.getTotal().getIndexFailedCount());
        MergeStats mergeStats = indicesStats == null ? null : indicesStats.getMerge();
        table.addCell(mergeStats == null ? null : mergeStats.getCurrent());
        table.addCell(mergeStats == null ? null : mergeStats.getCurrentNumDocs());
        table.addCell(mergeStats == null ? null : mergeStats.getCurrentSize());
        table.addCell(mergeStats == null ? null : mergeStats.getTotal());
        table.addCell(mergeStats == null ? null : mergeStats.getTotalNumDocs());
        table.addCell(mergeStats == null ? null : mergeStats.getTotalSize());
        table.addCell(mergeStats == null ? null : mergeStats.getTotalTime());
        RefreshStats refreshStats = indicesStats == null ? null : indicesStats.getRefresh();
        table.addCell(refreshStats == null ? null : refreshStats.getTotal());
        table.addCell(refreshStats == null ? null : refreshStats.getTotalTime());
        table.addCell(refreshStats == null ? null : refreshStats.getListeners());
        ScriptStats scriptStats = stats == null ? null : stats.getScriptStats();
        table.addCell(scriptStats == null ? null : scriptStats.getCompilations());
        table.addCell(scriptStats == null ? null : scriptStats.getCacheEvictions());
        SearchStats searchStats = indicesStats == null ? null : indicesStats.getSearch();
        table.addCell(searchStats == null ? null : searchStats.getTotal().getFetchCurrent());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getFetchTime());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getFetchCount());
        table.addCell(searchStats == null ? null : searchStats.getOpenContexts());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryCurrent());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryTime());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getQueryCount());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollCurrent());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollTime());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getScrollCount());
        SegmentsStats segmentsStats = indicesStats == null ? null : indicesStats.getSegments();
        table.addCell(segmentsStats == null ? null : segmentsStats.getCount());
        table.addCell(segmentsStats == null ? null : segmentsStats.getMemory());
        table.addCell(segmentsStats == null ? null : segmentsStats.getIndexWriterMemory());
        table.addCell(segmentsStats == null ? null : segmentsStats.getVersionMapMemory());
        table.addCell(segmentsStats == null ? null : segmentsStats.getBitsetMemory());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestCurrent());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestTime());
        table.addCell(searchStats == null ? null : searchStats.getTotal().getSuggestCount());
        table.endRow();
    }
    return table;
}

Example 3 with ProcessStats

use of org.elasticsearch.monitor.process.ProcessStats in project elasticsearch by elastic.

the class NodeStatsTests method createNodeStats.

private static NodeStats createNodeStats() {
    DiscoveryNode node = new DiscoveryNode("test_node", buildNewFakeTransportAddress(), emptyMap(), emptySet(), VersionUtils.randomVersion(random()));
    OsStats osStats = null;
    if (frequently()) {
        double[] loadAverages = new double[3];
        for (int i = 0; i < 3; i++) {
            loadAverages[i] = randomBoolean() ? randomDouble() : -1;
        }
        osStats = new OsStats(System.currentTimeMillis(), new OsStats.Cpu(randomShort(), loadAverages), new OsStats.Mem(randomLong(), randomLong()), new OsStats.Swap(randomLong(), randomLong()), new OsStats.Cgroup(randomAsciiOfLength(8), randomNonNegativeLong(), randomAsciiOfLength(8), randomNonNegativeLong(), randomNonNegativeLong(), new OsStats.Cgroup.CpuStat(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong())));
    }
    ProcessStats processStats = frequently() ? new ProcessStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), new ProcessStats.Cpu(randomShort(), randomNonNegativeLong()), new ProcessStats.Mem(randomNonNegativeLong())) : null;
    JvmStats jvmStats = null;
    if (frequently()) {
        int numMemoryPools = randomIntBetween(0, 10);
        List<JvmStats.MemoryPool> memoryPools = new ArrayList<>(numMemoryPools);
        for (int i = 0; i < numMemoryPools; i++) {
            memoryPools.add(new JvmStats.MemoryPool(randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
        }
        JvmStats.Threads threads = new JvmStats.Threads(randomIntBetween(1, 1000), randomIntBetween(1, 1000));
        int numGarbageCollectors = randomIntBetween(0, 10);
        JvmStats.GarbageCollector[] garbageCollectorsArray = new JvmStats.GarbageCollector[numGarbageCollectors];
        for (int i = 0; i < numGarbageCollectors; i++) {
            garbageCollectorsArray[i] = new JvmStats.GarbageCollector(randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(), randomNonNegativeLong());
        }
        JvmStats.GarbageCollectors garbageCollectors = new JvmStats.GarbageCollectors(garbageCollectorsArray);
        int numBufferPools = randomIntBetween(0, 10);
        List<JvmStats.BufferPool> bufferPoolList = new ArrayList<>();
        for (int i = 0; i < numBufferPools; i++) {
            bufferPoolList.add(new JvmStats.BufferPool(randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
        }
        JvmStats.Classes classes = new JvmStats.Classes(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
        jvmStats = frequently() ? new JvmStats(randomNonNegativeLong(), randomNonNegativeLong(), new JvmStats.Mem(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), memoryPools), threads, garbageCollectors, randomBoolean() ? Collections.emptyList() : bufferPoolList, classes) : null;
    }
    ThreadPoolStats threadPoolStats = null;
    if (frequently()) {
        int numThreadPoolStats = randomIntBetween(0, 10);
        List<ThreadPoolStats.Stats> threadPoolStatsList = new ArrayList<>();
        for (int i = 0; i < numThreadPoolStats; i++) {
            threadPoolStatsList.add(new ThreadPoolStats.Stats(randomAsciiOfLengthBetween(3, 10), randomIntBetween(1, 1000), randomIntBetween(1, 1000), randomIntBetween(1, 1000), randomNonNegativeLong(), randomIntBetween(1, 1000), randomIntBetween(1, 1000)));
        }
        threadPoolStats = new ThreadPoolStats(threadPoolStatsList);
    }
    FsInfo fsInfo = null;
    if (frequently()) {
        int numDeviceStats = randomIntBetween(0, 10);
        FsInfo.DeviceStats[] deviceStatsArray = new FsInfo.DeviceStats[numDeviceStats];
        for (int i = 0; i < numDeviceStats; i++) {
            FsInfo.DeviceStats previousDeviceStats = randomBoolean() ? null : new FsInfo.DeviceStats(randomInt(), randomInt(), randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), null);
            deviceStatsArray[i] = new FsInfo.DeviceStats(randomInt(), randomInt(), randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), previousDeviceStats);
        }
        FsInfo.IoStats ioStats = new FsInfo.IoStats(deviceStatsArray);
        int numPaths = randomIntBetween(0, 10);
        FsInfo.Path[] paths = new FsInfo.Path[numPaths];
        for (int i = 0; i < numPaths; i++) {
            paths[i] = new FsInfo.Path(randomAsciiOfLengthBetween(3, 10), randomBoolean() ? randomAsciiOfLengthBetween(3, 10) : null, randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
        }
        fsInfo = new FsInfo(randomNonNegativeLong(), ioStats, paths);
    }
    TransportStats transportStats = frequently() ? new TransportStats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()) : null;
    HttpStats httpStats = frequently() ? new HttpStats(randomNonNegativeLong(), randomNonNegativeLong()) : null;
    AllCircuitBreakerStats allCircuitBreakerStats = null;
    if (frequently()) {
        int numCircuitBreakerStats = randomIntBetween(0, 10);
        CircuitBreakerStats[] circuitBreakerStatsArray = new CircuitBreakerStats[numCircuitBreakerStats];
        for (int i = 0; i < numCircuitBreakerStats; i++) {
            circuitBreakerStatsArray[i] = new CircuitBreakerStats(randomAsciiOfLengthBetween(3, 10), randomNonNegativeLong(), randomNonNegativeLong(), randomDouble(), randomNonNegativeLong());
        }
        allCircuitBreakerStats = new AllCircuitBreakerStats(circuitBreakerStatsArray);
    }
    ScriptStats scriptStats = frequently() ? new ScriptStats(randomNonNegativeLong(), randomNonNegativeLong()) : null;
    DiscoveryStats discoveryStats = frequently() ? new DiscoveryStats(randomBoolean() ? new PendingClusterStateStats(randomInt(), randomInt(), randomInt()) : null) : null;
    IngestStats ingestStats = null;
    if (frequently()) {
        IngestStats.Stats totalStats = new IngestStats.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong());
        int numStatsPerPipeline = randomIntBetween(0, 10);
        Map<String, IngestStats.Stats> statsPerPipeline = new HashMap<>();
        for (int i = 0; i < numStatsPerPipeline; i++) {
            statsPerPipeline.put(randomAsciiOfLengthBetween(3, 10), new IngestStats.Stats(randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong(), randomNonNegativeLong()));
        }
        ingestStats = new IngestStats(totalStats, statsPerPipeline);
    }
    //TODO NodeIndicesStats are not tested here, way too complicated to create, also they need to be migrated to Writeable yet
    return new NodeStats(node, randomNonNegativeLong(), null, osStats, processStats, jvmStats, threadPoolStats, fsInfo, transportStats, httpStats, allCircuitBreakerStats, scriptStats, discoveryStats, ingestStats);
}

Example 4 with ProcessStats

use of org.elasticsearch.monitor.process.ProcessStats in project graylog2-server by Graylog2.

the class IndexerClusterCheckerThread method doRun.

@Override
public void doRun() {
    if (!notificationService.isFirst(Notification.Type.ES_OPEN_FILES)) {
        return;
    }
    try {
        cluster.health().getStatus();
    } catch (Exception e) {
        LOG.info("Indexer not fully initialized yet. Skipping periodic cluster check.");
        return;
    }
    boolean allHigher = true;
    final Map<String, NodeInfo> nodesInfos = cluster.getDataNodes();
    final Map<String, NodeStats> nodesStats = cluster.getNodesStats(nodesInfos.keySet().toArray(new String[nodesInfos.size()]));
    for (Map.Entry<String, NodeStats> entry : nodesStats.entrySet()) {
        final String nodeId = entry.getKey();
        final NodeStats nodeStats = entry.getValue();
        final NodeInfo nodeInfo = nodesInfos.get(nodeId);
        final String nodeName = nodeInfo.getNode().getName();
        // Check number of maximum open files.
        final ProcessStats processStats = nodeStats.getProcess();
        if (processStats == null) {
            LOG.debug("Couldn't read process stats of Elasticsearch node {}", nodeName);
            return;
        }
        final long maxFileDescriptors = processStats.getMaxFileDescriptors();
        final JvmInfo jvmInfo = nodeInfo.getJvm();
        if (jvmInfo == null) {
            LOG.debug("Couldn't read JVM info of Elasticsearch node {}", nodeName);
            return;
        }
        final String osName = jvmInfo.getSystemProperties().getOrDefault("os.name", "");
        if (osName.startsWith("Windows")) {
            LOG.debug("Skipping open file limit check for Indexer node <{}> on Windows", nodeName);
        } else if (maxFileDescriptors != -1 && maxFileDescriptors < MINIMUM_OPEN_FILES_LIMIT) {
            // Write notification.
            final Notification notification = notificationService.buildNow().addType(Notification.Type.ES_OPEN_FILES).addSeverity(Notification.Severity.URGENT).addDetail("hostname", nodeInfo.getHostname()).addDetail("max_file_descriptors", maxFileDescriptors);
            if (notificationService.publishIfFirst(notification)) {
                LOG.warn("Indexer node <{}> open file limit is too low: [{}]. Set it to at least {}.", nodeName, maxFileDescriptors, MINIMUM_OPEN_FILES_LIMIT);
            }
            allHigher = false;
        }
    }
    if (allHigher) {
        Notification notification = notificationService.build().addType(Notification.Type.ES_OPEN_FILES);
        notificationService.fixed(notification);
    }
}