Examples with GarbageCollectorMXBean java.lang.management.GarbageCollectorMXBean used on opensource projects

Example 16 with GarbageCollectorMXBean

use of java.lang.management.GarbageCollectorMXBean in project neo4j by neo4j.

the class GCMetrics method start.

@Override
public void start() {
    for (final GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) {
        registry.register(name(GC_TIME, prettifyName(gcBean.getName())), (Gauge<Long>) gcBean::getCollectionTime);
        registry.register(name(GC_COUNT, prettifyName(gcBean.getName())), (Gauge<Long>) gcBean::getCollectionCount);
    }
}

Example 17 with GarbageCollectorMXBean

use of java.lang.management.GarbageCollectorMXBean in project graphdb by neo4j-attic.

the class GraphDbInstance method start.

/**
     * Starts Neo4j with default configuration
     * @param graphDb The graph database service.
     *
     * @param storeDir path to directory where Neo4j store is located
     * @param create if true a new Neo4j store will be created if no store exist
     *            at <CODE>storeDir</CODE>
     * @param configuration parameters
     * @throws StartupFailedException if unable to start
     */
public synchronized Map<Object, Object> start(GraphDatabaseService graphDb, KernelExtensionLoader kernelExtensionLoader) {
    if (started) {
        throw new IllegalStateException("Neo4j instance already started");
    }
    Map<Object, Object> params = config.getParams();
    boolean useMemoryMapped = Boolean.parseBoolean((String) config.getInputParams().get(Config.USE_MEMORY_MAPPED_BUFFERS));
    boolean dumpToConsole = Boolean.parseBoolean((String) config.getInputParams().get(Config.DUMP_CONFIGURATION));
    storeDir = FileUtils.fixSeparatorsInPath(storeDir);
    StringLogger logger = StringLogger.getLogger(storeDir);
    AutoConfigurator autoConfigurator = new AutoConfigurator(storeDir, useMemoryMapped, dumpToConsole);
    autoConfigurator.configure(subset(config.getInputParams(), Config.USE_MEMORY_MAPPED_BUFFERS));
    // params.putAll( config.getInputParams() );
    String separator = System.getProperty("file.separator");
    String store = storeDir + separator + "neostore";
    params.put("store_dir", storeDir);
    params.put("neo_store", store);
    params.put("create", String.valueOf(create));
    String logicalLog = storeDir + separator + "nioneo_logical.log";
    params.put("logical_log", logicalLog);
    byte[] resourceId = UTF8.encode("414141");
    params.put(LockManager.class, config.getLockManager());
    params.put(LockReleaser.class, config.getLockReleaser());
    kernelExtensionLoader.configureKernelExtensions();
    config.getTxModule().registerDataSource(Config.DEFAULT_DATA_SOURCE_NAME, Config.NIO_NEO_DB_CLASS, resourceId, params);
    // hack for lucene index recovery if in path
    if (!config.isReadOnly() || config.isBackupSlave()) {
        try {
            Class clazz = Class.forName(Config.LUCENE_DS_CLASS);
            cleanWriteLocksInLuceneDirectory(storeDir + File.separator + "lucene");
            byte[] luceneId = UTF8.encode("162373");
            registerLuceneDataSource("lucene", clazz.getName(), config.getTxModule(), storeDir + File.separator + "lucene", config.getLockManager(), luceneId, params);
        } catch (ClassNotFoundException e) {
        // ok index util not on class path
        } catch (NoClassDefFoundError err) {
        // ok index util not on class path
        }
        try {
            Class clazz = Class.forName(Config.LUCENE_FULLTEXT_DS_CLASS);
            cleanWriteLocksInLuceneDirectory(storeDir + File.separator + "lucene-fulltext");
            byte[] luceneId = UTF8.encode("262374");
            registerLuceneDataSource("lucene-fulltext", clazz.getName(), config.getTxModule(), storeDir + File.separator + "lucene-fulltext", config.getLockManager(), luceneId, params);
        } catch (ClassNotFoundException e) {
        // ok index util not on class path
        } catch (NoClassDefFoundError err) {
        // ok index util not on class path
        }
    }
    persistenceSource = new NioNeoDbPersistenceSource();
    config.setPersistenceSource(Config.DEFAULT_DATA_SOURCE_NAME, create);
    config.getIdGeneratorModule().setPersistenceSourceInstance(persistenceSource);
    config.getTxModule().init();
    config.getPersistenceModule().init();
    persistenceSource.init();
    config.getIdGeneratorModule().init();
    config.getGraphDbModule().init();
    kernelExtensionLoader.initializeIndexProviders();
    config.getTxModule().start();
    config.getPersistenceModule().start(config.getTxModule().getTxManager(), persistenceSource, config.getSyncHookFactory());
    persistenceSource.start(config.getTxModule().getXaDataSourceManager());
    config.getIdGeneratorModule().start();
    config.getGraphDbModule().start(config.getLockReleaser(), config.getPersistenceModule().getPersistenceManager(), config.getRelationshipTypeCreator(), params);
    logger.logMessage("--- CONFIGURATION START ---");
    logger.logMessage(autoConfigurator.getNiceMemoryInformation());
    logger.logMessage("Kernel version: " + Version.getKernel());
    for (Version componentVersion : Service.load(Version.class)) {
        logger.logMessage(componentVersion.toString());
    }
    RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();
    OperatingSystemMXBean os = ManagementFactory.getOperatingSystemMXBean();
    logger.logMessage(String.format("Operating System: %s; version: %s; arch: %s; cpus: %s", os.getName(), os.getVersion(), os.getArch(), os.getAvailableProcessors()));
    logger.logMessage("VM Name: " + runtime.getVmName());
    logger.logMessage("VM Vendor: " + runtime.getVmVendor());
    logger.logMessage("VM Version: " + runtime.getVmVersion());
    if (runtime.isBootClassPathSupported()) {
        logger.logMessage("Boot Class Path: " + runtime.getBootClassPath());
    }
    logger.logMessage("Class Path: " + runtime.getClassPath());
    logger.logMessage("Library Path: " + runtime.getLibraryPath());
    for (GarbageCollectorMXBean gcBean : ManagementFactory.getGarbageCollectorMXBeans()) {
        logger.logMessage("Garbage Collector: " + gcBean.getName() + ": " + Arrays.toString(gcBean.getMemoryPoolNames()));
    }
    logger.logMessage("VM Arguments: " + runtime.getInputArguments());
    logger.logMessage("");
    logConfiguration(params, logger, dumpToConsole);
    logger.logMessage("--- CONFIGURATION END ---");
    logger.flush();
    started = true;
    return Collections.unmodifiableMap(params);
}

Example 18 with GarbageCollectorMXBean

use of java.lang.management.GarbageCollectorMXBean in project deeplearning4j by deeplearning4j.

the class BaseStatsListener method iterationDone.

@Override
public void iterationDone(Model model, int iteration) {
    StatsUpdateConfiguration config = updateConfig;
    ModelInfo modelInfo = getModelInfo(model);
    boolean backpropParamsOnly = backpropParamsOnly(model);
    long currentTime = getTime();
    if (modelInfo.iterCount == 0) {
        modelInfo.initTime = currentTime;
        doInit(model);
    }
    if (config.collectPerformanceStats()) {
        updateExamplesMinibatchesCounts(model);
    }
    if (config.reportingFrequency() > 1 && (iteration == 0 || iteration % config.reportingFrequency() != 0)) {
        modelInfo.iterCount = iteration;
        return;
    }
    StatsReport report = getNewStatsReport();
    //TODO support NTP time
    report.reportIDs(getSessionID(model), TYPE_ID, workerID, System.currentTimeMillis());
    //--- Performance and System Stats ---
    if (config.collectPerformanceStats()) {
        //Stats to collect: total runtime, total examples, total minibatches, iterations/second, examples/second
        double examplesPerSecond;
        double minibatchesPerSecond;
        if (modelInfo.iterCount == 0) {
            //Not possible to work out perf/second: first iteration...
            examplesPerSecond = 0.0;
            minibatchesPerSecond = 0.0;
        } else {
            long deltaTimeMS = currentTime - modelInfo.lastReportTime;
            examplesPerSecond = 1000.0 * modelInfo.examplesSinceLastReport / deltaTimeMS;
            minibatchesPerSecond = 1000.0 * modelInfo.minibatchesSinceLastReport / deltaTimeMS;
        }
        long totalRuntimeMS = currentTime - modelInfo.initTime;
        report.reportPerformance(totalRuntimeMS, modelInfo.totalExamples, modelInfo.totalMinibatches, examplesPerSecond, minibatchesPerSecond);
        modelInfo.examplesSinceLastReport = 0;
        modelInfo.minibatchesSinceLastReport = 0;
    }
    if (config.collectMemoryStats()) {
        Runtime runtime = Runtime.getRuntime();
        long jvmTotal = runtime.totalMemory();
        long jvmMax = runtime.maxMemory();
        //Off-heap memory
        long offheapTotal = Pointer.totalBytes();
        long offheapMax = Pointer.maxBytes();
        //GPU
        long[] gpuCurrentBytes = null;
        long[] gpuMaxBytes = null;
        NativeOps nativeOps = NativeOpsHolder.getInstance().getDeviceNativeOps();
        int nDevices = nativeOps.getAvailableDevices();
        if (nDevices > 0) {
            gpuCurrentBytes = new long[nDevices];
            gpuMaxBytes = new long[nDevices];
            for (int i = 0; i < nDevices; i++) {
                try {
                    Pointer p = getDevicePointer(i);
                    if (p == null) {
                        gpuMaxBytes[i] = 0;
                        gpuCurrentBytes[i] = 0;
                    } else {
                        gpuMaxBytes[i] = nativeOps.getDeviceTotalMemory(p);
                        gpuCurrentBytes[i] = gpuMaxBytes[i] - nativeOps.getDeviceFreeMemory(p);
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
        report.reportMemoryUse(jvmTotal, jvmMax, offheapTotal, offheapMax, gpuCurrentBytes, gpuMaxBytes);
    }
    if (config.collectGarbageCollectionStats()) {
        if (modelInfo.lastReportIteration == -1 || gcBeans == null) {
            //Haven't reported GC stats before...
            gcBeans = ManagementFactory.getGarbageCollectorMXBeans();
            gcStatsAtLastReport = new HashMap<>();
            for (GarbageCollectorMXBean bean : gcBeans) {
                long count = bean.getCollectionCount();
                long timeMs = bean.getCollectionTime();
                gcStatsAtLastReport.put(bean.getName(), new Pair<>(count, timeMs));
            }
        } else {
            for (GarbageCollectorMXBean bean : gcBeans) {
                long count = bean.getCollectionCount();
                long timeMs = bean.getCollectionTime();
                Pair<Long, Long> lastStats = gcStatsAtLastReport.get(bean.getName());
                long deltaGCCount = count - lastStats.getFirst();
                long deltaGCTime = timeMs - lastStats.getSecond();
                lastStats.setFirst(count);
                lastStats.setSecond(timeMs);
                report.reportGarbageCollection(bean.getName(), (int) deltaGCCount, (int) deltaGCTime);
            }
        }
    }
    //--- General ---
    //Always report score
    report.reportScore(model.score());
    if (config.collectLearningRates()) {
        Map<String, Double> lrs = new HashMap<>();
        if (model instanceof MultiLayerNetwork) {
            //Need to append "0_", "1_" etc to param names from layers...
            int layerIdx = 0;
            for (Layer l : ((MultiLayerNetwork) model).getLayers()) {
                NeuralNetConfiguration conf = l.conf();
                Map<String, Double> layerLrs = conf.getLearningRateByParam();
                Set<String> backpropParams = l.paramTable(true).keySet();
                for (Map.Entry<String, Double> entry : layerLrs.entrySet()) {
                    if (!backpropParams.contains(entry.getKey()))
                        //Skip pretrain params
                        continue;
                    lrs.put(layerIdx + "_" + entry.getKey(), entry.getValue());
                }
                layerIdx++;
            }
        } else if (model instanceof ComputationGraph) {
            for (Layer l : ((ComputationGraph) model).getLayers()) {
                //Need to append layer name
                NeuralNetConfiguration conf = l.conf();
                Map<String, Double> layerLrs = conf.getLearningRateByParam();
                String layerName = conf.getLayer().getLayerName();
                Set<String> backpropParams = l.paramTable(true).keySet();
                for (Map.Entry<String, Double> entry : layerLrs.entrySet()) {
                    if (!backpropParams.contains(entry.getKey()))
                        //Skip pretrain params
                        continue;
                    lrs.put(layerName + "_" + entry.getKey(), entry.getValue());
                }
            }
        } else if (model instanceof Layer) {
            Layer l = (Layer) model;
            Map<String, Double> map = l.conf().getLearningRateByParam();
            lrs.putAll(map);
        }
        report.reportLearningRates(lrs);
    }
    if (config.collectHistograms(StatsType.Parameters)) {
        Map<String, Histogram> paramHistograms = getHistograms(model.paramTable(backpropParamsOnly), config.numHistogramBins(StatsType.Parameters));
        report.reportHistograms(StatsType.Parameters, paramHistograms);
    }
    if (config.collectHistograms(StatsType.Gradients)) {
        Map<String, Histogram> gradientHistograms = getHistograms(gradientsPreUpdateMap, config.numHistogramBins(StatsType.Gradients));
        report.reportHistograms(StatsType.Gradients, gradientHistograms);
    }
    if (config.collectHistograms(StatsType.Updates)) {
        Map<String, Histogram> updateHistograms = getHistograms(model.gradient().gradientForVariable(), config.numHistogramBins(StatsType.Updates));
        report.reportHistograms(StatsType.Updates, updateHistograms);
    }
    if (config.collectHistograms(StatsType.Activations)) {
        Map<String, Histogram> activationHistograms = getHistograms(activationsMap, config.numHistogramBins(StatsType.Activations));
        report.reportHistograms(StatsType.Activations, activationHistograms);
    }
    if (config.collectMean(StatsType.Parameters)) {
        Map<String, Double> meanParams = calculateSummaryStats(model.paramTable(backpropParamsOnly), StatType.Mean);
        report.reportMean(StatsType.Parameters, meanParams);
    }
    if (config.collectMean(StatsType.Gradients)) {
        Map<String, Double> meanGradients = calculateSummaryStats(gradientsPreUpdateMap, StatType.Mean);
        report.reportMean(StatsType.Gradients, meanGradients);
    }
    if (config.collectMean(StatsType.Updates)) {
        Map<String, Double> meanUpdates = calculateSummaryStats(model.gradient().gradientForVariable(), StatType.Mean);
        report.reportMean(StatsType.Updates, meanUpdates);
    }
    if (config.collectMean(StatsType.Activations)) {
        Map<String, Double> meanActivations = calculateSummaryStats(activationsMap, StatType.Mean);
        report.reportMean(StatsType.Activations, meanActivations);
    }
    if (config.collectStdev(StatsType.Parameters)) {
        Map<String, Double> stdevParams = calculateSummaryStats(model.paramTable(backpropParamsOnly), StatType.Stdev);
        report.reportStdev(StatsType.Parameters, stdevParams);
    }
    if (config.collectStdev(StatsType.Gradients)) {
        Map<String, Double> stdevGradient = calculateSummaryStats(gradientsPreUpdateMap, StatType.Stdev);
        report.reportStdev(StatsType.Gradients, stdevGradient);
    }
    if (config.collectStdev(StatsType.Updates)) {
        Map<String, Double> stdevUpdates = calculateSummaryStats(model.gradient().gradientForVariable(), StatType.Stdev);
        report.reportStdev(StatsType.Updates, stdevUpdates);
    }
    if (config.collectStdev(StatsType.Activations)) {
        Map<String, Double> stdevActivations = calculateSummaryStats(activationsMap, StatType.Stdev);
        report.reportStdev(StatsType.Activations, stdevActivations);
    }
    if (config.collectMeanMagnitudes(StatsType.Parameters)) {
        Map<String, Double> meanMagParams = calculateSummaryStats(model.paramTable(backpropParamsOnly), StatType.MeanMagnitude);
        report.reportMeanMagnitudes(StatsType.Parameters, meanMagParams);
    }
    if (config.collectMeanMagnitudes(StatsType.Gradients)) {
        Map<String, Double> meanMagGradients = calculateSummaryStats(gradientsPreUpdateMap, StatType.MeanMagnitude);
        report.reportMeanMagnitudes(StatsType.Gradients, meanMagGradients);
    }
    if (config.collectMeanMagnitudes(StatsType.Updates)) {
        Map<String, Double> meanMagUpdates = calculateSummaryStats(model.gradient().gradientForVariable(), StatType.MeanMagnitude);
        report.reportMeanMagnitudes(StatsType.Updates, meanMagUpdates);
    }
    if (config.collectMeanMagnitudes(StatsType.Activations)) {
        Map<String, Double> meanMagActivations = calculateSummaryStats(activationsMap, StatType.MeanMagnitude);
        report.reportMeanMagnitudes(StatsType.Activations, meanMagActivations);
    }
    long endTime = getTime();
    //Amount of time required to alculate all histograms, means etc.
    report.reportStatsCollectionDurationMS((int) (endTime - currentTime));
    modelInfo.lastReportTime = currentTime;
    modelInfo.lastReportIteration = iteration;
    report.reportIterationCount(iteration);
    this.router.putUpdate(report);
    modelInfo.iterCount = iteration;
    activationsMap = null;
}

Example 19 with GarbageCollectorMXBean

use of java.lang.management.GarbageCollectorMXBean in project jdk8u_jdk by JetBrains.

the class MXBeanInteropTest1 method doGarbageCollectorMXBeanTest.

private final int doGarbageCollectorMXBeanTest(MBeanServerConnection mbsc) {
    int errorCount = 0;
    System.out.println("---- GarbageCollectorMXBean");
    try {
        ObjectName filterName = new ObjectName(ManagementFactory.GARBAGE_COLLECTOR_MXBEAN_DOMAIN_TYPE + ",*");
        Set<ObjectName> onSet = mbsc.queryNames(filterName, null);
        for (Iterator<ObjectName> iter = onSet.iterator(); iter.hasNext(); ) {
            ObjectName garbageName = iter.next();
            System.out.println("-------- " + garbageName);
            MBeanInfo mbInfo = mbsc.getMBeanInfo(garbageName);
            errorCount += checkNonEmpty(mbInfo);
            System.out.println("getMBeanInfo\t\t" + mbInfo);
            GarbageCollectorMXBean garbage = null;
            garbage = JMX.newMXBeanProxy(mbsc, garbageName, GarbageCollectorMXBean.class);
            System.out.println("getCollectionCount\t\t" + garbage.getCollectionCount());
            System.out.println("getCollectionTime\t\t" + garbage.getCollectionTime());
        }
        System.out.println("---- OK\n");
    } catch (Exception e) {
        Utils.printThrowable(e, true);
        errorCount++;
        System.out.println("---- ERROR\n");
    }
    return errorCount;
}

Example 20 with GarbageCollectorMXBean

use of java.lang.management.GarbageCollectorMXBean in project opentsdb by OpenTSDB.

the class StatsRpc method printJVMStats.

/**
   * Yield (chiefly memory-related) stats about this OpenTSDB instance's JVM.
   * @param tsdb The TSDB from which to fetch stats.
   * @param query The query to which to respond.
   */
private void printJVMStats(final TSDB tsdb, final HttpQuery query) {
    final Map<String, Map<String, Object>> map = new HashMap<String, Map<String, Object>>();
    final RuntimeMXBean runtime_bean = ManagementFactory.getRuntimeMXBean();
    final Map<String, Object> runtime = new HashMap<String, Object>();
    map.put("runtime", runtime);
    runtime.put("startTime", runtime_bean.getStartTime());
    runtime.put("uptime", runtime_bean.getUptime());
    runtime.put("vmName", runtime_bean.getVmName());
    runtime.put("vmVendor", runtime_bean.getVmVendor());
    runtime.put("vmVersion", runtime_bean.getVmVersion());
    final MemoryMXBean mem_bean = ManagementFactory.getMemoryMXBean();
    final Map<String, Object> memory = new HashMap<String, Object>();
    map.put("memory", memory);
    memory.put("heapMemoryUsage", mem_bean.getHeapMemoryUsage());
    memory.put("nonHeapMemoryUsage", mem_bean.getNonHeapMemoryUsage());
    memory.put("objectsPendingFinalization", mem_bean.getObjectPendingFinalizationCount());
    final List<GarbageCollectorMXBean> gc_beans = ManagementFactory.getGarbageCollectorMXBeans();
    final Map<String, Object> gc = new HashMap<String, Object>();
    map.put("gc", gc);
    for (final GarbageCollectorMXBean gc_bean : gc_beans) {
        final Map<String, Object> stats = new HashMap<String, Object>();
        final String name = formatStatName(gc_bean.getName());
        if (name == null) {
            LOG.warn("Null name for bean: " + gc_bean);
            continue;
        }
        gc.put(name, stats);
        stats.put("collectionCount", gc_bean.getCollectionCount());
        stats.put("collectionTime", gc_bean.getCollectionTime());
    }
    final List<MemoryPoolMXBean> pool_beans = ManagementFactory.getMemoryPoolMXBeans();
    final Map<String, Object> pools = new HashMap<String, Object>();
    map.put("pools", pools);
    for (final MemoryPoolMXBean pool_bean : pool_beans) {
        final Map<String, Object> stats = new HashMap<String, Object>();
        final String name = formatStatName(pool_bean.getName());
        if (name == null) {
            LOG.warn("Null name for bean: " + pool_bean);
            continue;
        }
        pools.put(name, stats);
        stats.put("collectionUsage", pool_bean.getCollectionUsage());
        stats.put("usage", pool_bean.getUsage());
        stats.put("peakUsage", pool_bean.getPeakUsage());
        stats.put("type", pool_bean.getType());
    }
    final OperatingSystemMXBean os_bean = ManagementFactory.getOperatingSystemMXBean();
    final Map<String, Object> os = new HashMap<String, Object>();
    map.put("os", os);
    os.put("systemLoadAverage", os_bean.getSystemLoadAverage());
    query.sendReply(query.serializer().formatJVMStatsV1(map));
}