Examples with PoolSubpageMetric io.netty.buffer.PoolSubpageMetric used on opensource projects

Example 1 with PoolSubpageMetric

use of io.netty.buffer.PoolSubpageMetric in project Mycat_plus by coderczp.

the class ShowDirectMemory method showDirectMemoryTotal.

public static void showDirectMemoryTotal(ManagerConnection c) {
    ByteBuffer buffer = c.allocate();
    // write header
    buffer = totalHeader.write(buffer, c, true);
    // write fields
    for (FieldPacket field : totalFields) {
        buffer = field.write(buffer, c, true);
    }
    // write eof
    buffer = totalEof.write(buffer, c, true);
    // write rows
    byte packetId = totalEof.packetId;
    long usedforMerge = 0;
    long usedforNetwork = 0;
    long chunkSizeBytes = 0;
    int chunkCount = 0;
    if (processorBufferPoolType == 2 && bufferPool instanceof NettyBufferPool) {
        /**
         *计算逻辑就是，1.先计算PoolChunk分配的页,表示已经消耗的内存，
         *             2.然后计算小于一页情况，记录小于一页内存使用情况，
         *             上面二者合起来就是整个netty 使用的内存，
         *             已经分配了，但是没有使用的内存的情况
         */
        List<PoolArenaMetric> list = ((NettyBufferPool) bufferPool).getAllocator().getAlloc().directArenas();
        chunkSizeBytes = ((NettyBufferPool) bufferPool).getAllocator().getChunkSize();
        long pageSize = ((NettyBufferPool) bufferPool).getAllocator().getPageSize();
        long chunksUsedBytes = 0;
        /**
         *PoolArenas
         */
        for (PoolArenaMetric pool : list) {
            List<PoolChunkListMetric> pcks = pool.chunkLists();
            /**
             *针对PoolChunkList
             */
            for (PoolChunkListMetric pck : pcks) {
                Iterator<PoolChunkMetric> it = pck.iterator();
                while (it.hasNext()) {
                    chunkCount++;
                    PoolChunkMetric p = it.next();
                    chunksUsedBytes += (chunkSizeBytes - p.freeBytes());
                }
            }
            List<PoolSubpageMetric> tinySubpages = pool.tinySubpages();
            for (PoolSubpageMetric tiny : tinySubpages) {
                chunksUsedBytes -= (pageSize - (tiny.maxNumElements() - tiny.numAvailable()) * tiny.elementSize());
            }
            List<PoolSubpageMetric> smallSubpages = pool.smallSubpages();
            for (PoolSubpageMetric small : smallSubpages) {
                chunksUsedBytes -= (pageSize - (small.maxNumElements() - small.numAvailable()) * small.elementSize());
            }
        }
        usedforNetwork = chunkCount * chunkSizeBytes;
    }
    ConcurrentHashMap<Long, Long> bufferpoolUsageMap = bufferPool.getNetDirectMemoryUsage();
    RowDataPacket row = new RowDataPacket(TOTAL_FIELD_COUNT);
    try {
        /**
         * 通过-XX:MaxDirectMemorySize=2048m设置的值
         */
        row.add(JavaUtils.bytesToString2(Platform.getMaxDirectMemory()).getBytes(c.getCharset()));
        if (useOffHeapForMerge == 1) {
            /**
             * 结果集合并时，总共消耗的DirectMemory内存
             */
            ConcurrentHashMap<Long, Long> concurrentHashMap = MycatServer.getInstance().getMyCatMemory().getResultMergeMemoryManager().getDirectMemorUsage();
            for (Map.Entry<Long, Long> entry : concurrentHashMap.entrySet()) {
                usedforMerge += entry.getValue();
            }
        }
        /**
         * 网络packet处理，在buffer pool 已经使用DirectMemory内存
         */
        if (processorBufferPoolType == 2) {
            usedforNetwork = chunkSizeBytes * chunkCount;
        } else {
            for (Map.Entry<Long, Long> entry : bufferpoolUsageMap.entrySet()) {
                usedforNetwork += entry.getValue();
            }
        }
        row.add(JavaUtils.bytesToString2(usedforMerge + usedforNetwork).getBytes(c.getCharset()));
        long totalAvailable = 0;
        if (useOffHeapForMerge == 1) {
            /**
             * 设置使用off-heap内存处理结果集时，防止客户把MaxDirectMemorySize设置到物理内存的极限。
             * Mycat能使用的DirectMemory是MaxDirectMemorySize*DIRECT_SAFETY_FRACTION大小，
             * DIRECT_SAFETY_FRACTION为安全系数，为OS，Heap预留空间，避免因大结果集造成系统物理内存被耗尽！
             */
            totalAvailable = (long) (Platform.getMaxDirectMemory() * MyCatMemory.DIRECT_SAFETY_FRACTION);
        } else {
            totalAvailable = Platform.getMaxDirectMemory();
        }
        row.add(JavaUtils.bytesToString2(totalAvailable - usedforMerge - usedforNetwork).getBytes(c.getCharset()));
        if (useOffHeapForMerge == 1) {
            /**
             * 输出安全系统DIRECT_SAFETY_FRACTION
             */
            row.add(("" + MyCatMemory.DIRECT_SAFETY_FRACTION).getBytes(c.getCharset()));
        } else {
            row.add(("1.0").getBytes(c.getCharset()));
        }
        long resevedForOs = 0;
        if (useOffHeapForMerge == 1) {
            /**
             * 预留OS系统部分内存！！！
             */
            resevedForOs = (long) ((1 - MyCatMemory.DIRECT_SAFETY_FRACTION) * (Platform.getMaxDirectMemory() - 2 * MycatServer.getInstance().getTotalNetWorkBufferSize()));
        }
        row.add(resevedForOs > 0 ? JavaUtils.bytesToString2(resevedForOs).getBytes(c.getCharset()) : "0".getBytes(c.getCharset()));
        row.packetId = ++packetId;
        buffer = row.write(buffer, c, true);
    } catch (UnsupportedEncodingException e) {
        e.printStackTrace();
    }
    // write last eof
    EOFPacket lastEof = new EOFPacket();
    lastEof.packetId = ++packetId;
    buffer = lastEof.write(buffer, c, true);
    // write buffer
    c.write(buffer);
}

Example 2 with PoolSubpageMetric

use of io.netty.buffer.PoolSubpageMetric in project jocean-http by isdom.

the class PooledAllocatorStats method metricsOfSubpages.

private static Map<String, Object> metricsOfSubpages(final int count, final List<PoolSubpageMetric> subpages) {
    final Map<String, Object> metrics = new HashMap<>();
    final int w = (int) (Math.log10(count)) + 1;
    final String fstr = "%0" + w + "d";
    int idx = 0;
    for (PoolSubpageMetric subpageMetric : subpages) {
        metrics.put(String.format(fstr, idx++) + "_tinySubpage", metricsOfPoolSubpage(subpageMetric));
    }
    return metrics;
}

Example 3 with PoolSubpageMetric

use of io.netty.buffer.PoolSubpageMetric in project ratpack by ratpack.

the class PooledByteBufAllocatorMetricSet method initSubpageMetrics.

private void initSubpageMetrics(final String prefix, final String subpageName, final Iterator<PoolSubpageMetric> subpagesIterator) {
    int counter = 0;
    while (subpagesIterator.hasNext()) {
        final PoolSubpageMetric poolSubpageMetric = subpagesIterator.next();
        final String subpagePrefix = format("%s.%s", subpageName, counter);
        metrics.put(format("%s.%s.elementSize", prefix, subpagePrefix), (Gauge<Integer>) () -> poolSubpageMetric.elementSize());
        metrics.put(format("%s.%s.maxNumElements", prefix, subpagePrefix), (Gauge<Integer>) () -> poolSubpageMetric.maxNumElements());
        metrics.put(format("%s.%s.numAvailable", prefix, subpagePrefix), (Gauge<Integer>) () -> poolSubpageMetric.numAvailable());
        metrics.put(format("%s.%s.pageSize", prefix, subpagePrefix), (Gauge<Integer>) () -> poolSubpageMetric.pageSize());
        counter++;
    }
}

Example 4 with PoolSubpageMetric

use of io.netty.buffer.PoolSubpageMetric in project ratpack by ratpack.

the class PooledByteBufAllocatorMetricSet method initPoolArenaMetrics.

private void initPoolArenaMetrics(final Iterator<PoolArenaMetric> poolArenasIterator) {
    int counter = 0;
    while (poolArenasIterator.hasNext()) {
        final PoolArenaMetric poolArenaMetric = poolArenasIterator.next();
        final String prefix = format("poolArena.%s", counter);
        metrics.put(format("%s.activeAllocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numActiveAllocations());
        metrics.put(format("%s.numActiveBytes", prefix), (Gauge<Long>) () -> poolArenaMetric.numActiveBytes());
        metrics.put(format("%s.numActiveHugeAllocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numActiveHugeAllocations());
        metrics.put(format("%s.numActiveNormalAllocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numActiveNormalAllocations());
        metrics.put(format("%s.numActiveSmallAllocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numActiveSmallAllocations());
        metrics.put(format("%s.numActiveTinyAllocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numActiveTinyAllocations());
        metrics.put(format("%s.numAllocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numAllocations());
        metrics.put(format("%s.numDeallocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numDeallocations());
        metrics.put(format("%s.numHugeAllocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numHugeAllocations());
        metrics.put(format("%s.numHugeDeallocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numHugeDeallocations());
        metrics.put(format("%s.numNormalAllocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numNormalAllocations());
        metrics.put(format("%s.numNormalDeallocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numNormalDeallocations());
        metrics.put(format("%s.numSmallAllocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numSmallAllocations());
        metrics.put(format("%s.numSmallDeallocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numSmallDeallocations());
        metrics.put(format("%s.numTinyAllocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numTinyAllocations());
        metrics.put(format("%s.numTinyDeallocations", prefix), (Gauge<Long>) () -> poolArenaMetric.numTinyDeallocations());
        metrics.put(format("%s.numSmallSubpages", prefix), (Gauge<Integer>) () -> poolArenaMetric.numSmallSubpages());
        metrics.put(format("%s.numTinySubpages", prefix), (Gauge<Integer>) () -> poolArenaMetric.numTinySubpages());
        final Iterator<PoolChunkListMetric> chunksIterator = poolArenaMetric.chunkLists().iterator();
        initPoolChunkListMetrics(prefix, chunksIterator);
        final Iterator<PoolSubpageMetric> smallSubpagesIterator = poolArenaMetric.smallSubpages().iterator();
        initSubpageMetrics(prefix, "smallSubpage", smallSubpagesIterator);
        final Iterator<PoolSubpageMetric> tinySubpagesIterator = poolArenaMetric.tinySubpages().iterator();
        initSubpageMetrics(prefix, "tinySubpage", tinySubpagesIterator);
        counter++;
    }
}

Example 5 with PoolSubpageMetric

use of io.netty.buffer.PoolSubpageMetric in project Mycat-Server by MyCATApache.

the class ShowDirectMemory method showDirectMemoryTotal.

public static void showDirectMemoryTotal(ManagerConnection c) {
    ByteBuffer buffer = c.allocate();
    // write header
    buffer = totalHeader.write(buffer, c, true);
    // write fields
    for (FieldPacket field : totalFields) {
        buffer = field.write(buffer, c, true);
    }
    // write eof
    buffer = totalEof.write(buffer, c, true);
    // write rows
    byte packetId = totalEof.packetId;
    long usedforMerge = 0;
    long usedforNetwork = 0;
    long chunkSizeBytes = 0;
    int chunkCount = 0;
    if (processorBufferPoolType == 2 && bufferPool instanceof NettyBufferPool) {
        /**
         *计算逻辑就是，1.先计算PoolChunk分配的页,表示已经消耗的内存，
         *             2.然后计算小于一页情况，记录小于一页内存使用情况，
         *             上面二者合起来就是整个netty 使用的内存，
         *             已经分配了，但是没有使用的内存的情况
         */
        List<PoolArenaMetric> list = ((NettyBufferPool) bufferPool).getAllocator().getAlloc().directArenas();
        chunkSizeBytes = ((NettyBufferPool) bufferPool).getAllocator().getChunkSize();
        long pageSize = ((NettyBufferPool) bufferPool).getAllocator().getPageSize();
        long chunksUsedBytes = 0;
        /**
         *PoolArenas
         */
        for (PoolArenaMetric pool : list) {
            List<PoolChunkListMetric> pcks = pool.chunkLists();
            /**
             *针对PoolChunkList
             */
            for (PoolChunkListMetric pck : pcks) {
                Iterator<PoolChunkMetric> it = pck.iterator();
                while (it.hasNext()) {
                    chunkCount++;
                    PoolChunkMetric p = it.next();
                    chunksUsedBytes += (chunkSizeBytes - p.freeBytes());
                }
            }
            List<PoolSubpageMetric> tinySubpages = pool.tinySubpages();
            for (PoolSubpageMetric tiny : tinySubpages) {
                chunksUsedBytes -= (pageSize - (tiny.maxNumElements() - tiny.numAvailable()) * tiny.elementSize());
            }
            List<PoolSubpageMetric> smallSubpages = pool.smallSubpages();
            for (PoolSubpageMetric small : smallSubpages) {
                chunksUsedBytes -= (pageSize - (small.maxNumElements() - small.numAvailable()) * small.elementSize());
            }
        }
        usedforNetwork = chunkCount * chunkSizeBytes;
    }
    ConcurrentHashMap<Long, Long> bufferpoolUsageMap = bufferPool.getNetDirectMemoryUsage();
    RowDataPacket row = new RowDataPacket(TOTAL_FIELD_COUNT);
    try {
        /**
         * 通过-XX:MaxDirectMemorySize=2048m设置的值
         */
        row.add(JavaUtils.bytesToString2(Platform.getMaxDirectMemory()).getBytes(c.getCharset()));
        if (useOffHeapForMerge == 1) {
            /**
             * 结果集合并时，总共消耗的DirectMemory内存
             */
            ConcurrentHashMap<Long, Long> concurrentHashMap = MycatServer.getInstance().getMyCatMemory().getResultMergeMemoryManager().getDirectMemorUsage();
            for (Map.Entry<Long, Long> entry : concurrentHashMap.entrySet()) {
                usedforMerge += entry.getValue();
            }
        }
        /**
         * 网络packet处理，在buffer pool 已经使用DirectMemory内存
         */
        if (processorBufferPoolType == 2) {
            usedforNetwork = chunkSizeBytes * chunkCount;
        } else {
            for (Map.Entry<Long, Long> entry : bufferpoolUsageMap.entrySet()) {
                usedforNetwork += entry.getValue();
            }
        }
        row.add(JavaUtils.bytesToString2(usedforMerge + usedforNetwork).getBytes(c.getCharset()));
        long totalAvailable = 0;
        if (useOffHeapForMerge == 1) {
            /**
             * 设置使用off-heap内存处理结果集时，防止客户把MaxDirectMemorySize设置到物理内存的极限。
             * Mycat能使用的DirectMemory是MaxDirectMemorySize*DIRECT_SAFETY_FRACTION大小，
             * DIRECT_SAFETY_FRACTION为安全系数，为OS，Heap预留空间，避免因大结果集造成系统物理内存被耗尽！
             */
            totalAvailable = (long) (Platform.getMaxDirectMemory() * MyCatMemory.DIRECT_SAFETY_FRACTION);
        } else {
            totalAvailable = Platform.getMaxDirectMemory();
        }
        row.add(JavaUtils.bytesToString2(totalAvailable - usedforMerge - usedforNetwork).getBytes(c.getCharset()));
        if (useOffHeapForMerge == 1) {
            /**
             * 输出安全系统DIRECT_SAFETY_FRACTION
             */
            row.add(("" + MyCatMemory.DIRECT_SAFETY_FRACTION).getBytes(c.getCharset()));
        } else {
            row.add(("1.0").getBytes(c.getCharset()));
        }
        long resevedForOs = 0;
        if (useOffHeapForMerge == 1) {
            /**
             * 预留OS系统部分内存！！！
             */
            resevedForOs = (long) ((1 - MyCatMemory.DIRECT_SAFETY_FRACTION) * (Platform.getMaxDirectMemory() - 2 * MycatServer.getInstance().getTotalNetWorkBufferSize()));
        }
        row.add(resevedForOs > 0 ? JavaUtils.bytesToString2(resevedForOs).getBytes(c.getCharset()) : "0".getBytes(c.getCharset()));
        row.packetId = ++packetId;
        buffer = row.write(buffer, c, true);
    } catch (UnsupportedEncodingException e) {
        e.printStackTrace();
    }
    // write last eof
    EOFPacket lastEof = new EOFPacket();
    lastEof.packetId = ++packetId;
    buffer = lastEof.write(buffer, c, true);
    // write buffer
    c.write(buffer);
}