Examples with MemtableAllocator org.apache.cassandra.utils.memory.MemtableAllocator used on opensource projects

Example 1 with MemtableAllocator

use of org.apache.cassandra.utils.memory.MemtableAllocator in project cassandra by apache.

the class ColumnFamilyStore method logFlush.

// print out size of all memtables we're enqueuing
private void logFlush() {
    // reclaiming includes that which we are GC-ing;
    float onHeapRatio = 0, offHeapRatio = 0;
    long onHeapTotal = 0, offHeapTotal = 0;
    Memtable memtable = getTracker().getView().getCurrentMemtable();
    onHeapRatio += memtable.getAllocator().onHeap().ownershipRatio();
    offHeapRatio += memtable.getAllocator().offHeap().ownershipRatio();
    onHeapTotal += memtable.getAllocator().onHeap().owns();
    offHeapTotal += memtable.getAllocator().offHeap().owns();
    for (ColumnFamilyStore indexCfs : indexManager.getAllIndexColumnFamilyStores()) {
        MemtableAllocator allocator = indexCfs.getTracker().getView().getCurrentMemtable().getAllocator();
        onHeapRatio += allocator.onHeap().ownershipRatio();
        offHeapRatio += allocator.offHeap().ownershipRatio();
        onHeapTotal += allocator.onHeap().owns();
        offHeapTotal += allocator.offHeap().owns();
    }
    logger.info("Enqueuing flush of {}: {}", name, String.format("%s (%.0f%%) on-heap, %s (%.0f%%) off-heap", FBUtilities.prettyPrintMemory(onHeapTotal), onHeapRatio * 100, FBUtilities.prettyPrintMemory(offHeapTotal), offHeapRatio * 100));
}

Example 2 with MemtableAllocator

use of org.apache.cassandra.utils.memory.MemtableAllocator in project cassandra by apache.

the class ColumnFamilyStore method flushLargestMemtable.

/**
 * Finds the largest memtable, as a percentage of *either* on- or off-heap memory limits, and immediately
 * queues it for flushing. If the memtable selected is flushed before this completes, no work is done.
 */
public static Future<Boolean> flushLargestMemtable() {
    float largestRatio = 0f;
    Memtable largest = null;
    float liveOnHeap = 0, liveOffHeap = 0;
    for (ColumnFamilyStore cfs : ColumnFamilyStore.all()) {
        // we take a reference to the current main memtable for the CF prior to snapping its ownership ratios
        // to ensure we have some ordering guarantee for performing the switchMemtableIf(), i.e. we will only
        // swap if the memtables we are measuring here haven't already been swapped by the time we try to swap them
        Memtable current = cfs.getTracker().getView().getCurrentMemtable();
        // find the total ownership ratio for the memtable and all SecondaryIndexes owned by this CF,
        // both on- and off-heap, and select the largest of the two ratios to weight this CF
        float onHeap = 0f, offHeap = 0f;
        onHeap += current.getAllocator().onHeap().ownershipRatio();
        offHeap += current.getAllocator().offHeap().ownershipRatio();
        for (ColumnFamilyStore indexCfs : cfs.indexManager.getAllIndexColumnFamilyStores()) {
            MemtableAllocator allocator = indexCfs.getTracker().getView().getCurrentMemtable().getAllocator();
            onHeap += allocator.onHeap().ownershipRatio();
            offHeap += allocator.offHeap().ownershipRatio();
        }
        float ratio = Math.max(onHeap, offHeap);
        if (ratio > largestRatio) {
            largest = current;
            largestRatio = ratio;
        }
        liveOnHeap += onHeap;
        liveOffHeap += offHeap;
    }
    Promise<Boolean> returnFuture = new AsyncPromise<>();
    if (largest != null) {
        float usedOnHeap = Memtable.MEMORY_POOL.onHeap.usedRatio();
        float usedOffHeap = Memtable.MEMORY_POOL.offHeap.usedRatio();
        float flushingOnHeap = Memtable.MEMORY_POOL.onHeap.reclaimingRatio();
        float flushingOffHeap = Memtable.MEMORY_POOL.offHeap.reclaimingRatio();
        float thisOnHeap = largest.getAllocator().onHeap().ownershipRatio();
        float thisOffHeap = largest.getAllocator().offHeap().ownershipRatio();
        logger.debug("Flushing largest {} to free up room. Used total: {}, live: {}, flushing: {}, this: {}", largest.cfs, ratio(usedOnHeap, usedOffHeap), ratio(liveOnHeap, liveOffHeap), ratio(flushingOnHeap, flushingOffHeap), ratio(thisOnHeap, thisOffHeap));
        Future<CommitLogPosition> flushFuture = largest.cfs.switchMemtableIfCurrent(largest);
        flushFuture.addListener(() -> {
            try {
                flushFuture.get();
                returnFuture.trySuccess(true);
            } catch (Throwable t) {
                returnFuture.tryFailure(t);
            }
        });
    } else {
        logger.debug("Flushing of largest memtable, not done, no memtable found");
        returnFuture.trySuccess(false);
    }
    return returnFuture;
}

Example 3 with MemtableAllocator

use of org.apache.cassandra.utils.memory.MemtableAllocator in project cassandra by apache.

the class Memtable method estimateRowOverhead.

private static int estimateRowOverhead(final int count) {
    // calculate row overhead
    try (final OpOrder.Group group = new OpOrder().start()) {
        int rowOverhead;
        MemtableAllocator allocator = MEMORY_POOL.newAllocator(null);
        ConcurrentNavigableMap<PartitionPosition, Object> partitions = new ConcurrentSkipListMap<>();
        final Object val = new Object();
        for (int i = 0; i < count; i++) partitions.put(allocator.clone(new BufferDecoratedKey(new LongToken(i), ByteBufferUtil.EMPTY_BYTE_BUFFER), group), val);
        double avgSize = ObjectSizes.measureDeep(partitions) / (double) count;
        rowOverhead = (int) ((avgSize - Math.floor(avgSize)) < 0.05 ? Math.floor(avgSize) : Math.ceil(avgSize));
        rowOverhead -= ObjectSizes.measureDeep(new LongToken(0));
        rowOverhead += AtomicBTreePartition.EMPTY_SIZE;
        rowOverhead += AbstractBTreePartition.HOLDER_UNSHARED_HEAP_SIZE;
        allocator.setDiscarding();
        allocator.setDiscarded();
        return rowOverhead;
    }
}