Examples with BufferLedger org.apache.drill.exec.memory.AllocationManager.BufferLedger used on opensource projects

Example 1 with BufferLedger

use of org.apache.drill.exec.memory.AllocationManager.BufferLedger in project drill by apache.

the class BaseAllocator method bufferWithoutReservation.

/**
   * Used by usual allocation as well as for allocating a pre-reserved buffer. Skips the typical accounting associated
   * with creating a new buffer.
   */
private DrillBuf bufferWithoutReservation(final int size, BufferManager bufferManager) throws OutOfMemoryException {
    assertOpen();
    final AllocationManager manager = new AllocationManager(this, size);
    // +1 ref cnt (required)
    final BufferLedger ledger = manager.associate(this);
    final DrillBuf buffer = ledger.newDrillBuf(0, size, bufferManager);
    // make sure that our allocation is equal to what we expected.
    Preconditions.checkArgument(buffer.capacity() == size, "Allocated capacity %d was not equal to requested capacity %d.", buffer.capacity(), size);
    return buffer;
}

Example 2 with BufferLedger

use of org.apache.drill.exec.memory.AllocationManager.BufferLedger in project drill by apache.

the class BaseAllocator method dumpBuffers.

private void dumpBuffers(final StringBuilder sb, final Set<BufferLedger> ledgerSet) {
    for (final BufferLedger ledger : ledgerSet) {
        if (!ledger.isOwningLedger()) {
            continue;
        }
        final UnsafeDirectLittleEndian udle = ledger.getUnderlying();
        sb.append("UnsafeDirectLittleEndian[dentityHashCode == ");
        sb.append(Integer.toString(System.identityHashCode(udle)));
        sb.append("] size ");
        sb.append(Integer.toString(udle.capacity()));
        sb.append('\n');
    }
}

Example 3 with BufferLedger

use of org.apache.drill.exec.memory.AllocationManager.BufferLedger in project drill by apache.

the class BaseAllocator method verifyAllocator.

/**
   * Verifies the accounting state of the allocator. Only works for DEBUG.
   *
   * <p>
   * This overload is used for recursive calls, allowing for checking that DrillBufs are unique across all allocators
   * that are checked.
   * </p>
   *
   * @param buffersSeen
   *          a map of buffers that have already been seen when walking a tree of allocators
   * @throws IllegalStateException
   *           when any problems are found
   */
private void verifyAllocator(final IdentityHashMap<UnsafeDirectLittleEndian, BaseAllocator> buffersSeen) {
    synchronized (DEBUG_LOCK) {
        // The remaining tests can only be performed if we're in debug mode.
        if (!DEBUG) {
            return;
        }
        final long allocated = getAllocatedMemory();
        // verify my direct descendants
        final Set<BaseAllocator> childSet = childAllocators.keySet();
        for (final BaseAllocator childAllocator : childSet) {
            childAllocator.verifyAllocator(buffersSeen);
        }
        /*
       * Verify my relationships with my descendants.
       *
       * The sum of direct child allocators' owned memory must be <= my allocated memory; my allocated memory also
       * includes DrillBuf's directly allocated by me.
       */
        long childTotal = 0;
        for (final BaseAllocator childAllocator : childSet) {
            childTotal += Math.max(childAllocator.getAllocatedMemory(), childAllocator.reservation);
        }
        if (childTotal > getAllocatedMemory()) {
            historicalLog.logHistory(logger);
            logger.debug("allocator[" + name + "] child event logs BEGIN");
            for (final BaseAllocator childAllocator : childSet) {
                childAllocator.historicalLog.logHistory(logger);
            }
            logger.debug("allocator[" + name + "] child event logs END");
            throw new IllegalStateException("Child allocators own more memory (" + childTotal + ") than their parent (name = " + name + " ) has allocated (" + getAllocatedMemory() + ')');
        }
        // Furthermore, the amount I've allocated should be that plus buffers I've allocated.
        long bufferTotal = 0;
        final Set<BufferLedger> ledgerSet = childLedgers.keySet();
        for (final BufferLedger ledger : ledgerSet) {
            if (!ledger.isOwningLedger()) {
                continue;
            }
            final UnsafeDirectLittleEndian udle = ledger.getUnderlying();
            /*
         * Even when shared, DrillBufs are rewrapped, so we should never see the same instance twice.
         */
            final BaseAllocator otherOwner = buffersSeen.get(udle);
            if (otherOwner != null) {
                throw new IllegalStateException("This allocator's drillBuf already owned by another allocator");
            }
            buffersSeen.put(udle, this);
            bufferTotal += udle.capacity();
        }
        // Preallocated space has to be accounted for
        final Set<Reservation> reservationSet = reservations.keySet();
        long reservedTotal = 0;
        for (final Reservation reservation : reservationSet) {
            if (!reservation.isUsed()) {
                reservedTotal += reservation.getSize();
            }
        }
        if (bufferTotal + reservedTotal + childTotal != getAllocatedMemory()) {
            final StringBuilder sb = new StringBuilder();
            sb.append("allocator[");
            sb.append(name);
            sb.append("]\nallocated: ");
            sb.append(Long.toString(allocated));
            sb.append(" allocated - (bufferTotal + reservedTotal + childTotal): ");
            sb.append(Long.toString(allocated - (bufferTotal + reservedTotal + childTotal)));
            sb.append('\n');
            if (bufferTotal != 0) {
                sb.append("buffer total: ");
                sb.append(Long.toString(bufferTotal));
                sb.append('\n');
                dumpBuffers(sb, ledgerSet);
            }
            if (childTotal != 0) {
                sb.append("child total: ");
                sb.append(Long.toString(childTotal));
                sb.append('\n');
                for (final BaseAllocator childAllocator : childSet) {
                    sb.append("child allocator[");
                    sb.append(childAllocator.name);
                    sb.append("] owned ");
                    sb.append(Long.toString(childAllocator.getAllocatedMemory()));
                    sb.append('\n');
                }
            }
            if (reservedTotal != 0) {
                sb.append(String.format("reserved total : %d bytes.", reservedTotal));
                for (final Reservation reservation : reservationSet) {
                    reservation.historicalLog.buildHistory(sb, 0, true);
                    sb.append('\n');
                }
            }
            logger.debug(sb.toString());
            final long allocated2 = getAllocatedMemory();
            if (allocated2 != allocated) {
                throw new IllegalStateException(String.format("allocator[%s]: allocated t1 (%d) + allocated t2 (%d). Someone released memory while in verification.", name, allocated, allocated2));
            }
            throw new IllegalStateException(String.format("allocator[%s]: buffer space (%d) + prealloc space (%d) + child space (%d) != allocated (%d)", name, bufferTotal, reservedTotal, childTotal, allocated));
        }
    }
}

Example 4 with BufferLedger

use of org.apache.drill.exec.memory.AllocationManager.BufferLedger in project drill by apache.

the class DrillBuf method transferOwnership.

/**
   * Transfer the memory accounting ownership of this DrillBuf to another allocator. This will generate a new DrillBuf
   * that carries an association with the underlying memory of this DrillBuf. If this DrillBuf is connected to the
   * owning BufferLedger of this memory, that memory ownership/accounting will be transferred to the taret allocator. If
   * this DrillBuf does not currently own the memory underlying it (and is only associated with it), this does not
   * transfer any ownership to the newly created DrillBuf.
   *
   * This operation has no impact on the reference count of this DrillBuf. The newly created DrillBuf with either have a
   * reference count of 1 (in the case that this is the first time this memory is being associated with the new
   * allocator) or the current value of the reference count for the other AllocationManager/BufferLedger combination in
   * the case that the provided allocator already had an association to this underlying memory.
   *
   * Transfers will always succeed, even if that puts the other allocator into an overlimit situation. This is possible
   * due to the fact that the original owning allocator may have allocated this memory out of a local reservation
   * whereas the target allocator may need to allocate new memory from a parent or RootAllocator. This operation is done
   * in a mostly-lockless but consistent manner. As such, the overlimit==true situation could occur slightly prematurely
   * to an actual overlimit==true condition. This is simply conservative behavior which means we may return overlimit
   * slightly sooner than is necessary.
   *
   * @param target
   *          The allocator to transfer ownership to.
   * @return A new transfer result with the impact of the transfer (whether it was overlimit) as well as the newly
   *         created DrillBuf.
   */
public TransferResult transferOwnership(BufferAllocator target) {
    if (isEmpty) {
        return new TransferResult(true, this);
    }
    final BufferLedger otherLedger = this.ledger.getLedgerForAllocator(target);
    final DrillBuf newBuf = otherLedger.newDrillBuf(offset, length, null);
    final boolean allocationFit = this.ledger.transferBalance(otherLedger);
    return new TransferResult(allocationFit, newBuf);
}

Example 5 with BufferLedger

use of org.apache.drill.exec.memory.AllocationManager.BufferLedger in project drill by apache.

the class BaseAllocator method print.

void print(StringBuilder sb, int level, Verbosity verbosity) {
    indent(sb, level).append("Allocator(").append(name).append(") ").append(reservation).append('/').append(getAllocatedMemory()).append('/').append(getPeakMemoryAllocation()).append('/').append(getLimit()).append(" (res/actual/peak/limit)").append('\n');
    if (DEBUG) {
        indent(sb, level + 1).append(String.format("child allocators: %d\n", childAllocators.size()));
        for (BaseAllocator child : childAllocators.keySet()) {
            child.print(sb, level + 2, verbosity);
        }
        indent(sb, level + 1).append(String.format("ledgers: %d\n", childLedgers.size()));
        for (BufferLedger ledger : childLedgers.keySet()) {
            ledger.print(sb, level + 2, verbosity);
        }
        final Set<Reservation> reservations = this.reservations.keySet();
        indent(sb, level + 1).append(String.format("reservations: %d\n", reservations.size()));
        for (final Reservation reservation : reservations) {
            if (verbosity.includeHistoricalLog) {
                reservation.historicalLog.buildHistory(sb, level + 3, true);
            }
        }
    }
}