Examples with MemoryBlock io.mycat.memory.unsafe.memory.MemoryBlock used on opensource projects

Example 1 with MemoryBlock

use of io.mycat.memory.unsafe.memory.MemoryBlock in project Mycat-Server by MyCATApache.

the class BytesToBytesMap method free.

/**
   * Free all allocated memory associated with this map, including the storage for keys and values
   * as well as the hash map array itself.
   *
   * This method is idempotent and can be called multiple times.
   */
public void free() {
    updatePeakMemoryUsed();
    if (longArray != null) {
        freeLongArray(longArray);
        longArray = null;
    }
    Iterator<MemoryBlock> dataPagesIterator = dataPages.iterator();
    while (dataPagesIterator.hasNext()) {
        MemoryBlock dataPage = dataPagesIterator.next();
        dataPagesIterator.remove();
        freePage(dataPage);
    }
    assert (dataPages.isEmpty());
    while (!spillWriters.isEmpty()) {
        File file = spillWriters.removeFirst().getFile();
        if (file != null && file.exists()) {
            if (!file.delete()) {
                logger.error("Was unable to delete spill file {}", file.getAbsolutePath());
            }
        }
    }
}

Example 2 with MemoryBlock

use of io.mycat.memory.unsafe.memory.MemoryBlock in project Mycat-Server by MyCATApache.

the class DataNodeMemoryManager method getPage.

/**
   * Get the page associated with an address encoded by
   * {@link DataNodeMemoryManager#encodePageNumberAndOffset(MemoryBlock, long)}
   */
public Object getPage(long pagePlusOffsetAddress) {
    if (tungstenMemoryMode == MemoryMode.ON_HEAP) {
        final int pageNumber = decodePageNumber(pagePlusOffsetAddress);
        assert (pageNumber >= 0 && pageNumber < PAGE_TABLE_SIZE);
        final MemoryBlock page = pageTable[pageNumber];
        assert (page != null);
        assert (page.getBaseObject() != null);
        return page.getBaseObject();
    } else {
        return null;
    }
}

Example 3 with MemoryBlock

use of io.mycat.memory.unsafe.memory.MemoryBlock in project Mycat-Server by MyCATApache.

the class DataNodeMemoryManager method allocatePage.

/**
   * Allocate a block of memory that will be tracked in the MemoryManager's page table; this is
   * intended for allocating large blocks of Tungsten memory that will be shared between operators.
   *
   * Returns `null` if there was not enough memory to allocate the page. May return a page that
   * contains fewer bytes than requested, so callers should verify the size of returned pages.
   */
public MemoryBlock allocatePage(long size, MemoryConsumer consumer) {
    if (size > MAXIMUM_PAGE_SIZE_BYTES) {
        throw new IllegalArgumentException("Cannot allocate a page with more than " + MAXIMUM_PAGE_SIZE_BYTES + " bytes");
    }
    /**
     * 这里spill到磁盘中，释放内存空间
     */
    long acquired = 0;
    try {
        acquired = acquireExecutionMemory(size, tungstenMemoryMode, consumer);
    } catch (InterruptedException e) {
        logger.error(e.getMessage());
    }
    if (acquired <= 0) {
        return null;
    }
    final int pageNumber;
    synchronized (this) {
        pageNumber = allocatedPages.nextClearBit(0);
        if (pageNumber >= PAGE_TABLE_SIZE) {
            releaseExecutionMemory(acquired, tungstenMemoryMode, consumer);
            throw new IllegalStateException("Have already allocated a maximum of " + PAGE_TABLE_SIZE + " pages");
        }
        allocatedPages.set(pageNumber);
    }
    MemoryBlock page = null;
    try {
        page = memoryManager.tungstenMemoryAllocator().allocate(acquired);
    } catch (OutOfMemoryError e) {
        logger.warn("Failed to allocate a page ({} bytes), try again.", acquired);
        // MemoryManager thought, we should keep the acquired memory.
        synchronized (this) {
            acquiredButNotUsed += acquired;
            allocatedPages.clear(pageNumber);
        }
        // this could trigger spilling to free some pages.
        return allocatePage(size, consumer);
    }
    page.pageNumber = pageNumber;
    pageTable[pageNumber] = page;
    return page;
}

Example 4 with MemoryBlock

use of io.mycat.memory.unsafe.memory.MemoryBlock in project Mycat-Server by MyCATApache.

the class MemoryConsumer method allocatePage.

/**
   * Allocate a memory block with at least `required` bytes.
   *
   * Throws IOException if there is not enough memory.
   *
   * @throws OutOfMemoryError
   */
protected MemoryBlock allocatePage(long required) {
    MemoryBlock page = dataNodeMemoryManager.allocatePage(Math.max(pageSize, required), this);
    if (page == null || page.size() < required) {
        long got = 0;
        if (page != null) {
            got = page.size();
            dataNodeMemoryManager.freePage(page, this);
        }
        dataNodeMemoryManager.showMemoryUsage();
        throw new OutOfMemoryError("Unable to acquire " + required + " bytes of memory, got " + got);
    }
    used += page.size();
    return page;
}

Example 5 with MemoryBlock

use of io.mycat.memory.unsafe.memory.MemoryBlock in project Mycat-Server by MyCATApache.

the class MemoryConsumer method allocateCharArray.

public CharArray allocateCharArray(long size) {
    long required = size * 2L;
    MemoryBlock page = dataNodeMemoryManager.allocatePage(required, this);
    if (page == null || page.size() < required) {
        long got = 0;
        if (page != null) {
            got = page.size();
            dataNodeMemoryManager.freePage(page, this);
        }
        dataNodeMemoryManager.showMemoryUsage();
        throw new OutOfMemoryError("Unable to acquire " + required + " bytes of memory, got " + got);
    }
    used += required;
    return new CharArray(page, this);
}