Examples with MemoryBlock io.mycat.memory.unsafe.memory.MemoryBlock used on opensource projects

Example 6 with MemoryBlock

use of io.mycat.memory.unsafe.memory.MemoryBlock in project Mycat-Server by MyCATApache.

the class MemoryConsumer method allocateLongArray.

/**
   * Allocates a LongArray of `size`.
   */
public LongArray allocateLongArray(long size) {
    long required = size * 8L;
    MemoryBlock page = dataNodeMemoryManager.allocatePage(required, this);
    if (page == null || page.size() < required) {
        long got = 0;
        if (page != null) {
            got = page.size();
            dataNodeMemoryManager.freePage(page, this);
        }
        dataNodeMemoryManager.showMemoryUsage();
        throw new OutOfMemoryError("Unable to acquire " + required + " bytes of memory, got " + got);
    }
    used += required;
    return new LongArray(page);
}

Example 7 with MemoryBlock

use of io.mycat.memory.unsafe.memory.MemoryBlock in project Mycat-Server by MyCATApache.

the class UnsafeExternalSorter method freeMemory.

/**
   * Free this sorter's data pages.
   *
   * @return the number of bytes freed.
   */
private long freeMemory() {
    updatePeakMemoryUsed();
    long memoryFreed = 0;
    for (MemoryBlock block : allocatedPages) {
        memoryFreed += block.size();
        freePage(block);
    }
    allocatedPages.clear();
    currentPage = null;
    pageCursor = 0;
    return memoryFreed;
}

Example 8 with MemoryBlock

use of io.mycat.memory.unsafe.memory.MemoryBlock in project Mycat-Server by MyCATApache.

the class UnsafeInMemorySorterSuite method testSortingOnlyByIntegerPrefix.

@Test
public void testSortingOnlyByIntegerPrefix() throws Exception {
    final String[] dataToSort = new String[] { "Boba", "Pearls", "Tapioca", "Taho", "Condensed Milk", "Jasmine", "Milk Tea", "Lychee", "Mango" };
    final DataNodeMemoryManager memoryManager = new DataNodeMemoryManager(new TestMemoryManager(new MycatPropertyConf().set("mycat.memory.offHeap.enabled", "false")), 0);
    final TestMemoryConsumer consumer = new TestMemoryConsumer(memoryManager);
    final MemoryBlock dataPage = memoryManager.allocatePage(2048, null);
    final Object baseObject = dataPage.getBaseObject();
    // Write the records into the data page:
    long position = dataPage.getBaseOffset();
    for (String str : dataToSort) {
        final byte[] strBytes = str.getBytes(StandardCharsets.UTF_8);
        Platform.putInt(baseObject, position, strBytes.length);
        position += 4;
        Platform.copyMemory(strBytes, Platform.BYTE_ARRAY_OFFSET, baseObject, position, strBytes.length);
        position += strBytes.length;
    }
    // Since the key fits within the 8-byte prefix, we don't need to do any record comparison, so
    // use a dummy comparator
    final RecordComparator recordComparator = new RecordComparator() {

        @Override
        public int compare(Object leftBaseObject, long leftBaseOffset, Object rightBaseObject, long rightBaseOffset) {
            return 0;
        }
    };
    // Compute key prefixes based on the records' partition ids
    final HashPartitioner hashPartitioner = new HashPartitioner(4);
    // Use integer comparison for comparing prefixes (which are partition ids, in this case)
    final PrefixComparator prefixComparator = PrefixComparators.LONG;
    UnsafeInMemorySorter sorter = new UnsafeInMemorySorter(consumer, memoryManager, recordComparator, prefixComparator, dataToSort.length, shouldUseRadixSort(), true);
    // Given a page of records, insert those records into the sorter one-by-one:
    position = dataPage.getBaseOffset();
    System.out.println("(0)address = " + position);
    for (int i = 0; i < dataToSort.length; i++) {
        if (!sorter.hasSpaceForAnotherRecord()) {
            sorter.expandPointerArray(consumer.allocateLongArray(sorter.getMemoryUsage() / 8 * 2));
        }
        // position now points to the start of a record (which holds its length).
        final int recordLength = Platform.getInt(baseObject, position);
        final long address = memoryManager.encodePageNumberAndOffset(dataPage, position);
        final String str = getStringFromDataPage(baseObject, position + 4, recordLength);
        final int partitionId = hashPartitioner.getPartition(str);
        System.out.println("(" + partitionId + "," + str + ")");
        sorter.insertRecord(address, partitionId);
        position += 4 + recordLength;
    }
    final UnsafeSorterIterator iter = sorter.getSortedIterator();
    int iterLength = 0;
    long prevPrefix = -1;
    Arrays.sort(dataToSort);
    while (iter.hasNext()) {
        iter.loadNext();
        final String str = getStringFromDataPage(iter.getBaseObject(), iter.getBaseOffset(), iter.getRecordLength());
        final long keyPrefix = iter.getKeyPrefix();
        assertThat(str, isIn(Arrays.asList(dataToSort)));
        assertThat(keyPrefix, greaterThanOrEqualTo(prevPrefix));
        prevPrefix = keyPrefix;
        iterLength++;
    }
    assertEquals(dataToSort.length, iterLength);
}

Example 9 with MemoryBlock

use of io.mycat.memory.unsafe.memory.MemoryBlock in project Mycat-Server by MyCATApache.

the class BytesToBytesMap method reset.

/**
   * Reset this map to initialized state.
   */
public void reset() {
    numKeys = 0;
    numValues = 0;
    longArray.zeroOut();
    while (dataPages.size() > 0) {
        MemoryBlock dataPage = dataPages.removeLast();
        freePage(dataPage);
    }
    currentPage = null;
    pageCursor = 0;
}

Example 10 with MemoryBlock

use of io.mycat.memory.unsafe.memory.MemoryBlock in project Mycat-Server by MyCATApache.

the class DataNodeMemoryManager method getOffsetInPage.

/**
   * Get the offset associated with an address encoded by
   * {@link DataNodeMemoryManager#encodePageNumberAndOffset(MemoryBlock, long)}
   */
public long getOffsetInPage(long pagePlusOffsetAddress) {
    final long offsetInPage = decodeOffset(pagePlusOffsetAddress);
    if (tungstenMemoryMode == MemoryMode.ON_HEAP) {
        return offsetInPage;
    } else {
        // In off-heap mode, an offset is an absolute address. In encodePageNumberAndOffset, we
        // converted the absolute address into a relative address. Here, we invert that operation:
        final int pageNumber = decodePageNumber(pagePlusOffsetAddress);
        assert (pageNumber >= 0 && pageNumber < PAGE_TABLE_SIZE);
        final MemoryBlock page = pageTable[pageNumber];
        assert (page != null);
        return page.getBaseOffset() + offsetInPage;
    }
}