Examples with MemoryBlock org.apache.carbondata.core.memory.MemoryBlock used on opensource projects

Example 11 with MemoryBlock

use of org.apache.carbondata.core.memory.MemoryBlock in project carbondata by apache.

the class UnsafeSortDataRows method initialize.

public void initialize() {
    MemoryBlock baseBlock = UnsafeMemoryManager.allocateMemoryWithRetry(this.taskId, inMemoryChunkSize);
    boolean isMemoryAvailable = UnsafeSortMemoryManager.INSTANCE.isMemoryAvailable(baseBlock.size());
    this.rowPage = new UnsafeCarbonRowPage(tableFieldStat, baseBlock, taskId, isMemoryAvailable);
}

Example 12 with MemoryBlock

use of org.apache.carbondata.core.memory.MemoryBlock in project carbondata by apache.

the class UnsafeSortDataRows method handlePreviousPage.

/**
 * Deal with the previous pages added to sort-memory. Carbondata will merge the in-memory pages
 * or merge the sort temp files if possible. After that, carbondata will add current page to
 * sort memory or just spill them.
 */
private void handlePreviousPage() {
    try {
        long startTime = System.currentTimeMillis();
        TimSort<UnsafeCarbonRow, IntPointerBuffer> timSort = new TimSort<>(new UnsafeIntSortDataFormat(rowPage));
        // if sort_columns is not none, sort by sort_columns
        if (parameters.getNumberOfNoDictSortColumns() > 0) {
            timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(), new UnsafeRowComparator(rowPage));
        } else {
            timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(), new UnsafeRowComparatorForNormalDims(rowPage));
        }
        // get sort storage memory block if memory is available in sort storage manager
        // if space is available then store it in memory, if memory is not available
        // then spill to disk
        MemoryBlock sortStorageMemoryBlock = null;
        if (!rowPage.isSaveToDisk()) {
            sortStorageMemoryBlock = UnsafeSortMemoryManager.INSTANCE.allocateMemory(taskId, rowPage.getDataBlock().size());
        }
        if (null == sortStorageMemoryBlock || rowPage.isSaveToDisk()) {
            // create a new file every time
            // create a new file and pick a temp directory randomly every time
            String tmpDir = parameters.getTempFileLocation()[new Random().nextInt(parameters.getTempFileLocation().length)];
            File sortTempFile = new File(tmpDir + File.separator + parameters.getTableName() + '_' + parameters.getRangeId() + '_' + instanceId + '_' + System.nanoTime() + CarbonCommonConstants.SORT_TEMP_FILE_EXT);
            writeDataToFile(rowPage, sortTempFile);
            LOGGER.info("Time taken to sort row page with size" + rowPage.getBuffer().getActualSize() + " and write is: " + (System.currentTimeMillis() - startTime) + ": location:" + sortTempFile + ", sort temp file size in MB is " + sortTempFile.length() * 0.1 * 10 / 1024 / 1024);
            rowPage.freeMemory();
            // add sort temp filename to and arrayList. When the list size reaches 20 then
            // intermediate merging of sort temp files will be triggered
            unsafeInMemoryIntermediateFileMerger.addFileToMerge(sortTempFile);
        } else {
            // copying data from working memory manager block to storage memory manager block
            CarbonUnsafe.getUnsafe().copyMemory(rowPage.getDataBlock().getBaseObject(), rowPage.getDataBlock().getBaseOffset(), sortStorageMemoryBlock.getBaseObject(), sortStorageMemoryBlock.getBaseOffset(), rowPage.getDataBlock().size());
            // free unsafememory manager
            rowPage.freeMemory();
            rowPage.setNewDataBlock(sortStorageMemoryBlock);
            // add sort temp filename to and arrayList. When the list size reaches 20 then
            // intermediate merging of sort temp files will be triggered
            rowPage.getBuffer().loadToUnsafe();
            unsafeInMemoryIntermediateFileMerger.addDataChunkToMerge(rowPage);
            LOGGER.info("Time taken to sort row page with size: " + rowPage.getBuffer().getActualSize() + " is: " + (System.currentTimeMillis() - startTime));
        }
    } catch (Throwable e) {
        try {
            threadStatusObserver.notifyFailed(e);
        } catch (CarbonSortKeyAndGroupByException ex) {
            LOGGER.error(e.getMessage(), e);
        }
    }
}

Example 13 with MemoryBlock

use of org.apache.carbondata.core.memory.MemoryBlock in project carbondata by apache.

the class UnsafeVarLengthColumnPageBase method ensureMemory.

/**
 * reallocate memory if capacity length than current size + request size
 */
protected void ensureMemory(int requestSize) {
    if (totalLength + requestSize > capacity) {
        int newSize = Math.max(2 * capacity, totalLength + requestSize);
        MemoryBlock newBlock = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, newSize);
        CarbonUnsafe.getUnsafe().copyMemory(baseAddress, baseOffset, newBlock.getBaseObject(), newBlock.getBaseOffset(), capacity);
        UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock);
        memoryBlock = newBlock;
        baseAddress = newBlock.getBaseObject();
        baseOffset = newBlock.getBaseOffset();
        capacity = newSize;
    }
}

Example 14 with MemoryBlock

use of org.apache.carbondata.core.memory.MemoryBlock in project carbondata by apache.

the class UnsafeMemoryDMStore method finishWriting.

public void finishWriting() {
    if (runningLength < allocatedSize) {
        MemoryBlock allocate = UnsafeMemoryManager.allocateMemoryWithRetry(MemoryType.ONHEAP, taskId, runningLength);
        getUnsafe().copyMemory(memoryBlock.getBaseObject(), memoryBlock.getBaseOffset(), allocate.getBaseObject(), allocate.getBaseOffset(), runningLength);
        UnsafeMemoryManager.INSTANCE.freeMemory(taskId, memoryBlock);
        memoryBlock = allocate;
    }
    // Compact pointers.
    if (rowCount < pointers.length) {
        int[] newPointer = new int[rowCount];
        System.arraycopy(pointers, 0, newPointer, 0, rowCount);
        this.pointers = newPointer;
    }
}

Example 15 with MemoryBlock

use of org.apache.carbondata.core.memory.MemoryBlock in project carbondata by apache.

the class UnsafeMemoryDMStore method increaseMemory.

private void increaseMemory(int requiredMemory) {
    MemoryBlock newMemoryBlock = UnsafeMemoryManager.allocateMemoryWithRetry(MemoryType.ONHEAP, taskId, allocatedSize + requiredMemory);
    getUnsafe().copyMemory(this.memoryBlock.getBaseObject(), this.memoryBlock.getBaseOffset(), newMemoryBlock.getBaseObject(), newMemoryBlock.getBaseOffset(), runningLength);
    UnsafeMemoryManager.INSTANCE.freeMemory(taskId, this.memoryBlock);
    allocatedSize = allocatedSize + requiredMemory;
    this.memoryBlock = newMemoryBlock;
}