Examples with TimSort org.apache.carbondata.processing.loading.sort.unsafe.sort.TimSort used on opensource projects

Example 1 with TimSort

use of org.apache.carbondata.processing.loading.sort.unsafe.sort.TimSort in project carbondata by apache.

the class UnsafeSortDataRows method startSorting.

/**
 * Below method will be used to start sorting process. This method will get
 * all the temp unsafe pages in memory and all the temp files and try to merge them if possible.
 * Also, it will spill the pages to disk or add it to unsafe sort memory.
 */
public void startSorting() {
    LOGGER.info("Unsafe based sorting will be used");
    if (this.rowPage.getUsedSize() > 0) {
        TimSort<UnsafeCarbonRow, IntPointerBuffer> timSort = new TimSort<>(new UnsafeIntSortDataFormat(rowPage));
        if (parameters.getNumberOfNoDictSortColumns() > 0) {
            timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(), new UnsafeRowComparator(rowPage));
        } else {
            timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(), new UnsafeRowComparatorForNormalDims(rowPage));
        }
        unsafeInMemoryIntermediateFileMerger.addDataChunkToMerge(rowPage);
    } else {
        rowPage.freeMemory();
    }
}

Example 2 with TimSort

use of org.apache.carbondata.processing.loading.sort.unsafe.sort.TimSort in project carbondata by apache.

the class UnsafeSortDataRows method handlePreviousPage.

/**
 * Deal with the previous pages added to sort-memory. Carbondata will merge the in-memory pages
 * or merge the sort temp files if possible. After that, carbondata will add current page to
 * sort memory or just spill them.
 */
private void handlePreviousPage() {
    try {
        long startTime = System.currentTimeMillis();
        TimSort<UnsafeCarbonRow, IntPointerBuffer> timSort = new TimSort<>(new UnsafeIntSortDataFormat(rowPage));
        // if sort_columns is not none, sort by sort_columns
        if (parameters.getNumberOfNoDictSortColumns() > 0) {
            timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(), new UnsafeRowComparator(rowPage));
        } else {
            timSort.sort(rowPage.getBuffer(), 0, rowPage.getBuffer().getActualSize(), new UnsafeRowComparatorForNormalDims(rowPage));
        }
        // get sort storage memory block if memory is available in sort storage manager
        // if space is available then store it in memory, if memory is not available
        // then spill to disk
        MemoryBlock sortStorageMemoryBlock = null;
        if (!rowPage.isSaveToDisk()) {
            sortStorageMemoryBlock = UnsafeSortMemoryManager.INSTANCE.allocateMemory(taskId, rowPage.getDataBlock().size());
        }
        if (null == sortStorageMemoryBlock || rowPage.isSaveToDisk()) {
            // create a new file every time
            // create a new file and pick a temp directory randomly every time
            String tmpDir = parameters.getTempFileLocation()[new Random().nextInt(parameters.getTempFileLocation().length)];
            File sortTempFile = new File(tmpDir + File.separator + parameters.getTableName() + '_' + parameters.getRangeId() + '_' + instanceId + '_' + System.nanoTime() + CarbonCommonConstants.SORT_TEMP_FILE_EXT);
            writeDataToFile(rowPage, sortTempFile);
            LOGGER.info("Time taken to sort row page with size" + rowPage.getBuffer().getActualSize() + " and write is: " + (System.currentTimeMillis() - startTime) + ": location:" + sortTempFile + ", sort temp file size in MB is " + sortTempFile.length() * 0.1 * 10 / 1024 / 1024);
            rowPage.freeMemory();
            // add sort temp filename to and arrayList. When the list size reaches 20 then
            // intermediate merging of sort temp files will be triggered
            unsafeInMemoryIntermediateFileMerger.addFileToMerge(sortTempFile);
        } else {
            // copying data from working memory manager block to storage memory manager block
            CarbonUnsafe.getUnsafe().copyMemory(rowPage.getDataBlock().getBaseObject(), rowPage.getDataBlock().getBaseOffset(), sortStorageMemoryBlock.getBaseObject(), sortStorageMemoryBlock.getBaseOffset(), rowPage.getDataBlock().size());
            // free unsafememory manager
            rowPage.freeMemory();
            rowPage.setNewDataBlock(sortStorageMemoryBlock);
            // add sort temp filename to and arrayList. When the list size reaches 20 then
            // intermediate merging of sort temp files will be triggered
            rowPage.getBuffer().loadToUnsafe();
            unsafeInMemoryIntermediateFileMerger.addDataChunkToMerge(rowPage);
            LOGGER.info("Time taken to sort row page with size: " + rowPage.getBuffer().getActualSize() + " is: " + (System.currentTimeMillis() - startTime));
        }
    } catch (Throwable e) {
        try {
            threadStatusObserver.notifyFailed(e);
        } catch (CarbonSortKeyAndGroupByException ex) {
            LOGGER.error(e.getMessage(), e);
        }
    }
}