Search in sources :

Example 1 with UnsafeFinalMergePageHolder

use of org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeFinalMergePageHolder in project carbondata by apache.

the class UnsafeSingleThreadFinalSortFilesMerger method startSorting.

/**
 * Below method will be used to start storing process This method will get
 * all the temp files present in sort temp folder then it will create the
 * record holder heap and then it will read first record from each file and
 * initialize the heap
 */
private void startSorting(UnsafeCarbonRowPage[] rowPages, List<UnsafeInMemoryIntermediateDataMerger> merges) throws CarbonDataWriterException {
    try {
        List<File> filesToMergeSort = getFilesToMergeSort();
        this.fileCounter = rowPages.length + filesToMergeSort.size() + merges.size();
        if (fileCounter == 0) {
            LOGGER.info("No files to merge sort");
            return;
        }
        LOGGER.info("Starting final merger");
        LOGGER.info("Number of row pages: " + this.fileCounter);
        // create record holder heap
        createRecordHolderQueue();
        // iterate over file list and create chunk holder and add to heap
        LOGGER.info("Started adding first record from each page");
        for (final UnsafeCarbonRowPage rowPage : rowPages) {
            SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage);
            // initialize
            sortTempFileChunkHolder.readRow();
            recordHolderHeapLocal.add(sortTempFileChunkHolder);
        }
        for (final UnsafeInMemoryIntermediateDataMerger merger : merges) {
            SortTempChunkHolder sortTempFileChunkHolder = new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionarySortColumn());
            // initialize
            sortTempFileChunkHolder.readRow();
            recordHolderHeapLocal.add(sortTempFileChunkHolder);
        }
        for (final File file : filesToMergeSort) {
            SortTempChunkHolder sortTempFileChunkHolder = new UnsafeSortTempFileChunkHolder(file, parameters);
            // initialize
            sortTempFileChunkHolder.readRow();
            recordHolderHeapLocal.add(sortTempFileChunkHolder);
        }
        LOGGER.info("Heap Size: " + this.recordHolderHeapLocal.size());
    } catch (Exception e) {
        LOGGER.error(e);
        throw new CarbonDataWriterException(e);
    }
}
Also used : UnsafeFinalMergePageHolder(org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeFinalMergePageHolder) SortTempChunkHolder(org.apache.carbondata.processing.loading.sort.unsafe.holder.SortTempChunkHolder) UnsafeCarbonRowPage(org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage) File(java.io.File) UnsafeInmemoryHolder(org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeInmemoryHolder) UnsafeSortTempFileChunkHolder(org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeSortTempFileChunkHolder) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException)

Aggregations

File (java.io.File)1 CarbonDataWriterException (org.apache.carbondata.core.datastore.exception.CarbonDataWriterException)1 UnsafeCarbonRowPage (org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage)1 SortTempChunkHolder (org.apache.carbondata.processing.loading.sort.unsafe.holder.SortTempChunkHolder)1 UnsafeFinalMergePageHolder (org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeFinalMergePageHolder)1 UnsafeInmemoryHolder (org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeInmemoryHolder)1 UnsafeSortTempFileChunkHolder (org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeSortTempFileChunkHolder)1