Examples with TableFieldStat org.apache.carbondata.processing.sort.sortdata.TableFieldStat used on opensource projects

Example 1 with TableFieldStat

use of org.apache.carbondata.processing.sort.sortdata.TableFieldStat in project carbondata by apache.

the class UnsafeSingleThreadFinalSortFilesMerger method startSorting.

/**
 * Below method will be used to start storing process This method will get
 * all the temp files present in sort temp folder then it will create the
 * record holder heap and then it will read first record from each file and
 * initialize the heap
 */
private void startSorting(UnsafeCarbonRowPage[] rowPages, List<UnsafeInMemoryIntermediateDataMerger> merges) throws CarbonDataWriterException {
    try {
        List<File> filesToMergeSort = getFilesToMergeSort();
        this.fileCounter = rowPages.length + filesToMergeSort.size() + merges.size();
        if (fileCounter == 0) {
            LOGGER.info("No files to merge sort");
            return;
        }
        LOGGER.info(String.format("Starting final merge of %d pages, including row pages: %d" + ", sort temp files: %d, intermediate merges: %d", this.fileCounter, rowPages.length, filesToMergeSort.size(), merges.size()));
        // create record holder heap
        createRecordHolderQueue();
        TableFieldStat tableFieldStat = new TableFieldStat(parameters);
        // iterate over file list and create chunk holder and add to heap
        LOGGER.info("Started adding first record from each page");
        for (final UnsafeCarbonRowPage rowPage : rowPages) {
            SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage);
            // initialize
            sortTempFileChunkHolder.readRow();
            recordHolderHeapLocal.add(sortTempFileChunkHolder);
        }
        for (final UnsafeInMemoryIntermediateDataMerger merger : merges) {
            SortTempChunkHolder sortTempFileChunkHolder = new UnsafeFinalMergePageHolder(merger, tableFieldStat);
            // initialize
            sortTempFileChunkHolder.readRow();
            recordHolderHeapLocal.add(sortTempFileChunkHolder);
        }
        for (final File file : filesToMergeSort) {
            SortTempChunkHolder sortTempFileChunkHolder = new UnsafeSortTempFileChunkHolder(file, parameters, true, tableFieldStat);
            // initialize
            sortTempFileChunkHolder.readRow();
            recordHolderHeapLocal.add(sortTempFileChunkHolder);
        }
        LOGGER.info("Heap Size: " + this.recordHolderHeapLocal.size());
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        throw new CarbonDataWriterException(e);
    }
}