use of org.apache.carbondata.processing.loading.sort.unsafe.holder.UnsafeFinalMergePageHolder in project carbondata by apache.
the class UnsafeSingleThreadFinalSortFilesMerger method startSorting.
/**
* Below method will be used to start storing process This method will get
* all the temp files present in sort temp folder then it will create the
* record holder heap and then it will read first record from each file and
* initialize the heap
*/
private void startSorting(UnsafeCarbonRowPage[] rowPages, List<UnsafeInMemoryIntermediateDataMerger> merges) throws CarbonDataWriterException {
try {
List<File> filesToMergeSort = getFilesToMergeSort();
this.fileCounter = rowPages.length + filesToMergeSort.size() + merges.size();
if (fileCounter == 0) {
LOGGER.info("No files to merge sort");
return;
}
LOGGER.info("Starting final merger");
LOGGER.info("Number of row pages: " + this.fileCounter);
// create record holder heap
createRecordHolderQueue();
// iterate over file list and create chunk holder and add to heap
LOGGER.info("Started adding first record from each page");
for (final UnsafeCarbonRowPage rowPage : rowPages) {
SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage);
// initialize
sortTempFileChunkHolder.readRow();
recordHolderHeapLocal.add(sortTempFileChunkHolder);
}
for (final UnsafeInMemoryIntermediateDataMerger merger : merges) {
SortTempChunkHolder sortTempFileChunkHolder = new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionarySortColumn());
// initialize
sortTempFileChunkHolder.readRow();
recordHolderHeapLocal.add(sortTempFileChunkHolder);
}
for (final File file : filesToMergeSort) {
SortTempChunkHolder sortTempFileChunkHolder = new UnsafeSortTempFileChunkHolder(file, parameters);
// initialize
sortTempFileChunkHolder.readRow();
recordHolderHeapLocal.add(sortTempFileChunkHolder);
}
LOGGER.info("Heap Size: " + this.recordHolderHeapLocal.size());
} catch (Exception e) {
LOGGER.error(e);
throw new CarbonDataWriterException(e);
}
}
Aggregations