use of org.apache.carbondata.processing.newflow.sort.unsafe.holder.UnsafeInmemoryHolder in project carbondata by apache.
the class UnsafeSingleThreadFinalSortFilesMerger method startSorting.
/**
* Below method will be used to start storing process This method will get
* all the temp files present in sort temp folder then it will create the
* record holder heap and then it will read first record from each file and
* initialize the heap
*
*/
private void startSorting(UnsafeCarbonRowPage[] rowPages, List<UnsafeInMemoryIntermediateDataMerger> merges) throws CarbonDataWriterException {
try {
File[] filesToMergeSort = getFilesToMergeSort();
this.fileCounter = rowPages.length + filesToMergeSort.length + merges.size();
if (fileCounter == 0) {
LOGGER.info("No files to merge sort");
return;
}
LOGGER.info("Number of row pages: " + this.fileCounter);
// create record holder heap
createRecordHolderQueue();
// iterate over file list and create chunk holder and add to heap
LOGGER.info("Started adding first record from each page");
for (final UnsafeCarbonRowPage rowPage : rowPages) {
SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage, parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters.getMeasureColCount(), parameters.getNumberOfSortColumns());
// initialize
sortTempFileChunkHolder.readRow();
recordHolderHeapLocal.add(sortTempFileChunkHolder);
}
for (final UnsafeInMemoryIntermediateDataMerger merger : merges) {
SortTempChunkHolder sortTempFileChunkHolder = new UnsafeFinalMergePageHolder(merger, parameters.getNoDictionarySortColumn(), parameters.getDimColCount() + parameters.getComplexDimColCount() + parameters.getMeasureColCount());
// initialize
sortTempFileChunkHolder.readRow();
recordHolderHeapLocal.add(sortTempFileChunkHolder);
}
for (final File file : filesToMergeSort) {
SortTempChunkHolder sortTempFileChunkHolder = new UnsafeSortTempFileChunkHolder(file, parameters);
// initialize
sortTempFileChunkHolder.readRow();
recordHolderHeapLocal.add(sortTempFileChunkHolder);
}
LOGGER.info("Heap Size" + this.recordHolderHeapLocal.size());
} catch (Exception e) {
LOGGER.error(e);
throw new CarbonDataWriterException(e.getMessage());
}
}
Aggregations