Examples with SingleThreadFinalSortFilesMerger org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger used on opensource projects

Example 1 with SingleThreadFinalSortFilesMerger

use of org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger in project carbondata by apache.

the class ParallelReadMergeSorterImpl method initialize.

@Override
public void initialize(SortParameters sortParameters) {
    this.sortParameters = sortParameters;
    intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);
    String[] storeLocations = CarbonDataProcessorUtil.getLocalDataFolderLocation(sortParameters.getCarbonTable(), String.valueOf(sortParameters.getTaskNo()), sortParameters.getSegmentId(), false, false);
    // Set the data file location
    String[] dataFolderLocations = CarbonDataProcessorUtil.arrayAppend(storeLocations, File.separator, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
    finalMerger = new SingleThreadFinalSortFilesMerger(dataFolderLocations, sortParameters.getTableName(), sortParameters);
    // Delete if any older file exists in sort temp folder
    CarbonDataProcessorUtil.deleteSortLocationIfExists(sortParameters.getTempFileLocation());
    // create new sort temp directory
    CarbonDataProcessorUtil.createLocations(sortParameters.getTempFileLocation());
}

Example 2 with SingleThreadFinalSortFilesMerger

use of org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger in project carbondata by apache.

the class CompactionResultSortProcessor method initializeFinalThreadMergerForMergeSort.

/**
 * create an instance of finalThread merger which will perform merge sort on all the
 * sort temp files
 */
private void initializeFinalThreadMergerForMergeSort() {
    boolean[] noDictionarySortColumnMapping = CarbonDataProcessorUtil.getNoDictSortColMapping(carbonTable);
    sortParameters.setNoDictionarySortColumn(noDictionarySortColumnMapping);
    Map<String, int[]> columnIdxMap = CarbonDataProcessorUtil.getColumnIdxBasedOnSchemaInRow(carbonTable);
    sortParameters.setNoDictSortColumnSchemaOrderMapping(columnIdxMap.get("columnIdxBasedOnSchemaInRow"));
    sortParameters.setNoDictSortColIdxSchemaOrderMapping(columnIdxMap.get("noDictSortIdxBasedOnSchemaInRow"));
    sortParameters.setDictSortColIdxSchemaOrderMapping(columnIdxMap.get("dictSortIdxBasedOnSchemaInRow"));
    String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation, CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
    finalMerger = new SingleThreadFinalSortFilesMerger(sortTempFileLocation, tableName, sortParameters);
}

Example 3 with SingleThreadFinalSortFilesMerger

use of org.apache.carbondata.processing.sort.sortdata.SingleThreadFinalSortFilesMerger in project carbondata by apache.

the class SecondaryIndexQueryResultProcessor method initializeFinalThreadMergerForMergeSort.

/**
 * create an instance of finalThread merger which will perform merge sort on all the
 * sort temp files
 */
private void initializeFinalThreadMergerForMergeSort() {
    String[] sortTempFileLocation = CarbonDataProcessorUtil.arrayAppend(tempStoreLocation, CarbonCommonConstants.FILE_SEPARATOR, CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
    sortParameters.setNoDictionarySortColumn(CarbonDataProcessorUtil.getNoDictSortColMapping(indexTable));
    finalMerger = new SingleThreadFinalSortFilesMerger(sortTempFileLocation, indexTable.getTableName(), sortParameters);
}