use of org.apache.carbondata.processing.newflow.sort.unsafe.merger.UnsafeIntermediateMerger in project carbondata by apache.
the class UnsafeParallelReadMergeSorterImpl method initialize.
@Override
public void initialize(SortParameters sortParameters) {
this.sortParameters = sortParameters;
unsafeIntermediateFileMerger = new UnsafeIntermediateMerger(sortParameters);
String storeLocation = CarbonDataProcessorUtil.getLocalDataFolderLocation(sortParameters.getDatabaseName(), sortParameters.getTableName(), String.valueOf(sortParameters.getTaskNo()), sortParameters.getPartitionID(), sortParameters.getSegmentId() + "", false);
// Set the data file location
String dataFolderLocation = storeLocation + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION;
finalMerger = new UnsafeSingleThreadFinalSortFilesMerger(sortParameters, sortParameters.getTempFileLocation());
}
use of org.apache.carbondata.processing.newflow.sort.unsafe.merger.UnsafeIntermediateMerger in project carbondata by apache.
the class UnsafeParallelReadMergeSorterWithBucketingImpl method sort.
@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
UnsafeSortDataRows[] sortDataRows = new UnsafeSortDataRows[bucketingInfo.getNumberOfBuckets()];
UnsafeIntermediateMerger[] intermediateFileMergers = new UnsafeIntermediateMerger[sortDataRows.length];
int inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB();
inMemoryChunkSizeInMB = inMemoryChunkSizeInMB / bucketingInfo.getNumberOfBuckets();
if (inMemoryChunkSizeInMB < 5) {
inMemoryChunkSizeInMB = 5;
}
try {
for (int i = 0; i < bucketingInfo.getNumberOfBuckets(); i++) {
SortParameters parameters = sortParameters.getCopy();
parameters.setPartitionID(i + "");
setTempLocation(parameters);
intermediateFileMergers[i] = new UnsafeIntermediateMerger(parameters);
sortDataRows[i] = new UnsafeSortDataRows(parameters, intermediateFileMergers[i], inMemoryChunkSizeInMB);
sortDataRows[i].initialize();
}
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
this.executorService = Executors.newFixedThreadPool(iterators.length);
final int batchSize = CarbonProperties.getInstance().getBatchSize();
try {
for (int i = 0; i < iterators.length; i++) {
executorService.submit(new SortIteratorThread(iterators[i], sortDataRows));
}
executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.DAYS);
processRowToNextStep(sortDataRows, sortParameters);
} catch (Exception e) {
throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
}
try {
for (int i = 0; i < intermediateFileMergers.length; i++) {
intermediateFileMergers[i].finish();
}
} catch (Exception e) {
throw new CarbonDataLoadingException(e);
}
Iterator<CarbonRowBatch>[] batchIterator = new Iterator[bucketingInfo.getNumberOfBuckets()];
for (int i = 0; i < sortDataRows.length; i++) {
batchIterator[i] = new MergedDataIterator(String.valueOf(i), batchSize, intermediateFileMergers[i]);
}
return batchIterator;
}
Aggregations