use of org.apache.carbondata.processing.sort.sortdata.SortParameters in project carbondata by apache.
the class UnsafeParallelReadMergeSorterWithColumnRangeImpl method sort.
@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
UnsafeSortDataRows[] sortDataRows = new UnsafeSortDataRows[columnRangeInfo.getNumOfRanges()];
intermediateFileMergers = new UnsafeIntermediateMerger[columnRangeInfo.getNumOfRanges()];
SortParameters[] sortParameterArray = new SortParameters[columnRangeInfo.getNumOfRanges()];
try {
for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
SortParameters parameters = originSortParameters.getCopy();
parameters.setPartitionID(i + "");
parameters.setRangeId(i);
sortParameterArray[i] = parameters;
setTempLocation(parameters);
intermediateFileMergers[i] = new UnsafeIntermediateMerger(parameters);
sortDataRows[i] = new UnsafeSortDataRows(parameters, intermediateFileMergers[i], inMemoryChunkSizeInMB);
sortDataRows[i].initialize();
}
} catch (Exception e) {
throw new CarbonDataLoadingException(e);
}
ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
this.threadStatusObserver = new ThreadStatusObserver(executorService);
final int batchSize = CarbonProperties.getInstance().getBatchSize();
try {
for (int i = 0; i < iterators.length; i++) {
executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, rowCounter, this.insideRowCounterList, this.threadStatusObserver));
}
executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.DAYS);
processRowToNextStep(sortDataRows, originSortParameters);
} catch (Exception e) {
checkError();
throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
}
checkError();
try {
for (int i = 0; i < intermediateFileMergers.length; i++) {
intermediateFileMergers[i].finish();
}
} catch (Exception e) {
throw new CarbonDataLoadingException(e);
}
Iterator<CarbonRowBatch>[] batchIterator = new Iterator[columnRangeInfo.getNumOfRanges()];
for (int i = 0; i < sortDataRows.length; i++) {
batchIterator[i] = new MergedDataIterator(sortParameterArray[i], batchSize, intermediateFileMergers[i]);
}
return batchIterator;
}
use of org.apache.carbondata.processing.sort.sortdata.SortParameters in project carbondata by apache.
the class ParallelReadMergeSorterWithColumnRangeImpl method sort.
@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
SortDataRows[] sortDataRows = new SortDataRows[columnRangeInfo.getNumOfRanges()];
intermediateFileMergers = new SortIntermediateFileMerger[columnRangeInfo.getNumOfRanges()];
SortParameters[] sortParameterArray = new SortParameters[columnRangeInfo.getNumOfRanges()];
for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
SortParameters parameters = originSortParameters.getCopy();
parameters.setPartitionID(i + "");
parameters.setRangeId(i);
sortParameterArray[i] = parameters;
setTempLocation(parameters);
parameters.setBufferSize(sortBufferSize);
intermediateFileMergers[i] = new SortIntermediateFileMerger(parameters);
sortDataRows[i] = new SortDataRows(parameters, intermediateFileMergers[i]);
sortDataRows[i].initialize();
}
ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
this.threadStatusObserver = new ThreadStatusObserver(executorService);
final int batchSize = CarbonProperties.getInstance().getBatchSize();
try {
// dispatch rows to sortDataRows by range id
for (int i = 0; i < iterators.length; i++) {
executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, rowCounter, this.insideRowCounterList, this.threadStatusObserver));
}
executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.DAYS);
processRowToNextStep(sortDataRows, originSortParameters);
} catch (Exception e) {
checkError();
throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
}
checkError();
try {
for (int i = 0; i < intermediateFileMergers.length; i++) {
intermediateFileMergers[i].finish();
}
} catch (CarbonDataWriterException | CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
Iterator<CarbonRowBatch>[] batchIterator = new Iterator[columnRangeInfo.getNumOfRanges()];
for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
batchIterator[i] = new MergedDataIterator(sortParameterArray[i], batchSize);
}
return batchIterator;
}
use of org.apache.carbondata.processing.sort.sortdata.SortParameters in project carbondata by apache.
the class SortProcessorStepImpl method initialize.
@Override
public void initialize() throws IOException {
super.initialize();
child.initialize();
SortParameters sortParameters = SortParameters.createSortParameters(configuration);
sorter = SorterFactory.createSorter(configuration, rowCounter);
sorter.initialize(sortParameters);
}
Aggregations