use of org.apache.carbondata.processing.sort.sortdata.SortDataRows in project carbondata by apache.
the class CompactionResultSortProcessor method initSortDataRows.
/**
* create an instance of sort data rows
*/
private void initSortDataRows() throws Exception {
measureCount = carbonTable.getMeasureByTableName(tableName).size();
List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);
noDictionaryColMapping = new boolean[dimensions.size()];
int i = 0;
for (CarbonDimension dimension : dimensions) {
if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
i++;
continue;
}
noDictionaryColMapping[i++] = true;
noDictionaryCount++;
}
dimensionColumnCount = dimensions.size();
sortParameters = createSortParameters();
intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);
// TODO: Now it is only supported onheap merge, but we can have unsafe merge
// as well by using UnsafeSortDataRows.
this.sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);
try {
this.sortDataRows.initialize();
} catch (CarbonSortKeyAndGroupByException e) {
LOGGER.error(e);
throw new Exception("Error initializing sort data rows object during compaction: " + e.getMessage());
}
}
use of org.apache.carbondata.processing.sort.sortdata.SortDataRows in project carbondata by apache.
the class ParallelReadMergeSorterImpl method sort.
@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
SortDataRows sortDataRow = new SortDataRows(sortParameters, intermediateFileMerger);
final int batchSize = CarbonProperties.getInstance().getBatchSize();
try {
sortDataRow.initialize();
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
this.executorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("SafeParallelSorterPool:" + sortParameters.getTableName()));
this.threadStatusObserver = new ThreadStatusObserver(executorService);
try {
for (int i = 0; i < iterators.length; i++) {
executorService.execute(new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter, threadStatusObserver));
}
executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.DAYS);
processRowToNextStep(sortDataRow, sortParameters);
} catch (Exception e) {
checkError();
throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
}
checkError();
try {
intermediateFileMerger.finish();
intermediateFileMerger = null;
finalMerger.startFinalMerge();
} catch (CarbonDataWriterException e) {
throw new CarbonDataLoadingException(e);
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
// Creates the iterator to read from merge sorter.
Iterator<CarbonRowBatch> batchIterator = new CarbonIterator<CarbonRowBatch>() {
@Override
public boolean hasNext() {
return finalMerger.hasNext();
}
@Override
public CarbonRowBatch next() {
int counter = 0;
CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
while (finalMerger.hasNext() && counter < batchSize) {
rowBatch.addRow(new CarbonRow(finalMerger.next()));
counter++;
}
return rowBatch;
}
};
return new Iterator[] { batchIterator };
}
use of org.apache.carbondata.processing.sort.sortdata.SortDataRows in project carbondata by apache.
the class ParallelReadMergeSorterWithColumnRangeImpl method sort.
@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
SortDataRows[] sortDataRows = new SortDataRows[columnRangeInfo.getNumOfRanges()];
intermediateFileMergers = new SortIntermediateFileMerger[columnRangeInfo.getNumOfRanges()];
SortParameters[] sortParameterArray = new SortParameters[columnRangeInfo.getNumOfRanges()];
try {
for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
SortParameters parameters = originSortParameters.getCopy();
parameters.setPartitionID(i + "");
parameters.setRangeId(i);
sortParameterArray[i] = parameters;
setTempLocation(parameters);
parameters.setBufferSize(sortBufferSize);
intermediateFileMergers[i] = new SortIntermediateFileMerger(parameters);
sortDataRows[i] = new SortDataRows(parameters, intermediateFileMergers[i]);
sortDataRows[i].initialize();
}
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
this.threadStatusObserver = new ThreadStatusObserver(executorService);
final int batchSize = CarbonProperties.getInstance().getBatchSize();
try {
// dispatch rows to sortDataRows by range id
for (int i = 0; i < iterators.length; i++) {
executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, rowCounter, this.insideRowCounterList, this.threadStatusObserver));
}
executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.DAYS);
processRowToNextStep(sortDataRows, originSortParameters);
} catch (Exception e) {
checkError();
throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
}
checkError();
try {
for (int i = 0; i < intermediateFileMergers.length; i++) {
intermediateFileMergers[i].finish();
}
} catch (CarbonDataWriterException e) {
throw new CarbonDataLoadingException(e);
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
Iterator<CarbonRowBatch>[] batchIterator = new Iterator[columnRangeInfo.getNumOfRanges()];
for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
batchIterator[i] = new MergedDataIterator(sortParameterArray[i], batchSize);
}
return batchIterator;
}
Aggregations