use of org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.
the class UnsafeIntermediateMerger method finish.
public void finish() throws CarbonSortKeyAndGroupByException {
try {
executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.DAYS);
} catch (InterruptedException e) {
throw new CarbonSortKeyAndGroupByException("Problem while shutdown the server ", e);
}
checkForFailure();
}
use of org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.
the class ParallelReadMergeSorterImpl method processRowToNextStep.
/**
* Below method will be used to process data to next step
*/
private boolean processRowToNextStep(SortDataRows sortDataRows, SortParameters parameters) throws CarbonDataLoadingException {
if (null == sortDataRows) {
LOGGER.info("Record Processed For table: " + parameters.getTableName());
LOGGER.info("Number of Records was Zero");
String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
LOGGER.info(logMessage);
return false;
}
try {
// start sorting
sortDataRows.startSorting();
// check any more rows are present
LOGGER.info("Record Processed For table: " + parameters.getTableName());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
return false;
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
}
use of org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.
the class ParallelReadMergeSorterImpl method sort.
@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
SortDataRows sortDataRow = new SortDataRows(sortParameters, intermediateFileMerger);
final int batchSize = CarbonProperties.getInstance().getBatchSize();
try {
sortDataRow.initialize();
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
this.executorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("SafeParallelSorterPool:" + sortParameters.getTableName()));
this.threadStatusObserver = new ThreadStatusObserver(executorService);
try {
for (int i = 0; i < iterators.length; i++) {
executorService.execute(new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter, threadStatusObserver));
}
executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.DAYS);
processRowToNextStep(sortDataRow, sortParameters);
} catch (Exception e) {
checkError();
throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
}
checkError();
try {
intermediateFileMerger.finish();
intermediateFileMerger = null;
finalMerger.startFinalMerge();
} catch (CarbonDataWriterException e) {
throw new CarbonDataLoadingException(e);
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
// Creates the iterator to read from merge sorter.
Iterator<CarbonRowBatch> batchIterator = new CarbonIterator<CarbonRowBatch>() {
@Override
public boolean hasNext() {
return finalMerger.hasNext();
}
@Override
public CarbonRowBatch next() {
int counter = 0;
CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
while (finalMerger.hasNext() && counter < batchSize) {
rowBatch.addRow(new CarbonRow(finalMerger.next()));
counter++;
}
return rowBatch;
}
};
return new Iterator[] { batchIterator };
}
use of org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.
the class ParallelReadMergeSorterWithColumnRangeImpl method sort.
@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
SortDataRows[] sortDataRows = new SortDataRows[columnRangeInfo.getNumOfRanges()];
intermediateFileMergers = new SortIntermediateFileMerger[columnRangeInfo.getNumOfRanges()];
SortParameters[] sortParameterArray = new SortParameters[columnRangeInfo.getNumOfRanges()];
try {
for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
SortParameters parameters = originSortParameters.getCopy();
parameters.setPartitionID(i + "");
parameters.setRangeId(i);
sortParameterArray[i] = parameters;
setTempLocation(parameters);
parameters.setBufferSize(sortBufferSize);
intermediateFileMergers[i] = new SortIntermediateFileMerger(parameters);
sortDataRows[i] = new SortDataRows(parameters, intermediateFileMergers[i]);
sortDataRows[i].initialize();
}
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
this.threadStatusObserver = new ThreadStatusObserver(executorService);
final int batchSize = CarbonProperties.getInstance().getBatchSize();
try {
// dispatch rows to sortDataRows by range id
for (int i = 0; i < iterators.length; i++) {
executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, rowCounter, this.insideRowCounterList, this.threadStatusObserver));
}
executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.DAYS);
processRowToNextStep(sortDataRows, originSortParameters);
} catch (Exception e) {
checkError();
throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
}
checkError();
try {
for (int i = 0; i < intermediateFileMergers.length; i++) {
intermediateFileMergers[i].finish();
}
} catch (CarbonDataWriterException e) {
throw new CarbonDataLoadingException(e);
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
Iterator<CarbonRowBatch>[] batchIterator = new Iterator[columnRangeInfo.getNumOfRanges()];
for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
batchIterator[i] = new MergedDataIterator(sortParameterArray[i], batchSize);
}
return batchIterator;
}
use of org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.
the class ParallelReadMergeSorterWithColumnRangeImpl method processRowToNextStep.
/**
* Below method will be used to process data to next step
*/
private boolean processRowToNextStep(SortDataRows[] sortDataRows, SortParameters parameters) throws CarbonDataLoadingException {
if (null == sortDataRows || sortDataRows.length == 0) {
LOGGER.info("Record Processed For table: " + parameters.getTableName());
LOGGER.info("Number of Records was Zero");
String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
LOGGER.info(logMessage);
return false;
}
try {
for (int i = 0; i < sortDataRows.length; i++) {
// start sorting
sortDataRows[i].startSorting();
}
// check any more rows are present
LOGGER.info("Record Processed For table: " + parameters.getTableName());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
return false;
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
}
Aggregations