use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.
the class ParallelReadMergeSorterImpl method processRowToNextStep.
/**
* Below method will be used to process data to next step
*/
private boolean processRowToNextStep(SortDataRows sortDataRows, SortParameters parameters) throws CarbonDataLoadingException {
if (null == sortDataRows) {
LOGGER.info("Record Processed For table: " + parameters.getTableName());
LOGGER.info("Number of Records was Zero");
String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
LOGGER.info(logMessage);
return false;
}
try {
// start sorting
sortDataRows.startSorting();
// check any more rows are present
LOGGER.info("Record Processed For table: " + parameters.getTableName());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
return false;
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
}
use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.
the class ParallelReadMergeSorterWithBucketingImpl method sort.
@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
SortDataRows[] sortDataRows = new SortDataRows[bucketingInfo.getNumberOfBuckets()];
intermediateFileMergers = new SortIntermediateFileMerger[sortDataRows.length];
try {
for (int i = 0; i < bucketingInfo.getNumberOfBuckets(); i++) {
SortParameters parameters = sortParameters.getCopy();
parameters.setPartitionID(i + "");
setTempLocation(parameters);
parameters.setBufferSize(sortBufferSize);
intermediateFileMergers[i] = new SortIntermediateFileMerger(parameters);
sortDataRows[i] = new SortDataRows(parameters, intermediateFileMergers[i]);
sortDataRows[i].initialize();
}
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
this.executorService = Executors.newFixedThreadPool(iterators.length);
this.threadStatusObserver = new ThreadStatusObserver(this.executorService);
final int batchSize = CarbonProperties.getInstance().getBatchSize();
try {
for (int i = 0; i < iterators.length; i++) {
executorService.submit(new SortIteratorThread(iterators[i], sortDataRows, rowCounter, this.threadStatusObserver));
}
executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.DAYS);
processRowToNextStep(sortDataRows, sortParameters);
} catch (Exception e) {
checkError();
throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
}
checkError();
try {
for (int i = 0; i < intermediateFileMergers.length; i++) {
intermediateFileMergers[i].finish();
}
} catch (CarbonDataWriterException e) {
throw new CarbonDataLoadingException(e);
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
Iterator<CarbonRowBatch>[] batchIterator = new Iterator[bucketingInfo.getNumberOfBuckets()];
for (int i = 0; i < bucketingInfo.getNumberOfBuckets(); i++) {
batchIterator[i] = new MergedDataIterator(String.valueOf(i), batchSize);
}
return batchIterator;
}
use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.
the class ParallelReadMergeSorterWithBucketingImpl method processRowToNextStep.
/**
* Below method will be used to process data to next step
*/
private boolean processRowToNextStep(SortDataRows[] sortDataRows, SortParameters parameters) throws CarbonDataLoadingException {
if (null == sortDataRows || sortDataRows.length == 0) {
LOGGER.info("Record Processed For table: " + parameters.getTableName());
LOGGER.info("Number of Records was Zero");
String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
LOGGER.info(logMessage);
return false;
}
try {
for (int i = 0; i < sortDataRows.length; i++) {
// start sorting
sortDataRows[i].startSorting();
}
// check any more rows are present
LOGGER.info("Record Processed For table: " + parameters.getTableName());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
return false;
} catch (CarbonSortKeyAndGroupByException e) {
throw new CarbonDataLoadingException(e);
}
}
use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.
the class DataWriterProcessorStepImpl method processBatch.
private void processBatch(CarbonRowBatch batch, CarbonFactHandler dataHandler) throws CarbonDataLoadingException {
try {
while (batch.hasNext()) {
CarbonRow row = batch.next();
dataHandler.addDataToStore(row);
readCounter++;
}
} catch (Exception e) {
throw new CarbonDataLoadingException("unable to generate the mdkey", e);
}
rowCounter.getAndAdd(batch.getSize());
}
use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.
the class DummyThread method execute.
@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
Iterator<CarbonRowBatch>[] iterators = child.execute();
this.executorService = Executors.newFixedThreadPool(iterators.length);
try {
for (int i = 0; i < iterators.length; i++) {
executorService.submit(new DummyThread(iterators[i]));
}
executorService.shutdown();
executorService.awaitTermination(2, TimeUnit.DAYS);
} catch (Exception e) {
throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
}
return null;
}
Aggregations