Examples with CarbonDataLoadingException org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException used on opensource projects

Example 16 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class ParallelReadMergeSorterWithColumnRangeImpl method processRowToNextStep.

/**
 * Below method will be used to process data to next step
 */
private boolean processRowToNextStep(SortDataRows[] sortDataRows, SortParameters parameters) throws CarbonDataLoadingException {
    if (null == sortDataRows || sortDataRows.length == 0) {
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        LOGGER.info("Number of Records was Zero");
        String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
        LOGGER.info(logMessage);
        return false;
    }
    try {
        for (int i = 0; i < sortDataRows.length; i++) {
            // start sorting
            sortDataRows[i].startSorting();
        }
        // check any more rows are present
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        return false;
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
}

Example 17 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method processBatch.

private void processBatch(CarbonRowBatch batch, CarbonFactHandler dataHandler, int iteratorIndex) throws CarbonDataLoadingException {
    try {
        while (batch.hasNext()) {
            CarbonRow row = batch.next();
            CarbonRow converted = convertRow(row);
            dataHandler.addDataToStore(converted);
            readCounter[iteratorIndex]++;
        }
        writeCounter[iteratorIndex] += batch.getSize();
    } catch (Exception e) {
        throw new CarbonDataLoadingException("unable to generate the mdkey", e);
    }
    rowCounter.getAndAdd(batch.getSize());
}

Example 18 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class DataWriterProcessorStepImpl method processRow.

public void processRow(CarbonRow row, CarbonFactHandler dataHandler) throws KeyGenException {
    try {
        readCounter++;
        dataHandler.addDataToStore(row);
    } catch (Exception e) {
        throw new CarbonDataLoadingException("unable to generate the mdkey", e);
    }
    rowCounter.getAndAdd(1);
}

Example 19 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class UnsafeBatchParallelReadMergeSorterImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    this.executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(this.executorService);
    int batchSize = CarbonProperties.getInstance().getBatchSize();
    final SortBatchHolder sortBatchHolder = new SortBatchHolder(sortParameters, iterators.length, this.threadStatusObserver);
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.execute(new SortIteratorThread(iterators[i], sortBatchHolder, batchSize, rowCounter, this.threadStatusObserver));
        }
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    // Creates the iterator to read from merge sorter.
    Iterator<CarbonSortBatch> batchIterator = new CarbonIterator<CarbonSortBatch>() {

        @Override
        public boolean hasNext() {
            return sortBatchHolder.hasNext();
        }

        @Override
        public CarbonSortBatch next() {
            return new CarbonSortBatch(sortBatchHolder.next());
        }
    };
    return new Iterator[] { batchIterator };
}

Example 20 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class UnsafeParallelReadMergeSorterImpl method processRowToNextStep.

/**
 * Below method will be used to process data to next step
 */
private boolean processRowToNextStep(UnsafeSortDataRows sortDataRows, SortParameters parameters) throws CarbonDataLoadingException {
    if (null == sortDataRows) {
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        LOGGER.info("Number of Records was Zero");
        String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
        LOGGER.info(logMessage);
        return false;
    }
    try {
        // start sorting
        sortDataRows.startSorting();
        // check any more rows are present
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        return false;
    } catch (InterruptedException e) {
        throw new CarbonDataLoadingException(e);
    }
}