Search in sources :

Example 6 with CarbonDataLoadingException

use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.

the class ParallelReadMergeSorterImpl method processRowToNextStep.

/**
   * Below method will be used to process data to next step
   */
private boolean processRowToNextStep(SortDataRows sortDataRows, SortParameters parameters) throws CarbonDataLoadingException {
    if (null == sortDataRows) {
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        LOGGER.info("Number of Records was Zero");
        String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
        LOGGER.info(logMessage);
        return false;
    }
    try {
        // start sorting
        sortDataRows.startSorting();
        // check any more rows are present
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        return false;
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
}
Also used : CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException)

Example 7 with CarbonDataLoadingException

use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.

the class ParallelReadMergeSorterWithBucketingImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    SortDataRows[] sortDataRows = new SortDataRows[bucketingInfo.getNumberOfBuckets()];
    intermediateFileMergers = new SortIntermediateFileMerger[sortDataRows.length];
    try {
        for (int i = 0; i < bucketingInfo.getNumberOfBuckets(); i++) {
            SortParameters parameters = sortParameters.getCopy();
            parameters.setPartitionID(i + "");
            setTempLocation(parameters);
            parameters.setBufferSize(sortBufferSize);
            intermediateFileMergers[i] = new SortIntermediateFileMerger(parameters);
            sortDataRows[i] = new SortDataRows(parameters, intermediateFileMergers[i]);
            sortDataRows[i].initialize();
        }
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    this.executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(this.executorService);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.submit(new SortIteratorThread(iterators[i], sortDataRows, rowCounter, this.threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRows, sortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        for (int i = 0; i < intermediateFileMergers.length; i++) {
            intermediateFileMergers[i].finish();
        }
    } catch (CarbonDataWriterException e) {
        throw new CarbonDataLoadingException(e);
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    Iterator<CarbonRowBatch>[] batchIterator = new Iterator[bucketingInfo.getNumberOfBuckets()];
    for (int i = 0; i < bucketingInfo.getNumberOfBuckets(); i++) {
        batchIterator[i] = new MergedDataIterator(String.valueOf(i), batchSize);
    }
    return batchIterator;
}
Also used : CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException) CarbonDataWriterException(org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException) SortIntermediateFileMerger(org.apache.carbondata.processing.sortandgroupby.sortdata.SortIntermediateFileMerger) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException) CarbonDataWriterException(org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException) CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException) SortDataRows(org.apache.carbondata.processing.sortandgroupby.sortdata.SortDataRows) SortParameters(org.apache.carbondata.processing.sortandgroupby.sortdata.SortParameters) CarbonIterator(org.apache.carbondata.common.CarbonIterator) Iterator(java.util.Iterator) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException)

Example 8 with CarbonDataLoadingException

use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.

the class ParallelReadMergeSorterWithBucketingImpl method processRowToNextStep.

/**
   * Below method will be used to process data to next step
   */
private boolean processRowToNextStep(SortDataRows[] sortDataRows, SortParameters parameters) throws CarbonDataLoadingException {
    if (null == sortDataRows || sortDataRows.length == 0) {
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        LOGGER.info("Number of Records was Zero");
        String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
        LOGGER.info(logMessage);
        return false;
    }
    try {
        for (int i = 0; i < sortDataRows.length; i++) {
            // start sorting
            sortDataRows[i].startSorting();
        }
        // check any more rows are present
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        return false;
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
}
Also used : CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException)

Example 9 with CarbonDataLoadingException

use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.

the class DataWriterProcessorStepImpl method processBatch.

private void processBatch(CarbonRowBatch batch, CarbonFactHandler dataHandler) throws CarbonDataLoadingException {
    try {
        while (batch.hasNext()) {
            CarbonRow row = batch.next();
            dataHandler.addDataToStore(row);
            readCounter++;
        }
    } catch (Exception e) {
        throw new CarbonDataLoadingException("unable to generate the mdkey", e);
    }
    rowCounter.getAndAdd(batch.getSize());
}
Also used : CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException) CarbonRow(org.apache.carbondata.processing.newflow.row.CarbonRow) CarbonDataWriterException(org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException) IOException(java.io.IOException) CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException)

Example 10 with CarbonDataLoadingException

use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.

the class DummyThread method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    Iterator<CarbonRowBatch>[] iterators = child.execute();
    this.executorService = Executors.newFixedThreadPool(iterators.length);
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.submit(new DummyThread(iterators[i]));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
    } catch (Exception e) {
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    return null;
}
Also used : CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException) Iterator(java.util.Iterator) CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException)

Aggregations

CarbonDataLoadingException (org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException)20 Iterator (java.util.Iterator)9 CarbonSortKeyAndGroupByException (org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException)8 CarbonDataWriterException (org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException)8 IOException (java.io.IOException)6 CarbonIterator (org.apache.carbondata.common.CarbonIterator)5 CarbonRow (org.apache.carbondata.processing.newflow.row.CarbonRow)5 CarbonRowBatch (org.apache.carbondata.processing.newflow.row.CarbonRowBatch)4 CarbonTableIdentifier (org.apache.carbondata.core.metadata.CarbonTableIdentifier)3 CarbonFactDataHandlerModel (org.apache.carbondata.processing.store.CarbonFactDataHandlerModel)3 KeyGenException (org.apache.carbondata.core.keygenerator.KeyGenException)2 UnsafeSortDataRows (org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeSortDataRows)2 SortDataRows (org.apache.carbondata.processing.sortandgroupby.sortdata.SortDataRows)2 SortParameters (org.apache.carbondata.processing.sortandgroupby.sortdata.SortParameters)2 CarbonFactHandler (org.apache.carbondata.processing.store.CarbonFactHandler)2 ByteArrayOutputStream (java.io.ByteArrayOutputStream)1 DataOutputStream (java.io.DataOutputStream)1 ExecutorService (java.util.concurrent.ExecutorService)1 Future (java.util.concurrent.Future)1 DictionaryGenerationException (org.apache.carbondata.core.devapi.DictionaryGenerationException)1