Search in sources :

Example 1 with SortDataRows

use of org.apache.carbondata.processing.sort.sortdata.SortDataRows in project carbondata by apache.

the class CompactionResultSortProcessor method initSortDataRows.

/**
 * create an instance of sort data rows
 */
private void initSortDataRows() throws Exception {
    measureCount = carbonTable.getMeasureByTableName(tableName).size();
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);
    noDictionaryColMapping = new boolean[dimensions.size()];
    int i = 0;
    for (CarbonDimension dimension : dimensions) {
        if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
            i++;
            continue;
        }
        noDictionaryColMapping[i++] = true;
        noDictionaryCount++;
    }
    dimensionColumnCount = dimensions.size();
    sortParameters = createSortParameters();
    intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);
    // TODO: Now it is only supported onheap merge, but we can have unsafe merge
    // as well by using UnsafeSortDataRows.
    this.sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);
    try {
        this.sortDataRows.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
        LOGGER.error(e);
        throw new Exception("Error initializing sort data rows object during compaction: " + e.getMessage());
    }
}
Also used : SortDataRows(org.apache.carbondata.processing.sort.sortdata.SortDataRows) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException) SortIntermediateFileMerger(org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) IOException(java.io.IOException) CarbonDimension(org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension)

Example 2 with SortDataRows

use of org.apache.carbondata.processing.sort.sortdata.SortDataRows in project carbondata by apache.

the class ParallelReadMergeSorterImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    SortDataRows sortDataRow = new SortDataRows(sortParameters, intermediateFileMerger);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        sortDataRow.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    this.executorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("SafeParallelSorterPool:" + sortParameters.getTableName()));
    this.threadStatusObserver = new ThreadStatusObserver(executorService);
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.execute(new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter, threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRow, sortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        intermediateFileMerger.finish();
        intermediateFileMerger = null;
        finalMerger.startFinalMerge();
    } catch (CarbonDataWriterException e) {
        throw new CarbonDataLoadingException(e);
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    // Creates the iterator to read from merge sorter.
    Iterator<CarbonRowBatch> batchIterator = new CarbonIterator<CarbonRowBatch>() {

        @Override
        public boolean hasNext() {
            return finalMerger.hasNext();
        }

        @Override
        public CarbonRowBatch next() {
            int counter = 0;
            CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
            while (finalMerger.hasNext() && counter < batchSize) {
                rowBatch.addRow(new CarbonRow(finalMerger.next()));
                counter++;
            }
            return rowBatch;
        }
    };
    return new Iterator[] { batchIterator };
}
Also used : CarbonRowBatch(org.apache.carbondata.processing.loading.row.CarbonRowBatch) CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) CarbonRow(org.apache.carbondata.core.datastore.row.CarbonRow) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException) CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) SortDataRows(org.apache.carbondata.processing.sort.sortdata.SortDataRows) CarbonIterator(org.apache.carbondata.common.CarbonIterator) CarbonThreadFactory(org.apache.carbondata.core.util.CarbonThreadFactory) CarbonIterator(org.apache.carbondata.common.CarbonIterator) Iterator(java.util.Iterator) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException)

Example 3 with SortDataRows

use of org.apache.carbondata.processing.sort.sortdata.SortDataRows in project carbondata by apache.

the class ParallelReadMergeSorterWithColumnRangeImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    SortDataRows[] sortDataRows = new SortDataRows[columnRangeInfo.getNumOfRanges()];
    intermediateFileMergers = new SortIntermediateFileMerger[columnRangeInfo.getNumOfRanges()];
    SortParameters[] sortParameterArray = new SortParameters[columnRangeInfo.getNumOfRanges()];
    try {
        for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
            SortParameters parameters = originSortParameters.getCopy();
            parameters.setPartitionID(i + "");
            parameters.setRangeId(i);
            sortParameterArray[i] = parameters;
            setTempLocation(parameters);
            parameters.setBufferSize(sortBufferSize);
            intermediateFileMergers[i] = new SortIntermediateFileMerger(parameters);
            sortDataRows[i] = new SortDataRows(parameters, intermediateFileMergers[i]);
            sortDataRows[i].initialize();
        }
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(executorService);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        // dispatch rows to sortDataRows by range id
        for (int i = 0; i < iterators.length; i++) {
            executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, rowCounter, this.insideRowCounterList, this.threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRows, originSortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        for (int i = 0; i < intermediateFileMergers.length; i++) {
            intermediateFileMergers[i].finish();
        }
    } catch (CarbonDataWriterException e) {
        throw new CarbonDataLoadingException(e);
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    Iterator<CarbonRowBatch>[] batchIterator = new Iterator[columnRangeInfo.getNumOfRanges()];
    for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
        batchIterator[i] = new MergedDataIterator(sortParameterArray[i], batchSize);
    }
    return batchIterator;
}
Also used : CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) SortIntermediateFileMerger(org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) SortDataRows(org.apache.carbondata.processing.sort.sortdata.SortDataRows) SortParameters(org.apache.carbondata.processing.sort.sortdata.SortParameters) ExecutorService(java.util.concurrent.ExecutorService) CarbonIterator(org.apache.carbondata.common.CarbonIterator) Iterator(java.util.Iterator) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException)

Aggregations

CarbonDataWriterException (org.apache.carbondata.core.datastore.exception.CarbonDataWriterException)3 CarbonSortKeyAndGroupByException (org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException)3 SortDataRows (org.apache.carbondata.processing.sort.sortdata.SortDataRows)3 Iterator (java.util.Iterator)2 CarbonIterator (org.apache.carbondata.common.CarbonIterator)2 CarbonDataLoadingException (org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException)2 SortIntermediateFileMerger (org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger)2 IOException (java.io.IOException)1 ExecutorService (java.util.concurrent.ExecutorService)1 CarbonRow (org.apache.carbondata.core.datastore.row.CarbonRow)1 CarbonDimension (org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension)1 CarbonThreadFactory (org.apache.carbondata.core.util.CarbonThreadFactory)1 CarbonRowBatch (org.apache.carbondata.processing.loading.row.CarbonRowBatch)1 SortParameters (org.apache.carbondata.processing.sort.sortdata.SortParameters)1