Search in sources :

Example 1 with SortParameters

use of org.apache.carbondata.processing.sort.sortdata.SortParameters in project carbondata by apache.

the class UnsafeParallelReadMergeSorterWithColumnRangeImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    UnsafeSortDataRows[] sortDataRows = new UnsafeSortDataRows[columnRangeInfo.getNumOfRanges()];
    intermediateFileMergers = new UnsafeIntermediateMerger[columnRangeInfo.getNumOfRanges()];
    SortParameters[] sortParameterArray = new SortParameters[columnRangeInfo.getNumOfRanges()];
    try {
        for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
            SortParameters parameters = originSortParameters.getCopy();
            parameters.setPartitionID(i + "");
            parameters.setRangeId(i);
            sortParameterArray[i] = parameters;
            setTempLocation(parameters);
            intermediateFileMergers[i] = new UnsafeIntermediateMerger(parameters);
            sortDataRows[i] = new UnsafeSortDataRows(parameters, intermediateFileMergers[i], inMemoryChunkSizeInMB);
            sortDataRows[i].initialize();
        }
    } catch (Exception e) {
        throw new CarbonDataLoadingException(e);
    }
    ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(executorService);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, rowCounter, this.insideRowCounterList, this.threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRows, originSortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        for (int i = 0; i < intermediateFileMergers.length; i++) {
            intermediateFileMergers[i].finish();
        }
    } catch (Exception e) {
        throw new CarbonDataLoadingException(e);
    }
    Iterator<CarbonRowBatch>[] batchIterator = new Iterator[columnRangeInfo.getNumOfRanges()];
    for (int i = 0; i < sortDataRows.length; i++) {
        batchIterator[i] = new MergedDataIterator(sortParameterArray[i], batchSize, intermediateFileMergers[i]);
    }
    return batchIterator;
}
Also used : CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) UnsafeIntermediateMerger(org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeIntermediateMerger) CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) SortParameters(org.apache.carbondata.processing.sort.sortdata.SortParameters) ExecutorService(java.util.concurrent.ExecutorService) CarbonIterator(org.apache.carbondata.common.CarbonIterator) Iterator(java.util.Iterator) UnsafeSortDataRows(org.apache.carbondata.processing.loading.sort.unsafe.UnsafeSortDataRows)

Example 2 with SortParameters

use of org.apache.carbondata.processing.sort.sortdata.SortParameters in project carbondata by apache.

the class ParallelReadMergeSorterWithColumnRangeImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    SortDataRows[] sortDataRows = new SortDataRows[columnRangeInfo.getNumOfRanges()];
    intermediateFileMergers = new SortIntermediateFileMerger[columnRangeInfo.getNumOfRanges()];
    SortParameters[] sortParameterArray = new SortParameters[columnRangeInfo.getNumOfRanges()];
    for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
        SortParameters parameters = originSortParameters.getCopy();
        parameters.setPartitionID(i + "");
        parameters.setRangeId(i);
        sortParameterArray[i] = parameters;
        setTempLocation(parameters);
        parameters.setBufferSize(sortBufferSize);
        intermediateFileMergers[i] = new SortIntermediateFileMerger(parameters);
        sortDataRows[i] = new SortDataRows(parameters, intermediateFileMergers[i]);
        sortDataRows[i].initialize();
    }
    ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(executorService);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        // dispatch rows to sortDataRows by range id
        for (int i = 0; i < iterators.length; i++) {
            executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, rowCounter, this.insideRowCounterList, this.threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRows, originSortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        for (int i = 0; i < intermediateFileMergers.length; i++) {
            intermediateFileMergers[i].finish();
        }
    } catch (CarbonDataWriterException | CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    Iterator<CarbonRowBatch>[] batchIterator = new Iterator[columnRangeInfo.getNumOfRanges()];
    for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
        batchIterator[i] = new MergedDataIterator(sortParameterArray[i], batchSize);
    }
    return batchIterator;
}
Also used : CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) SortIntermediateFileMerger(org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) SortDataRows(org.apache.carbondata.processing.sort.sortdata.SortDataRows) SortParameters(org.apache.carbondata.processing.sort.sortdata.SortParameters) ExecutorService(java.util.concurrent.ExecutorService) CarbonIterator(org.apache.carbondata.common.CarbonIterator) Iterator(java.util.Iterator) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException)

Example 3 with SortParameters

use of org.apache.carbondata.processing.sort.sortdata.SortParameters in project carbondata by apache.

the class SortProcessorStepImpl method initialize.

@Override
public void initialize() throws IOException {
    super.initialize();
    child.initialize();
    SortParameters sortParameters = SortParameters.createSortParameters(configuration);
    sorter = SorterFactory.createSorter(configuration, rowCounter);
    sorter.initialize(sortParameters);
}
Also used : SortParameters(org.apache.carbondata.processing.sort.sortdata.SortParameters)

Aggregations

SortParameters (org.apache.carbondata.processing.sort.sortdata.SortParameters)3 Iterator (java.util.Iterator)2 ExecutorService (java.util.concurrent.ExecutorService)2 CarbonIterator (org.apache.carbondata.common.CarbonIterator)2 CarbonDataLoadingException (org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException)2 CarbonDataWriterException (org.apache.carbondata.core.datastore.exception.CarbonDataWriterException)1 UnsafeSortDataRows (org.apache.carbondata.processing.loading.sort.unsafe.UnsafeSortDataRows)1 UnsafeIntermediateMerger (org.apache.carbondata.processing.loading.sort.unsafe.merger.UnsafeIntermediateMerger)1 CarbonSortKeyAndGroupByException (org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException)1 SortDataRows (org.apache.carbondata.processing.sort.sortdata.SortDataRows)1 SortIntermediateFileMerger (org.apache.carbondata.processing.sort.sortdata.SortIntermediateFileMerger)1