Search in sources :

Example 1 with CarbonIterator

use of org.apache.carbondata.common.CarbonIterator in project carbondata by apache.

the class UnsafeBatchParallelReadMergeSorterImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    this.executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(this.executorService);
    int batchSize = CarbonProperties.getInstance().getBatchSize();
    final SortBatchHolder sortBatchHolder = new SortBatchHolder(sortParameters, iterators.length, this.threadStatusObserver);
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.submit(new SortIteratorThread(iterators[i], sortBatchHolder, batchSize, rowCounter, this.threadStatusObserver));
        }
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    // Creates the iterator to read from merge sorter.
    Iterator<CarbonSortBatch> batchIterator = new CarbonIterator<CarbonSortBatch>() {

        @Override
        public boolean hasNext() {
            return sortBatchHolder.hasNext();
        }

        @Override
        public CarbonSortBatch next() {
            return new CarbonSortBatch(sortBatchHolder.next());
        }
    };
    return new Iterator[] { batchIterator };
}
Also used : CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException) CarbonIterator(org.apache.carbondata.common.CarbonIterator) CarbonIterator(org.apache.carbondata.common.CarbonIterator) Iterator(java.util.Iterator) CarbonSortBatch(org.apache.carbondata.processing.newflow.row.CarbonSortBatch) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException) CarbonDataWriterException(org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException) CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException)

Example 2 with CarbonIterator

use of org.apache.carbondata.common.CarbonIterator in project carbondata by apache.

the class UnsafeParallelReadMergeSorterImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    int inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB();
    UnsafeSortDataRows sortDataRow = new UnsafeSortDataRows(sortParameters, unsafeIntermediateFileMerger, inMemoryChunkSizeInMB);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        sortDataRow.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    this.executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(this.executorService);
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.submit(new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter, this.threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRow, sortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        unsafeIntermediateFileMerger.finish();
        List<UnsafeCarbonRowPage> rowPages = unsafeIntermediateFileMerger.getRowPages();
        finalMerger.startFinalMerge(rowPages.toArray(new UnsafeCarbonRowPage[rowPages.size()]), unsafeIntermediateFileMerger.getMergedPages());
    } catch (CarbonDataWriterException e) {
        throw new CarbonDataLoadingException(e);
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    // Creates the iterator to read from merge sorter.
    Iterator<CarbonRowBatch> batchIterator = new CarbonIterator<CarbonRowBatch>() {

        @Override
        public boolean hasNext() {
            return finalMerger.hasNext();
        }

        @Override
        public CarbonRowBatch next() {
            int counter = 0;
            CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
            while (finalMerger.hasNext() && counter < batchSize) {
                rowBatch.addRow(new CarbonRow(finalMerger.next()));
                counter++;
            }
            return rowBatch;
        }
    };
    return new Iterator[] { batchIterator };
}
Also used : CarbonRowBatch(org.apache.carbondata.processing.newflow.row.CarbonRowBatch) CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException) CarbonRow(org.apache.carbondata.processing.newflow.row.CarbonRow) UnsafeCarbonRowPage(org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeCarbonRowPage) CarbonDataWriterException(org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException) CarbonDataWriterException(org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException) CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException) CarbonIterator(org.apache.carbondata.common.CarbonIterator) CarbonIterator(org.apache.carbondata.common.CarbonIterator) Iterator(java.util.Iterator) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException) UnsafeSortDataRows(org.apache.carbondata.processing.newflow.sort.unsafe.UnsafeSortDataRows)

Example 3 with CarbonIterator

use of org.apache.carbondata.common.CarbonIterator in project carbondata by apache.

the class ParallelReadMergeSorterImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    SortDataRows sortDataRow = new SortDataRows(sortParameters, intermediateFileMerger);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        sortDataRow.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    this.executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(executorService);
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.submit(new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter, threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRow, sortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        intermediateFileMerger.finish();
        intermediateFileMerger = null;
        finalMerger.startFinalMerge();
    } catch (CarbonDataWriterException e) {
        throw new CarbonDataLoadingException(e);
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    // Creates the iterator to read from merge sorter.
    Iterator<CarbonRowBatch> batchIterator = new CarbonIterator<CarbonRowBatch>() {

        @Override
        public boolean hasNext() {
            return finalMerger.hasNext();
        }

        @Override
        public CarbonRowBatch next() {
            int counter = 0;
            CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
            while (finalMerger.hasNext() && counter < batchSize) {
                rowBatch.addRow(new CarbonRow(finalMerger.next()));
                counter++;
            }
            return rowBatch;
        }
    };
    return new Iterator[] { batchIterator };
}
Also used : CarbonRowBatch(org.apache.carbondata.processing.newflow.row.CarbonRowBatch) CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException) CarbonRow(org.apache.carbondata.processing.newflow.row.CarbonRow) CarbonDataWriterException(org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException) CarbonDataWriterException(org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException) CarbonDataLoadingException(org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException) SortDataRows(org.apache.carbondata.processing.sortandgroupby.sortdata.SortDataRows) CarbonIterator(org.apache.carbondata.common.CarbonIterator) CarbonIterator(org.apache.carbondata.common.CarbonIterator) Iterator(java.util.Iterator) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException)

Example 4 with CarbonIterator

use of org.apache.carbondata.common.CarbonIterator in project carbondata by apache.

the class CarbondataRecordSet method cursor.

/**
 * get data blocks via Carbondata QueryModel API.
 */
@Override
public RecordCursor cursor() {
    CarbonLocalInputSplit carbonLocalInputSplit = split.getLocalInputSplit();
    List<CarbonInputSplit> splitList = new ArrayList<>(1);
    splitList.add(CarbonLocalInputSplit.convertSplit(carbonLocalInputSplit));
    List<TableBlockInfo> tableBlockInfoList = CarbonInputSplit.createBlocks(splitList);
    queryModel.setTableBlockInfos(tableBlockInfoList);
    queryExecutor = QueryExecutorFactory.getQueryExecutor(queryModel);
    try {
        readSupport.initialize(queryModel.getProjectionColumns(), queryModel.getTable());
        CarbonIterator iterator = queryExecutor.execute(queryModel);
        PrestoCarbonVectorizedRecordReader vectorReader = new PrestoCarbonVectorizedRecordReader(queryExecutor, queryModel, (AbstractDetailQueryResultIterator) iterator);
        return new CarbondataRecordCursor(readSupport, vectorReader, columns, split);
    } catch (QueryExecutionException e) {
        throw new RuntimeException(e.getMessage(), e);
    } catch (Exception ex) {
        throw new RuntimeException(ex.getMessage(), ex);
    }
}
Also used : TableBlockInfo(org.apache.carbondata.core.datastore.block.TableBlockInfo) QueryExecutionException(org.apache.carbondata.core.scan.executor.exception.QueryExecutionException) CarbonIterator(org.apache.carbondata.common.CarbonIterator) CarbonLocalInputSplit(org.apache.carbondata.presto.impl.CarbonLocalInputSplit) ArrayList(java.util.ArrayList) CarbonInputSplit(org.apache.carbondata.hadoop.CarbonInputSplit) QueryExecutionException(org.apache.carbondata.core.scan.executor.exception.QueryExecutionException)

Example 5 with CarbonIterator

use of org.apache.carbondata.common.CarbonIterator in project carbondata by apache.

the class UnsafeParallelReadMergeSorterImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    int inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB();
    UnsafeSortDataRows sortDataRow = new UnsafeSortDataRows(sortParameters, unsafeIntermediateFileMerger, inMemoryChunkSizeInMB);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        sortDataRow.initialize();
    } catch (MemoryException e) {
        throw new CarbonDataLoadingException(e);
    }
    this.executorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("UnsafeParallelSorterPool:" + sortParameters.getTableName()));
    this.threadStatusObserver = new ThreadStatusObserver(executorService);
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.execute(new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter, this.threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRow, sortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        unsafeIntermediateFileMerger.finish();
        List<UnsafeCarbonRowPage> rowPages = unsafeIntermediateFileMerger.getRowPages();
        finalMerger.startFinalMerge(rowPages.toArray(new UnsafeCarbonRowPage[rowPages.size()]), unsafeIntermediateFileMerger.getMergedPages());
    } catch (CarbonDataWriterException e) {
        throw new CarbonDataLoadingException(e);
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    // Creates the iterator to read from merge sorter.
    Iterator<CarbonRowBatch> batchIterator = new CarbonIterator<CarbonRowBatch>() {

        @Override
        public boolean hasNext() {
            return finalMerger.hasNext();
        }

        @Override
        public CarbonRowBatch next() {
            int counter = 0;
            CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
            while (finalMerger.hasNext() && counter < batchSize) {
                rowBatch.addRow(new CarbonRow(finalMerger.next()));
                counter++;
            }
            return rowBatch;
        }
    };
    return new Iterator[] { batchIterator };
}
Also used : CarbonRowBatch(org.apache.carbondata.processing.loading.row.CarbonRowBatch) CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) CarbonRow(org.apache.carbondata.core.datastore.row.CarbonRow) UnsafeCarbonRowPage(org.apache.carbondata.processing.loading.sort.unsafe.UnsafeCarbonRowPage) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException) MemoryException(org.apache.carbondata.core.memory.MemoryException) CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) MemoryException(org.apache.carbondata.core.memory.MemoryException) CarbonIterator(org.apache.carbondata.common.CarbonIterator) CarbonThreadFactory(org.apache.carbondata.core.util.CarbonThreadFactory) CarbonIterator(org.apache.carbondata.common.CarbonIterator) Iterator(java.util.Iterator) CarbonSortKeyAndGroupByException(org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException) UnsafeSortDataRows(org.apache.carbondata.processing.loading.sort.unsafe.UnsafeSortDataRows)

Aggregations

CarbonIterator (org.apache.carbondata.common.CarbonIterator)10 Iterator (java.util.Iterator)6 CarbonDataWriterException (org.apache.carbondata.core.datastore.exception.CarbonDataWriterException)3 CarbonThreadFactory (org.apache.carbondata.core.util.CarbonThreadFactory)3 CarbonDataLoadingException (org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException)3 CarbonDataLoadingException (org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException)3 CarbonSortKeyAndGroupByException (org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException)3 CarbonSortKeyAndGroupByException (org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException)3 CarbonDataWriterException (org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException)3 CarbonRow (org.apache.carbondata.core.datastore.row.CarbonRow)2 MemoryException (org.apache.carbondata.core.memory.MemoryException)2 CarbonRowBatch (org.apache.carbondata.processing.loading.row.CarbonRowBatch)2 CarbonRow (org.apache.carbondata.processing.newflow.row.CarbonRow)2 CarbonRowBatch (org.apache.carbondata.processing.newflow.row.CarbonRowBatch)2 IOException (java.io.IOException)1 ArrayList (java.util.ArrayList)1 ExecutionException (java.util.concurrent.ExecutionException)1 ExecutorService (java.util.concurrent.ExecutorService)1 Future (java.util.concurrent.Future)1 TableBlockInfo (org.apache.carbondata.core.datastore.block.TableBlockInfo)1