Examples with CarbonIterator org.apache.carbondata.common.CarbonIterator used on opensource projects

Example 6 with CarbonIterator

use of org.apache.carbondata.common.CarbonIterator in project carbondata by apache.

the class ParallelReadMergeSorterImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    SortDataRows sortDataRow = new SortDataRows(sortParameters, intermediateFileMerger);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        sortDataRow.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    this.executorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("SafeParallelSorterPool:" + sortParameters.getTableName()));
    this.threadStatusObserver = new ThreadStatusObserver(executorService);
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.execute(new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter, threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRow, sortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        intermediateFileMerger.finish();
        intermediateFileMerger = null;
        finalMerger.startFinalMerge();
    } catch (CarbonDataWriterException e) {
        throw new CarbonDataLoadingException(e);
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    // Creates the iterator to read from merge sorter.
    Iterator<CarbonRowBatch> batchIterator = new CarbonIterator<CarbonRowBatch>() {

        @Override
        public boolean hasNext() {
            return finalMerger.hasNext();
        }

        @Override
        public CarbonRowBatch next() {
            int counter = 0;
            CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
            while (finalMerger.hasNext() && counter < batchSize) {
                rowBatch.addRow(new CarbonRow(finalMerger.next()));
                counter++;
            }
            return rowBatch;
        }
    };
    return new Iterator[] { batchIterator };
}

Example 7 with CarbonIterator

use of org.apache.carbondata.common.CarbonIterator in project carbondata by apache.

the class InputProcessorStepImpl method close.

@Override
public void close() {
    if (!closed) {
        super.close();
        executorService.shutdown();
        for (CarbonIterator inputIterator : inputIterators) {
            inputIterator.close();
        }
    }
}

Example 8 with CarbonIterator

use of org.apache.carbondata.common.CarbonIterator in project carbondata by apache.

the class InputProcessorStepImpl method close.

@Override
public void close() {
    if (!closed) {
        super.close();
        executorService.shutdown();
        for (CarbonIterator inputIterator : inputIterators) {
            inputIterator.close();
        }
    }
}

Example 9 with CarbonIterator

use of org.apache.carbondata.common.CarbonIterator in project carbondata by apache.

the class UnsafeBatchParallelReadMergeSorterImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    this.executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(this.executorService);
    int batchSize = CarbonProperties.getInstance().getBatchSize();
    final SortBatchHolder sortBatchHolder = new SortBatchHolder(sortParameters, iterators.length, this.threadStatusObserver);
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.execute(new SortIteratorThread(iterators[i], sortBatchHolder, batchSize, rowCounter, this.threadStatusObserver));
        }
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    // Creates the iterator to read from merge sorter.
    Iterator<CarbonSortBatch> batchIterator = new CarbonIterator<CarbonSortBatch>() {

        @Override
        public boolean hasNext() {
            return sortBatchHolder.hasNext();
        }

        @Override
        public CarbonSortBatch next() {
            return new CarbonSortBatch(sortBatchHolder.next());
        }
    };
    return new Iterator[] { batchIterator };
}

Example 10 with CarbonIterator

use of org.apache.carbondata.common.CarbonIterator in project carbondata by apache.

the class CarbonTableOutputFormat method getRecordWriter.

@Override
public RecordWriter<NullWritable, ObjectArrayWritable> getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException {
    final CarbonLoadModel loadModel = getLoadModel(taskAttemptContext.getConfiguration());
    loadModel.setTaskNo(taskAttemptContext.getConfiguration().get("carbon.outputformat.taskno", String.valueOf(System.nanoTime())));
    loadModel.setDataWritePath(taskAttemptContext.getConfiguration().get("carbon.outputformat.writepath"));
    final String[] tempStoreLocations = getTempStoreLocations(taskAttemptContext);
    final CarbonOutputIteratorWrapper iteratorWrapper = new CarbonOutputIteratorWrapper();
    final DataLoadExecutor dataLoadExecutor = new DataLoadExecutor();
    ExecutorService executorService = Executors.newFixedThreadPool(1, new CarbonThreadFactory("CarbonRecordWriter:" + loadModel.getTableName()));
    ;
    // It should be started in new thread as the underlying iterator uses blocking queue.
    Future future = executorService.submit(new Thread() {

        @Override
        public void run() {
            try {
                dataLoadExecutor.execute(loadModel, tempStoreLocations, new CarbonIterator[] { iteratorWrapper });
            } catch (Exception e) {
                dataLoadExecutor.close();
                // clean up the folders and files created locally for data load operation
                TableProcessingOperations.deleteLocalDataLoadFolderLocation(loadModel, false, false);
                throw new RuntimeException(e);
            }
        }
    });
    return new CarbonRecordWriter(iteratorWrapper, dataLoadExecutor, loadModel, future, executorService);
}