Examples with CarbonOutputIteratorWrapper org.apache.carbondata.processing.loading.iterator.CarbonOutputIteratorWrapper used on opensource projects

Example 1 with CarbonOutputIteratorWrapper

use of org.apache.carbondata.processing.loading.iterator.CarbonOutputIteratorWrapper in project carbondata by apache.

the class CarbonTableOutputFormat method getRecordWriter.

@Override
public RecordWriter<NullWritable, ObjectArrayWritable> getRecordWriter(TaskAttemptContext taskAttemptContext) throws IOException {
    final CarbonLoadModel loadModel = getLoadModel(taskAttemptContext.getConfiguration());
    loadModel.setTaskNo(taskAttemptContext.getConfiguration().get("carbon.outputformat.taskno", String.valueOf(System.nanoTime())));
    loadModel.setDataWritePath(taskAttemptContext.getConfiguration().get("carbon.outputformat.writepath"));
    final String[] tempStoreLocations = getTempStoreLocations(taskAttemptContext);
    final CarbonOutputIteratorWrapper iteratorWrapper = new CarbonOutputIteratorWrapper();
    final DataLoadExecutor dataLoadExecutor = new DataLoadExecutor();
    ExecutorService executorService = Executors.newFixedThreadPool(1, new CarbonThreadFactory("CarbonRecordWriter:" + loadModel.getTableName()));
    ;
    // It should be started in new thread as the underlying iterator uses blocking queue.
    Future future = executorService.submit(new Thread() {

        @Override
        public void run() {
            try {
                dataLoadExecutor.execute(loadModel, tempStoreLocations, new CarbonIterator[] { iteratorWrapper });
            } catch (Exception e) {
                dataLoadExecutor.close();
                // clean up the folders and files created locally for data load operation
                TableProcessingOperations.deleteLocalDataLoadFolderLocation(loadModel, false, false);
                throw new RuntimeException(e);
            }
        }
    });
    return new CarbonRecordWriter(iteratorWrapper, dataLoadExecutor, loadModel, future, executorService);
}