Examples with CarbonDataWriterException org.apache.carbondata.core.datastore.exception.CarbonDataWriterException used on opensource projects

Example 11 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class ParallelReadMergeSorterImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    this.executorService = Executors.newFixedThreadPool(sortParameters.getNumberOfCores(), new CarbonThreadFactory("SafeParallelSorterPool:" + sortParameters.getTableName(), true));
    this.threadStatusObserver = new ThreadStatusObserver(executorService);
    try {
        for (int i = 0; i < iterators.length; i++) {
            SortDataRows sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);
            sortDataRows.setInstanceId(i);
            executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, batchSize, rowCounter, threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        LOGGER.info("Record Processed For table: " + sortParameters.getTableName());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(sortParameters.getPartitionID(), System.currentTimeMillis());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(sortParameters.getPartitionID(), System.currentTimeMillis());
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        intermediateFileMerger.finish();
        intermediateFileMerger = null;
        finalMerger.startFinalMerge();
    } catch (CarbonDataWriterException e) {
        throw new CarbonDataLoadingException(e);
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    // Creates the iterator to read from merge sorter.
    Iterator<CarbonRowBatch> batchIterator = new CarbonIterator<CarbonRowBatch>() {

        @Override
        public boolean hasNext() {
            return finalMerger.hasNext();
        }

        @Override
        public CarbonRowBatch next() {
            int counter = 0;
            CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
            while (finalMerger.hasNext() && counter < batchSize) {
                rowBatch.addRow(new CarbonRow(finalMerger.next()));
                counter++;
            }
            return rowBatch;
        }
    };
    return new Iterator[] { batchIterator };
}

Example 12 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class ParallelReadMergeSorterWithColumnRangeImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    SortDataRows[] sortDataRows = new SortDataRows[columnRangeInfo.getNumOfRanges()];
    intermediateFileMergers = new SortIntermediateFileMerger[columnRangeInfo.getNumOfRanges()];
    SortParameters[] sortParameterArray = new SortParameters[columnRangeInfo.getNumOfRanges()];
    for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
        SortParameters parameters = originSortParameters.getCopy();
        parameters.setPartitionID(i + "");
        parameters.setRangeId(i);
        sortParameterArray[i] = parameters;
        setTempLocation(parameters);
        parameters.setBufferSize(sortBufferSize);
        intermediateFileMergers[i] = new SortIntermediateFileMerger(parameters);
        sortDataRows[i] = new SortDataRows(parameters, intermediateFileMergers[i]);
        sortDataRows[i].initialize();
    }
    ExecutorService executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(executorService);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        // dispatch rows to sortDataRows by range id
        for (int i = 0; i < iterators.length; i++) {
            executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, rowCounter, this.insideRowCounterList, this.threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRows, originSortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        for (int i = 0; i < intermediateFileMergers.length; i++) {
            intermediateFileMergers[i].finish();
        }
    } catch (CarbonDataWriterException | CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    Iterator<CarbonRowBatch>[] batchIterator = new Iterator[columnRangeInfo.getNumOfRanges()];
    for (int i = 0; i < columnRangeInfo.getNumOfRanges(); i++) {
        batchIterator[i] = new MergedDataIterator(sortParameterArray[i], batchSize);
    }
    return batchIterator;
}

Example 13 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class UnsafeParallelReadMergeSorterImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    int inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB();
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    this.executorService = Executors.newFixedThreadPool(sortParameters.getNumberOfCores(), new CarbonThreadFactory("UnsafeParallelSorterPool:" + sortParameters.getTableName(), true));
    this.threadStatusObserver = new ThreadStatusObserver(executorService);
    try {
        for (int i = 0; i < iterators.length; i++) {
            UnsafeSortDataRows sortDataRows = new UnsafeSortDataRows(sortParameters, unsafeIntermediateFileMerger, inMemoryChunkSizeInMB);
            sortDataRows.setInstanceId(i);
            executorService.execute(new SortIteratorThread(iterators[i], sortDataRows, batchSize, rowCounter, this.threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        if (!sortParameters.getObserver().isFailed()) {
            LOGGER.info("Record Processed For table: " + sortParameters.getTableName());
            CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(sortParameters.getPartitionID(), System.currentTimeMillis());
            CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(sortParameters.getPartitionID(), System.currentTimeMillis());
        }
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        unsafeIntermediateFileMerger.finish();
        List<UnsafeCarbonRowPage> rowPages = unsafeIntermediateFileMerger.getRowPages();
        finalMerger.startFinalMerge(rowPages.toArray(new UnsafeCarbonRowPage[rowPages.size()]), unsafeIntermediateFileMerger.getMergedPages());
    } catch (CarbonDataWriterException e) {
        throw new CarbonDataLoadingException(e);
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    // Creates the iterator to read from merge sorter.
    Iterator<CarbonRowBatch> batchIterator = new CarbonIterator<CarbonRowBatch>() {

        @Override
        public boolean hasNext() {
            return finalMerger.hasNext();
        }

        @Override
        public CarbonRowBatch next() {
            int counter = 0;
            CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
            while (finalMerger.hasNext() && counter < batchSize) {
                rowBatch.addRow(new CarbonRow(finalMerger.next()));
                counter++;
            }
            return rowBatch;
        }
    };
    return new Iterator[] { batchIterator };
}

Example 14 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class AbstractFactDataWriter method closeExecutorService.

/**
 * This method will close the executor service which is used for copying carbon
 * data files to carbon store path
 *
 * @throws CarbonDataWriterException
 */
protected void closeExecutorService() throws CarbonDataWriterException {
    CarbonDataWriterException exception = null;
    if (listener != null) {
        try {
            listener.finish();
            listener = null;
        } catch (IOException e) {
            exception = new CarbonDataWriterException(e);
        }
    }
    try {
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.HOURS);
        for (int i = 0; i < executorServiceSubmitList.size(); i++) {
            executorServiceSubmitList.get(i).get();
        }
    } catch (InterruptedException | ExecutionException e) {
        if (null == exception) {
            exception = new CarbonDataWriterException(e);
        }
    }
    if (null != fallbackExecutorService) {
        fallbackExecutorService.shutdownNow();
    }
    if (exception != null) {
        throw exception;
    }
}

Example 15 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method finish.

private void finish(CarbonFactHandler dataHandler, int iteratorIndex) {
    CarbonDataWriterException exception = null;
    try {
        dataHandler.finish();
    } catch (Exception e) {
        // if throw exception from here dataHandler will not be closed.
        // so just holding exception and later throwing exception
        LOGGER.error("Failed for table: " + tableName + " in  finishing data handler", e);
        exception = new CarbonDataWriterException("Failed for table: " + tableName + " in  finishing data handler", e);
    }
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Record Processed For table: " + tableName);
        String logMessage = "Finished Carbon DataWriterProcessorStepImpl: Read: " + readCounter[iteratorIndex] + ": Write: " + readCounter[iteratorIndex];
        LOGGER.debug(logMessage);
    }
    CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordTotalRecords(rowCounter.get());
    try {
        processingComplete(dataHandler);
    } catch (CarbonDataLoadingException e) {
        // else it will erase original root cause
        if (null == exception) {
            exception = new CarbonDataWriterException(e);
        }
    }
    CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PARTITION_ID, System.currentTimeMillis());
    CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordMdkGenerateTotalTime(CarbonTablePath.DEPRECATED_PARTITION_ID, System.currentTimeMillis());
    if (null != exception) {
        throw exception;
    }
}