Examples with CarbonSortKeyAndGroupByException org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException used on opensource projects

Example 1 with CarbonSortKeyAndGroupByException

use of org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.

the class CompactionResultSortProcessor method initSortDataRows.

/**
 * create an instance of sort data rows
 */
private void initSortDataRows() throws Exception {
    measureCount = carbonTable.getMeasureByTableName(tableName).size();
    List<CarbonDimension> dimensions = carbonTable.getDimensionByTableName(tableName);
    noDictionaryColMapping = new boolean[dimensions.size()];
    int i = 0;
    for (CarbonDimension dimension : dimensions) {
        if (CarbonUtil.hasEncoding(dimension.getEncoder(), Encoding.DICTIONARY)) {
            i++;
            continue;
        }
        noDictionaryColMapping[i++] = true;
        noDictionaryCount++;
    }
    dimensionColumnCount = dimensions.size();
    sortParameters = createSortParameters();
    intermediateFileMerger = new SortIntermediateFileMerger(sortParameters);
    // TODO: Now it is only supported onheap merge, but we can have unsafe merge
    // as well by using UnsafeSortDataRows.
    this.sortDataRows = new SortDataRows(sortParameters, intermediateFileMerger);
    try {
        this.sortDataRows.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
        LOGGER.error(e);
        throw new Exception("Error initializing sort data rows object during compaction: " + e.getMessage());
    }
}

Example 2 with CarbonSortKeyAndGroupByException

use of org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.

the class CompactionResultSortProcessor method processResult.

/**
 * This method will iterate over the query result and perform row sorting operation
 *
 * @param resultIteratorList
 */
private void processResult(List<RawResultIterator> resultIteratorList) throws Exception {
    for (RawResultIterator resultIterator : resultIteratorList) {
        if (CompactionType.STREAMING == compactionType) {
            while (resultIterator.hasNext()) {
                // the input iterator of streaming segment is already using raw row
                addRowForSorting(resultIterator.next());
                isRecordFound = true;
            }
        } else {
            while (resultIterator.hasNext()) {
                addRowForSorting(prepareRowObjectForSorting(resultIterator.next()));
                isRecordFound = true;
            }
        }
    }
    try {
        sortDataRows.startSorting();
    } catch (CarbonSortKeyAndGroupByException e) {
        LOGGER.error(e);
        throw new Exception("Problem loading data during compaction: " + e.getMessage());
    }
}

Example 3 with CarbonSortKeyAndGroupByException

use of org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.

the class SingleThreadFinalSortFilesMerger method startSorting.

/**
 * Below method will be used to start storing process This method will get
 * all the temp files present in sort temp folder then it will create the
 * record holder heap and then it will read first record from each file and
 * initialize the heap
 *
 * @throws CarbonSortKeyAndGroupByException
 */
private void startSorting(List<File> files) throws CarbonDataWriterException {
    this.fileCounter = files.size();
    if (fileCounter == 0) {
        LOGGER.info("No files to merge sort");
        return;
    }
    this.fileBufferSize = CarbonDataProcessorUtil.getFileBufferSize(this.fileCounter, CarbonProperties.getInstance(), CarbonCommonConstants.CONSTANT_SIZE_TEN);
    LOGGER.info("Started Final Merge");
    LOGGER.info("Number of temp file: " + this.fileCounter);
    LOGGER.info("File Buffer Size: " + this.fileBufferSize);
    // create record holder heap
    createRecordHolderQueue();
    // iterate over file list and create chunk holder and add to heap
    LOGGER.info("Started adding first record from each file");
    this.executorService = Executors.newFixedThreadPool(maxThreadForSorting);
    for (final File tempFile : files) {
        Callable<Void> callable = new Callable<Void>() {

            @Override
            public Void call() throws CarbonSortKeyAndGroupByException {
                // create chunk holder
                SortTempFileChunkHolder sortTempFileChunkHolder = new SortTempFileChunkHolder(tempFile, sortParameters, tableName);
                try {
                    // initialize
                    sortTempFileChunkHolder.initialize();
                    sortTempFileChunkHolder.readRow();
                } catch (CarbonSortKeyAndGroupByException ex) {
                    sortTempFileChunkHolder.closeStream();
                    notifyFailure(ex);
                }
                synchronized (LOCKOBJECT) {
                    recordHolderHeapLocal.add(sortTempFileChunkHolder);
                }
                return null;
            }
        };
        mergerTask.add(executorService.submit(callable));
    }
    executorService.shutdown();
    try {
        executorService.awaitTermination(2, TimeUnit.HOURS);
    } catch (Exception e) {
        throw new CarbonDataWriterException(e.getMessage(), e);
    }
    checkFailure();
    LOGGER.info("final merger Heap Size" + this.recordHolderHeapLocal.size());
}

Example 4 with CarbonSortKeyAndGroupByException

use of org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.

the class SortDataRows method writeDataToFile.

/**
 * Below method will be used to write data to sort temp file
 *
 * @throws CarbonSortKeyAndGroupByException problem while writing
 */
private void writeDataToFile(Object[][] recordHolderList, int entryCountLocal, File file) throws CarbonSortKeyAndGroupByException {
    DataOutputStream stream = null;
    try {
        // open stream
        stream = FileFactory.getDataOutputStream(file.getPath(), FileFactory.FileType.LOCAL, parameters.getFileWriteBufferSize(), parameters.getSortTempCompressorName());
        // write number of entries to the file
        stream.writeInt(entryCountLocal);
        for (int i = 0; i < entryCountLocal; i++) {
            sortStepRowHandler.writeRawRowAsIntermediateSortTempRowToOutputStream(recordHolderList[i], stream, rowBuffer.get());
        }
    } catch (IOException e) {
        throw new CarbonSortKeyAndGroupByException("Problem while writing the file", e);
    } finally {
        // close streams
        CarbonUtil.closeStreams(stream);
    }
}

Example 5 with CarbonSortKeyAndGroupByException

use of org.apache.carbondata.processing.sort.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.

the class SortDataRows method addRowBatch.

/**
 * This method will be used to add new row
 *
 * @param rowBatch new rowBatch
 * @throws CarbonSortKeyAndGroupByException problem while writing
 */
public void addRowBatch(Object[][] rowBatch, int size) throws CarbonSortKeyAndGroupByException {
    // sort the list and then write current list data to file
    synchronized (addRowsLock) {
        int sizeLeft = 0;
        if (entryCount + size >= sortBufferSize) {
            if (LOGGER.isDebugEnabled()) {
                LOGGER.debug("************ Writing to temp file ********** ");
            }
            intermediateFileMerger.startMergingIfPossible();
            Object[][] recordHolderListLocal = recordHolderList;
            sizeLeft = sortBufferSize - entryCount;
            if (sizeLeft > 0) {
                System.arraycopy(rowBatch, 0, recordHolderListLocal, entryCount, sizeLeft);
            }
            try {
                semaphore.acquire();
                dataSorterAndWriterExecutorService.execute(new DataSorterAndWriter(recordHolderListLocal));
            } catch (Exception e) {
                LOGGER.error("exception occurred while trying to acquire a semaphore lock: " + e.getMessage());
                throw new CarbonSortKeyAndGroupByException(e);
            }
            // create the new holder Array
            this.recordHolderList = new Object[this.sortBufferSize][];
            this.entryCount = 0;
            size = size - sizeLeft;
            if (size == 0) {
                return;
            }
        }
        System.arraycopy(rowBatch, sizeLeft, recordHolderList, entryCount, size);
        entryCount += size;
    }
}