Examples with CarbonSortKeyAndGroupByException org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException used on opensource projects

Example 6 with CarbonSortKeyAndGroupByException

use of org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.

the class ParallelReadMergeSorterImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    SortDataRows sortDataRow = new SortDataRows(sortParameters, intermediateFileMerger);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        sortDataRow.initialize();
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    this.executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(executorService);
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.submit(new SortIteratorThread(iterators[i], sortDataRow, batchSize, rowCounter, threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRow, sortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        intermediateFileMerger.finish();
        intermediateFileMerger = null;
        finalMerger.startFinalMerge();
    } catch (CarbonDataWriterException e) {
        throw new CarbonDataLoadingException(e);
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    // Creates the iterator to read from merge sorter.
    Iterator<CarbonRowBatch> batchIterator = new CarbonIterator<CarbonRowBatch>() {

        @Override
        public boolean hasNext() {
            return finalMerger.hasNext();
        }

        @Override
        public CarbonRowBatch next() {
            int counter = 0;
            CarbonRowBatch rowBatch = new CarbonRowBatch(batchSize);
            while (finalMerger.hasNext() && counter < batchSize) {
                rowBatch.addRow(new CarbonRow(finalMerger.next()));
                counter++;
            }
            return rowBatch;
        }
    };
    return new Iterator[] { batchIterator };
}

Example 7 with CarbonSortKeyAndGroupByException

use of org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.

the class ParallelReadMergeSorterImpl method processRowToNextStep.

/**
   * Below method will be used to process data to next step
   */
private boolean processRowToNextStep(SortDataRows sortDataRows, SortParameters parameters) throws CarbonDataLoadingException {
    if (null == sortDataRows) {
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        LOGGER.info("Number of Records was Zero");
        String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
        LOGGER.info(logMessage);
        return false;
    }
    try {
        // start sorting
        sortDataRows.startSorting();
        // check any more rows are present
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        return false;
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
}

Example 8 with CarbonSortKeyAndGroupByException

use of org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.

the class ParallelReadMergeSorterWithBucketingImpl method sort.

@Override
public Iterator<CarbonRowBatch>[] sort(Iterator<CarbonRowBatch>[] iterators) throws CarbonDataLoadingException {
    SortDataRows[] sortDataRows = new SortDataRows[bucketingInfo.getNumberOfBuckets()];
    intermediateFileMergers = new SortIntermediateFileMerger[sortDataRows.length];
    try {
        for (int i = 0; i < bucketingInfo.getNumberOfBuckets(); i++) {
            SortParameters parameters = sortParameters.getCopy();
            parameters.setPartitionID(i + "");
            setTempLocation(parameters);
            parameters.setBufferSize(sortBufferSize);
            intermediateFileMergers[i] = new SortIntermediateFileMerger(parameters);
            sortDataRows[i] = new SortDataRows(parameters, intermediateFileMergers[i]);
            sortDataRows[i].initialize();
        }
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    this.executorService = Executors.newFixedThreadPool(iterators.length);
    this.threadStatusObserver = new ThreadStatusObserver(this.executorService);
    final int batchSize = CarbonProperties.getInstance().getBatchSize();
    try {
        for (int i = 0; i < iterators.length; i++) {
            executorService.submit(new SortIteratorThread(iterators[i], sortDataRows, rowCounter, this.threadStatusObserver));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.DAYS);
        processRowToNextStep(sortDataRows, sortParameters);
    } catch (Exception e) {
        checkError();
        throw new CarbonDataLoadingException("Problem while shutdown the server ", e);
    }
    checkError();
    try {
        for (int i = 0; i < intermediateFileMergers.length; i++) {
            intermediateFileMergers[i].finish();
        }
    } catch (CarbonDataWriterException e) {
        throw new CarbonDataLoadingException(e);
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
    Iterator<CarbonRowBatch>[] batchIterator = new Iterator[bucketingInfo.getNumberOfBuckets()];
    for (int i = 0; i < bucketingInfo.getNumberOfBuckets(); i++) {
        batchIterator[i] = new MergedDataIterator(String.valueOf(i), batchSize);
    }
    return batchIterator;
}

Example 9 with CarbonSortKeyAndGroupByException

use of org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.

the class ParallelReadMergeSorterWithBucketingImpl method processRowToNextStep.

/**
   * Below method will be used to process data to next step
   */
private boolean processRowToNextStep(SortDataRows[] sortDataRows, SortParameters parameters) throws CarbonDataLoadingException {
    if (null == sortDataRows || sortDataRows.length == 0) {
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        LOGGER.info("Number of Records was Zero");
        String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
        LOGGER.info(logMessage);
        return false;
    }
    try {
        for (int i = 0; i < sortDataRows.length; i++) {
            // start sorting
            sortDataRows[i].startSorting();
        }
        // check any more rows are present
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        return false;
    } catch (CarbonSortKeyAndGroupByException e) {
        throw new CarbonDataLoadingException(e);
    }
}

Example 10 with CarbonSortKeyAndGroupByException

use of org.apache.carbondata.processing.sortandgroupby.exception.CarbonSortKeyAndGroupByException in project carbondata by apache.

the class UnsafeSortTempFileChunkHolder method getRowFromStream.

/**
   * @return
   * @throws CarbonSortKeyAndGroupByException
   */
private Object[] getRowFromStream() throws CarbonSortKeyAndGroupByException {
    Object[] row = new Object[dimensionCount + measureCount];
    try {
        int dimCount = 0;
        for (; dimCount < isNoDictionaryDimensionColumn.length; dimCount++) {
            if (isNoDictionaryDimensionColumn[dimCount]) {
                short aShort = stream.readShort();
                byte[] col = new byte[aShort];
                stream.readFully(col);
                row[dimCount] = col;
            } else {
                int anInt = stream.readInt();
                row[dimCount] = anInt;
            }
        }
        // write complex dimensions here.
        for (; dimCount < dimensionCount; dimCount++) {
            short aShort = stream.readShort();
            byte[] col = new byte[aShort];
            stream.readFully(col);
            row[dimCount] = col;
        }
        long[] words = new long[nullSetWordsLength];
        for (int i = 0; i < words.length; i++) {
            words[i] = stream.readLong();
        }
        for (int mesCount = 0; mesCount < measureCount; mesCount++) {
            if (UnsafeCarbonRowPage.isSet(words, mesCount)) {
                switch(measureDataType[mesCount]) {
                    case SHORT:
                        row[dimensionCount + mesCount] = stream.readShort();
                        break;
                    case INT:
                        row[dimensionCount + mesCount] = stream.readInt();
                        break;
                    case LONG:
                        row[dimensionCount + mesCount] = stream.readLong();
                        break;
                    case DOUBLE:
                        row[dimensionCount + mesCount] = stream.readDouble();
                        break;
                    case DECIMAL:
                        short aShort = stream.readShort();
                        byte[] bigDecimalInBytes = new byte[aShort];
                        stream.readFully(bigDecimalInBytes);
                        row[dimensionCount + mesCount] = bigDecimalInBytes;
                        break;
                }
            }
        }
        return row;
    } catch (Exception e) {
        throw new CarbonSortKeyAndGroupByException(e);
    }
}