Examples with RowComparator org.apache.spark.sql.secondaryindex.load.RowComparator used on opensource projects

Example 1 with RowComparator

use of org.apache.spark.sql.secondaryindex.load.RowComparator in project carbondata by apache.

the class SecondaryIndexQueryResultProcessor method readAndLoadDataFromSortTempFiles.

/**
 * This method will read sort temp files, perform merge sort and add it to store for data loading
 */
private void readAndLoadDataFromSortTempFiles() throws SecondaryIndexException {
    Throwable throwable = null;
    try {
        Object[] previousRow = null;
        // comparator for grouping the similar data, means every record
        // should be unique in index table
        RowComparator comparator = new RowComparator(noDictionaryColMapping, SecondaryIndexUtil.getNoDictDataTypes(indexTable));
        intermediateFileMerger.finish();
        sortDataRows = null;
        finalMerger.startFinalMerge();
        while (finalMerger.hasNext()) {
            Object[] rowRead = finalMerger.next();
            if (null == previousRow) {
                previousRow = rowRead;
            } else {
                int compareResult = comparator.compare(previousRow, rowRead);
                if (0 == compareResult) {
                    // skip similar data rows
                    continue;
                } else {
                    previousRow = rowRead;
                }
            }
            CarbonRow row = new CarbonRow(rowRead);
            dataHandler.addDataToStore(row);
        }
        dataHandler.finish();
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e);
        throw new SecondaryIndexException("Problem loading data while creating secondary index: ", e);
    } catch (CarbonSortKeyAndGroupByException e) {
        LOGGER.error(e);
        throw new SecondaryIndexException("Problem in merging intermediate files while creating secondary index: ", e);
    } catch (Throwable t) {
        LOGGER.error(t);
        throw new SecondaryIndexException("Problem while creating secondary index: ", t);
    } finally {
        if (null != dataHandler) {
            try {
                dataHandler.closeHandler();
            } catch (CarbonDataWriterException e) {
                LOGGER.error(e);
                throwable = e;
            }
        }
    }
    if (null != throwable) {
        throw new SecondaryIndexException("Problem closing data handler while creating secondary index: ", throwable);
    }
    dataHandler = null;
}