Examples with TablePageStatistics org.apache.carbondata.core.datastore.page.statistics.TablePageStatistics used on opensource projects

Example 1 with TablePageStatistics

use of org.apache.carbondata.core.datastore.page.statistics.TablePageStatistics in project carbondata by apache.

the class CarbonMetadataUtil method getBlockletIndex.

public static BlockletIndex getBlockletIndex(List<EncodedTablePage> encodedTablePageList, List<CarbonMeasure> carbonMeasureList) {
    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
    // Calculating min/max for every each column.
    TablePageStatistics stats = new TablePageStatistics(encodedTablePageList.get(0).getDimensions(), encodedTablePageList.get(0).getMeasures());
    byte[][] minCol = stats.getDimensionMinValue().clone();
    byte[][] maxCol = stats.getDimensionMaxValue().clone();
    for (EncodedTablePage encodedTablePage : encodedTablePageList) {
        stats = new TablePageStatistics(encodedTablePage.getDimensions(), encodedTablePage.getMeasures());
        byte[][] columnMaxData = stats.getDimensionMaxValue();
        byte[][] columnMinData = stats.getDimensionMinValue();
        for (int i = 0; i < maxCol.length; i++) {
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(columnMaxData[i], maxCol[i]) > 0) {
                maxCol[i] = columnMaxData[i];
            }
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(columnMinData[i], minCol[i]) < 0) {
                minCol[i] = columnMinData[i];
            }
        }
    }
    // Writing min/max to thrift file
    for (byte[] max : maxCol) {
        blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));
    }
    for (byte[] min : minCol) {
        blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));
    }
    stats = new TablePageStatistics(encodedTablePageList.get(0).getDimensions(), encodedTablePageList.get(0).getMeasures());
    byte[][] measureMaxValue = stats.getMeasureMaxValue().clone();
    byte[][] measureMinValue = stats.getMeasureMinValue().clone();
    byte[] minVal = null;
    byte[] maxVal = null;
    for (int i = 1; i < encodedTablePageList.size(); i++) {
        for (int j = 0; j < measureMinValue.length; j++) {
            stats = new TablePageStatistics(encodedTablePageList.get(i).getDimensions(), encodedTablePageList.get(i).getMeasures());
            minVal = stats.getMeasureMinValue()[j];
            maxVal = stats.getMeasureMaxValue()[j];
            if (compareMeasureData(measureMaxValue[j], maxVal, carbonMeasureList.get(j).getDataType()) < 0) {
                measureMaxValue[j] = maxVal.clone();
            }
            if (compareMeasureData(measureMinValue[j], minVal, carbonMeasureList.get(j).getDataType()) > 0) {
                measureMinValue[j] = minVal.clone();
            }
        }
    }
    for (byte[] max : measureMaxValue) {
        blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));
    }
    for (byte[] min : measureMinValue) {
        blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));
    }
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    byte[] startKey = encodedTablePageList.get(0).getPageKey().serializeStartKey();
    blockletBTreeIndex.setStart_key(startKey);
    byte[] endKey = encodedTablePageList.get(encodedTablePageList.size() - 1).getPageKey().serializeEndKey();
    blockletBTreeIndex.setEnd_key(endKey);
    BlockletIndex blockletIndex = new BlockletIndex();
    blockletIndex.setMin_max_index(blockletMinMaxIndex);
    blockletIndex.setB_tree_index(blockletBTreeIndex);
    return blockletIndex;
}

Example 2 with TablePageStatistics

use of org.apache.carbondata.core.datastore.page.statistics.TablePageStatistics in project carbondata by apache.

the class CarbonMetadataUtil method getBlockletIndex.

public static BlockletIndex getBlockletIndex(EncodedBlocklet encodedBlocklet, List<CarbonMeasure> carbonMeasureList) {
    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
    // merge writeMinMax flag for all the dimensions
    List<Boolean> writeMinMaxFlag = mergeWriteMinMaxFlagForAllPages(blockletMinMaxIndex, encodedBlocklet);
    // Calculating min/max for every each column.
    TablePageStatistics stats = new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, 0), getEncodedColumnPages(encodedBlocklet, false, 0));
    byte[][] minCol = stats.getDimensionMinValue().clone();
    byte[][] maxCol = stats.getDimensionMaxValue().clone();
    for (int pageIndex = 0; pageIndex < encodedBlocklet.getNumberOfPages(); pageIndex++) {
        stats = new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, pageIndex), getEncodedColumnPages(encodedBlocklet, false, pageIndex));
        byte[][] columnMaxData = stats.getDimensionMaxValue();
        byte[][] columnMinData = stats.getDimensionMinValue();
        for (int i = 0; i < maxCol.length; i++) {
            // and blocklet min/max with empty byte array
            if (!writeMinMaxFlag.get(i)) {
                maxCol[i] = new byte[0];
                minCol[i] = new byte[0];
                continue;
            }
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(columnMaxData[i], maxCol[i]) > 0) {
                maxCol[i] = columnMaxData[i];
            }
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(columnMinData[i], minCol[i]) < 0) {
                minCol[i] = columnMinData[i];
            }
        }
    }
    // Writing min/max to thrift file
    for (byte[] max : maxCol) {
        blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));
    }
    for (byte[] min : minCol) {
        blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));
    }
    stats = new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, 0), getEncodedColumnPages(encodedBlocklet, false, 0));
    byte[][] measureMaxValue = stats.getMeasureMaxValue().clone();
    byte[][] measureMinValue = stats.getMeasureMinValue().clone();
    byte[] minVal = null;
    byte[] maxVal = null;
    for (int i = 1; i < encodedBlocklet.getNumberOfPages(); i++) {
        for (int j = 0; j < measureMinValue.length; j++) {
            stats = new TablePageStatistics(getEncodedColumnPages(encodedBlocklet, true, i), getEncodedColumnPages(encodedBlocklet, false, i));
            minVal = stats.getMeasureMinValue()[j];
            maxVal = stats.getMeasureMaxValue()[j];
            if (CarbonUtil.compareMeasureData(measureMaxValue[j], maxVal, carbonMeasureList.get(j).getDataType()) < 0) {
                measureMaxValue[j] = maxVal.clone();
            }
            if (CarbonUtil.compareMeasureData(measureMinValue[j], minVal, carbonMeasureList.get(j).getDataType()) > 0) {
                measureMinValue[j] = minVal.clone();
            }
        }
    }
    for (byte[] max : measureMaxValue) {
        blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));
    }
    for (byte[] min : measureMinValue) {
        blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));
    }
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    blockletBTreeIndex.setStart_key(DEPRECATED_MDK);
    blockletBTreeIndex.setEnd_key(DEPRECATED_MDK);
    BlockletIndex blockletIndex = new BlockletIndex();
    blockletIndex.setMin_max_index(blockletMinMaxIndex);
    blockletIndex.setB_tree_index(blockletBTreeIndex);
    return blockletIndex;
}