Examples with BlockletIndex org.apache.carbondata.format.BlockletIndex used on opensource projects

Example 1 with BlockletIndex

use of org.apache.carbondata.format.BlockletIndex in project carbondata by apache.

the class CarbonMetadataUtil method getBlockletIndex.

public static BlockletIndex getBlockletIndex(List<NodeHolder> nodeHolderList, List<CarbonMeasure> carbonMeasureList) {
    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
    // Calculating min/max for every each column.
    byte[][] minCol = nodeHolderList.get(0).getColumnMinData().clone();
    byte[][] maxCol = nodeHolderList.get(0).getColumnMaxData().clone();
    for (NodeHolder nodeHolder : nodeHolderList) {
        byte[][] columnMaxData = nodeHolder.getColumnMaxData();
        byte[][] columnMinData = nodeHolder.getColumnMinData();
        for (int i = 0; i < maxCol.length; i++) {
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(columnMaxData[i], maxCol[i]) > 0) {
                maxCol[i] = columnMaxData[i];
            }
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(columnMinData[i], minCol[i]) < 0) {
                minCol[i] = columnMinData[i];
            }
        }
    }
    // Writing min/max to thrift file
    for (byte[] max : maxCol) {
        blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));
    }
    for (byte[] min : minCol) {
        blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));
    }
    byte[][] measureMaxValue = nodeHolderList.get(0).getMeasureColumnMaxData().clone();
    byte[][] measureMinValue = nodeHolderList.get(0).getMeasureColumnMinData().clone();
    byte[] minVal = null;
    byte[] maxVal = null;
    for (int i = 1; i < nodeHolderList.size(); i++) {
        for (int j = 0; j < measureMinValue.length; j++) {
            minVal = nodeHolderList.get(i).getMeasureColumnMinData()[j];
            maxVal = nodeHolderList.get(i).getMeasureColumnMaxData()[j];
            if (compareMeasureData(measureMaxValue[j], maxVal, carbonMeasureList.get(j).getDataType()) < 0) {
                measureMaxValue[j] = maxVal.clone();
            }
            if (compareMeasureData(measureMinValue[j], minVal, carbonMeasureList.get(j).getDataType()) > 0) {
                measureMinValue[j] = minVal.clone();
            }
        }
    }
    for (byte[] max : measureMaxValue) {
        blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));
    }
    for (byte[] min : measureMinValue) {
        blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));
    }
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    blockletBTreeIndex.setStart_key(nodeHolderList.get(0).getStartKey());
    blockletBTreeIndex.setEnd_key(nodeHolderList.get(nodeHolderList.size() - 1).getEndKey());
    BlockletIndex blockletIndex = new BlockletIndex();
    blockletIndex.setMin_max_index(blockletMinMaxIndex);
    blockletIndex.setB_tree_index(blockletBTreeIndex);
    return blockletIndex;
}

Example 2 with BlockletIndex

use of org.apache.carbondata.format.BlockletIndex in project carbondata by apache.

the class CarbonMetadataUtil method getBlockletIndex.

private static BlockletIndex getBlockletIndex(org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex info) {
    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
    for (int i = 0; i < info.getMinMaxIndex().getMaxValues().length; i++) {
        blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(info.getMinMaxIndex().getMaxValues()[i]));
        blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(info.getMinMaxIndex().getMinValues()[i]));
    }
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    blockletBTreeIndex.setStart_key(info.getBtreeIndex().getStartKey());
    blockletBTreeIndex.setEnd_key(info.getBtreeIndex().getEndKey());
    BlockletIndex blockletIndex = new BlockletIndex();
    blockletIndex.setMin_max_index(blockletMinMaxIndex);
    blockletIndex.setB_tree_index(blockletBTreeIndex);
    return blockletIndex;
}

Example 3 with BlockletIndex

use of org.apache.carbondata.format.BlockletIndex in project carbondata by apache.

the class CarbonMetadataUtil method getFileFooter3.

/**
   * Below method will be used to get the file footer object
   *
   * @param infoList         blocklet info
   * @param blockletIndexs
   * @param cardinalities    cardinlaity of dimension columns
   * @param numberOfColumns
   * @return file footer
   */
private static FileFooter3 getFileFooter3(List<BlockletInfo3> infoList, List<BlockletIndex> blockletIndexs, int[] cardinalities, int numberOfColumns) {
    SegmentInfo segmentInfo = new SegmentInfo();
    segmentInfo.setNum_cols(numberOfColumns);
    segmentInfo.setColumn_cardinalities(CarbonUtil.convertToIntegerList(cardinalities));
    FileFooter3 footer = new FileFooter3();
    footer.setNum_rows(getNumberOfRowForFooter(infoList));
    footer.setSegment_info(segmentInfo);
    for (BlockletIndex info : blockletIndexs) {
        footer.addToBlocklet_index_list(info);
    }
    return footer;
}

Example 4 with BlockletIndex

use of org.apache.carbondata.format.BlockletIndex in project carbondata by apache.

the class CarbonMetadataUtil method getBlockletIndex.

private static BlockletIndex getBlockletIndex(BlockletInfoColumnar info) {
    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
    for (byte[] max : info.getColumnMaxData()) {
        blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(max));
    }
    for (byte[] min : info.getColumnMinData()) {
        blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(min));
    }
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    blockletBTreeIndex.setStart_key(info.getStartKey());
    blockletBTreeIndex.setEnd_key(info.getEndKey());
    BlockletIndex blockletIndex = new BlockletIndex();
    blockletIndex.setMin_max_index(blockletMinMaxIndex);
    blockletIndex.setB_tree_index(blockletBTreeIndex);
    return blockletIndex;
}

Example 5 with BlockletIndex

use of org.apache.carbondata.format.BlockletIndex in project carbondata by apache.

the class CarbonMetadataUtil method setBlockletIndex.

private static void setBlockletIndex(FileFooter footer, List<BlockletInfoColumnar> listOfNodeInfo) {
    List<BlockletIndex> blockletIndexList = footer.getBlocklet_index_list();
    for (int i = 0; i < blockletIndexList.size(); i++) {
        BlockletBTreeIndex bTreeIndexList = blockletIndexList.get(i).getB_tree_index();
        BlockletMinMaxIndex minMaxIndexList = blockletIndexList.get(i).getMin_max_index();
        listOfNodeInfo.get(i).setStartKey(bTreeIndexList.getStart_key());
        listOfNodeInfo.get(i).setEndKey(bTreeIndexList.getEnd_key());
        byte[][] min = new byte[minMaxIndexList.getMin_values().size()][];
        byte[][] max = new byte[minMaxIndexList.getMax_values().size()][];
        for (int j = 0; j < minMaxIndexList.getMax_valuesSize(); j++) {
            min[j] = minMaxIndexList.getMin_values().get(j).array();
            max[j] = minMaxIndexList.getMax_values().get(j).array();
        }
        listOfNodeInfo.get(i).setColumnMaxData(max);
    }
}