Examples with BlockIndexInfo org.apache.carbondata.core.metadata.index.BlockIndexInfo used on opensource projects

Example 1 with BlockIndexInfo

use of org.apache.carbondata.core.metadata.index.BlockIndexInfo in project carbondata by apache.

the class AbstractFactDataWriter method fillBlockIndexInfoDetails.

/**
   * Below method will be used to fill the vlock info details
   *
   * @param numberOfRows    number of rows in file
   * @param carbonDataFileName The name of carbonData file
   * @param currentPosition current offset
   */
protected void fillBlockIndexInfoDetails(long numberOfRows, String carbonDataFileName, long currentPosition) {
    // as min-max will change for each blocklet and second blocklet min-max can be lesser than
    // the first blocklet so we need to calculate the complete block level min-max by taking
    // the min value of each column and max value of each column
    byte[][] currentMinValue = blockletInfoList.get(0).getColumnMinData().clone();
    byte[][] currentMaxValue = blockletInfoList.get(0).getColumnMaxData().clone();
    byte[][] minValue = null;
    byte[][] maxValue = null;
    for (int i = 1; i < blockletInfoList.size(); i++) {
        minValue = blockletInfoList.get(i).getColumnMinData();
        maxValue = blockletInfoList.get(i).getColumnMaxData();
        for (int j = 0; j < maxValue.length; j++) {
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMinValue[j], minValue[j]) > 0) {
                currentMinValue[j] = minValue[j].clone();
            }
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMaxValue[j], maxValue[j]) < 0) {
                currentMaxValue[j] = maxValue[j].clone();
            }
        }
    }
    // start and end key we can take based on first blocklet
    // start key will be the block start key as
    // it is the least key and end blocklet end key will be the block end key as it is the max key
    BlockletBTreeIndex btree = new BlockletBTreeIndex(blockletInfoList.get(0).getStartKey(), blockletInfoList.get(blockletInfoList.size() - 1).getEndKey());
    BlockletMinMaxIndex minmax = new BlockletMinMaxIndex();
    minmax.setMinValues(currentMinValue);
    minmax.setMaxValues(currentMaxValue);
    BlockletIndex blockletIndex = new BlockletIndex(btree, minmax);
    BlockIndexInfo blockIndexInfo = new BlockIndexInfo(numberOfRows, carbonDataFileName, currentPosition, blockletIndex);
    blockIndexInfoList.add(blockIndexInfo);
}

Example 2 with BlockIndexInfo

use of org.apache.carbondata.core.metadata.index.BlockIndexInfo in project carbondata by apache.

the class CarbonMetadataUtilTest method testGetBlockIndexInfo.

@Test
public void testGetBlockIndexInfo() throws Exception {
    byte[] startKey = { 1, 2, 3, 4, 5 };
    byte[] endKey = { 9, 3, 5, 5, 5 };
    byte[] byteArr = { 1, 2, 3, 4, 5 };
    List<ByteBuffer> minList = new ArrayList<>();
    minList.add(ByteBuffer.wrap(byteArr));
    byte[] byteArr1 = { 9, 9, 8, 6, 7 };
    List<ByteBuffer> maxList = new ArrayList<>();
    maxList.add(ByteBuffer.wrap(byteArr1));
    org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex blockletMinMaxIndex = new org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex(minList, maxList);
    org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex blockletBTreeIndex = new org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex(startKey, endKey);
    org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex blockletIndex = new org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex(blockletBTreeIndex, blockletMinMaxIndex);
    BlockIndexInfo blockIndexInfo = new BlockIndexInfo(1, "file", 1, blockletIndex);
    List<BlockIndexInfo> blockIndexInfoList = new ArrayList<>();
    blockIndexInfoList.add(blockIndexInfo);
    List<BlockIndex> result = getBlockIndexInfo(blockIndexInfoList);
    String expected = "file";
    assertEquals(result.get(0).file_name, expected);
}

Example 3 with BlockIndexInfo

use of org.apache.carbondata.core.metadata.index.BlockIndexInfo in project carbondata by apache.

the class CarbonMetadataUtil method getBlockIndexInfo.

/**
 * Below method will be used to get the block index info thrift object for
 * each block present in the segment
 *
 * @param blockIndexInfoList block index info list
 * @return list of block index
 */
public static List<BlockIndex> getBlockIndexInfo(List<BlockIndexInfo> blockIndexInfoList) {
    List<BlockIndex> thriftBlockIndexList = new ArrayList<BlockIndex>();
    BlockIndex blockIndex = null;
    // below code to create block index info object for each block
    for (BlockIndexInfo blockIndexInfo : blockIndexInfoList) {
        blockIndex = new BlockIndex();
        blockIndex.setNum_rows(blockIndexInfo.getNumberOfRows());
        blockIndex.setOffset(blockIndexInfo.getOffset());
        blockIndex.setFile_name(blockIndexInfo.getFileName());
        blockIndex.setBlock_index(getBlockletIndex(blockIndexInfo.getBlockletIndex()));
        if (blockIndexInfo.getBlockletInfo() != null) {
            blockIndex.setBlocklet_info(getBlocletInfo3(blockIndexInfo.getBlockletInfo()));
        }
        thriftBlockIndexList.add(blockIndex);
    }
    return thriftBlockIndexList;
}

Example 4 with BlockIndexInfo

use of org.apache.carbondata.core.metadata.index.BlockIndexInfo in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method fillBlockIndexInfoDetails.

/**
 * Below method will be used to fill the block info details
 *
 * @param numberOfRows       number of rows in file
 * @param carbonDataFileName The name of carbonData file
 * @param currentPosition    current offset
 */
@Override
protected void fillBlockIndexInfoDetails(long numberOfRows, String carbonDataFileName, long currentPosition) {
    int i = 0;
    DataFileFooterConverterV3 converterV3 = new DataFileFooterConverterV3();
    for (org.apache.carbondata.format.BlockletIndex index : blockletIndex) {
        BlockletInfo3 blockletInfo3 = blockletMetadata.get(i);
        BlockletInfo blockletInfo = converterV3.getBlockletInfo(blockletInfo3, model.getSegmentProperties().getDimensions().size());
        BlockletBTreeIndex bTreeIndex = new BlockletBTreeIndex(index.b_tree_index.getStart_key(), index.b_tree_index.getEnd_key());
        BlockletMinMaxIndex minMaxIndex = new BlockletMinMaxIndex();
        minMaxIndex.setMinValues(toByteArray(index.getMin_max_index().getMin_values()));
        minMaxIndex.setMaxValues(toByteArray(index.getMin_max_index().getMax_values()));
        org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex bIndex = new org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex(bTreeIndex, minMaxIndex);
        BlockIndexInfo biInfo = new BlockIndexInfo(numberOfRows, carbonDataFileName, currentPosition, bIndex, blockletInfo);
        blockIndexInfoList.add(biInfo);
        i++;
    }
}