Examples with BlockletBTreeIndex org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex used on opensource projects

Example 1 with BlockletBTreeIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex in project carbondata by apache.

the class AbstractFactDataWriter method fillBlockIndexInfoDetails.

/**
   * Below method will be used to fill the vlock info details
   *
   * @param numberOfRows    number of rows in file
   * @param carbonDataFileName The name of carbonData file
   * @param currentPosition current offset
   */
protected void fillBlockIndexInfoDetails(long numberOfRows, String carbonDataFileName, long currentPosition) {
    // as min-max will change for each blocklet and second blocklet min-max can be lesser than
    // the first blocklet so we need to calculate the complete block level min-max by taking
    // the min value of each column and max value of each column
    byte[][] currentMinValue = blockletInfoList.get(0).getColumnMinData().clone();
    byte[][] currentMaxValue = blockletInfoList.get(0).getColumnMaxData().clone();
    byte[][] minValue = null;
    byte[][] maxValue = null;
    for (int i = 1; i < blockletInfoList.size(); i++) {
        minValue = blockletInfoList.get(i).getColumnMinData();
        maxValue = blockletInfoList.get(i).getColumnMaxData();
        for (int j = 0; j < maxValue.length; j++) {
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMinValue[j], minValue[j]) > 0) {
                currentMinValue[j] = minValue[j].clone();
            }
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMaxValue[j], maxValue[j]) < 0) {
                currentMaxValue[j] = maxValue[j].clone();
            }
        }
    }
    // start and end key we can take based on first blocklet
    // start key will be the block start key as
    // it is the least key and end blocklet end key will be the block end key as it is the max key
    BlockletBTreeIndex btree = new BlockletBTreeIndex(blockletInfoList.get(0).getStartKey(), blockletInfoList.get(blockletInfoList.size() - 1).getEndKey());
    BlockletMinMaxIndex minmax = new BlockletMinMaxIndex();
    minmax.setMinValues(currentMinValue);
    minmax.setMaxValues(currentMaxValue);
    BlockletIndex blockletIndex = new BlockletIndex(btree, minmax);
    BlockIndexInfo blockIndexInfo = new BlockIndexInfo(numberOfRows, carbonDataFileName, currentPosition, blockletIndex);
    blockIndexInfoList.add(blockIndexInfo);
}

Example 2 with BlockletBTreeIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex in project carbondata by apache.

the class BTreeBlockFinderTest method getFileMatadataWithOnlyNoDictionaryKey.

private DataFileFooter getFileMatadataWithOnlyNoDictionaryKey(byte[] startKey, byte[] endKey, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) {
    DataFileFooter footer = new DataFileFooter();
    BlockletIndex index = new BlockletIndex();
    BlockletBTreeIndex btreeIndex = new BlockletBTreeIndex();
    ByteBuffer buffer = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryStartKey.length);
    buffer.putInt(0);
    buffer.putInt(noDictionaryStartKey.length);
    buffer.put(noDictionaryStartKey);
    buffer.rewind();
    btreeIndex.setStartKey(buffer.array());
    ByteBuffer buffer1 = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryEndKey.length);
    buffer1.putInt(0);
    buffer1.putInt(noDictionaryEndKey.length);
    buffer1.put(noDictionaryEndKey);
    buffer1.rewind();
    btreeIndex.setEndKey(buffer1.array());
    BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
    minMax.setMaxValues(new byte[][] { endKey, noDictionaryEndKey });
    minMax.setMinValues(new byte[][] { startKey, noDictionaryStartKey });
    index.setBtreeIndex(btreeIndex);
    index.setMinMaxIndex(minMax);
    footer.setBlockletIndex(index);
    return footer;
}

Example 3 with BlockletBTreeIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex in project carbondata by apache.

the class BTreeBlockFinderTest method getFileFooterWithOnlyDictionaryKey.

private DataFileFooter getFileFooterWithOnlyDictionaryKey(byte[] startKey, byte[] endKey, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) {
    DataFileFooter footer = new DataFileFooter();
    BlockletIndex index = new BlockletIndex();
    BlockletBTreeIndex btreeIndex = new BlockletBTreeIndex();
    ByteBuffer buffer = ByteBuffer.allocate(4 + startKey.length + 4 + 0);
    buffer.putInt(startKey.length);
    buffer.putInt(0);
    buffer.put(startKey);
    buffer.rewind();
    btreeIndex.setStartKey(buffer.array());
    ByteBuffer buffer1 = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryEndKey.length);
    buffer1.putInt(endKey.length);
    buffer1.putInt(0);
    buffer1.put(endKey);
    buffer1.rewind();
    btreeIndex.setEndKey(buffer1.array());
    BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
    minMax.setMaxValues(new byte[][] { endKey });
    minMax.setMinValues(new byte[][] { startKey });
    index.setBtreeIndex(btreeIndex);
    index.setMinMaxIndex(minMax);
    footer.setBlockletIndex(index);
    return footer;
}

Example 4 with BlockletBTreeIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex in project carbondata by apache.

the class AbstractDataFileFooterConverter method getBlockletIndexForDataFileFooter.

/**
 * Below method will be used to get blocklet index for data file meta
 *
 * @param blockletIndexList
 * @return blocklet index
 */
protected BlockletIndex getBlockletIndexForDataFileFooter(List<BlockletIndex> blockletIndexList) {
    BlockletIndex blockletIndex = new BlockletIndex();
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    blockletBTreeIndex.setStartKey(blockletIndexList.get(0).getBtreeIndex().getStartKey());
    blockletBTreeIndex.setEndKey(blockletIndexList.get(blockletIndexList.size() - 1).getBtreeIndex().getEndKey());
    blockletIndex.setBtreeIndex(blockletBTreeIndex);
    byte[][] currentMinValue = blockletIndexList.get(0).getMinMaxIndex().getMinValues().clone();
    byte[][] currentMaxValue = blockletIndexList.get(0).getMinMaxIndex().getMaxValues().clone();
    byte[][] minValue = null;
    byte[][] maxValue = null;
    for (int i = 1; i < blockletIndexList.size(); i++) {
        minValue = blockletIndexList.get(i).getMinMaxIndex().getMinValues();
        maxValue = blockletIndexList.get(i).getMinMaxIndex().getMaxValues();
        for (int j = 0; j < maxValue.length; j++) {
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMinValue[j], minValue[j]) > 0) {
                currentMinValue[j] = minValue[j].clone();
            }
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMaxValue[j], maxValue[j]) < 0) {
                currentMaxValue[j] = maxValue[j].clone();
            }
        }
    }
    BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
    minMax.setMaxValues(currentMaxValue);
    minMax.setMinValues(currentMinValue);
    blockletIndex.setMinMaxIndex(minMax);
    return blockletIndex;
}

Example 5 with BlockletBTreeIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletBTreeIndex in project carbondata by apache.

the class AbstractDataFileFooterConverter method getBlockletIndex.

/**
 * Below method will be used to convert the blocklet index of thrift to
 * wrapper
 *
 * @param blockletIndexThrift
 * @return blocklet index wrapper
 */
protected BlockletIndex getBlockletIndex(org.apache.carbondata.format.BlockletIndex blockletIndexThrift) {
    org.apache.carbondata.format.BlockletBTreeIndex btreeIndex = blockletIndexThrift.getB_tree_index();
    org.apache.carbondata.format.BlockletMinMaxIndex minMaxIndex = blockletIndexThrift.getMin_max_index();
    return new BlockletIndex(new BlockletBTreeIndex(btreeIndex.getStart_key(), btreeIndex.getEnd_key()), new BlockletMinMaxIndex(minMaxIndex.getMin_values(), minMaxIndex.getMax_values()));
}