Examples with BlockletMinMaxIndex org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex used on opensource projects

Example 1 with BlockletMinMaxIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex in project carbondata by apache.

the class AbstractDataFileFooterConverter method getBlockletIndexForDataFileFooter.

/**
   * Below method will be used to get blocklet index for data file meta
   *
   * @param blockletIndexList
   * @return blocklet index
   */
protected BlockletIndex getBlockletIndexForDataFileFooter(List<BlockletIndex> blockletIndexList) {
    BlockletIndex blockletIndex = new BlockletIndex();
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    blockletBTreeIndex.setStartKey(blockletIndexList.get(0).getBtreeIndex().getStartKey());
    blockletBTreeIndex.setEndKey(blockletIndexList.get(blockletIndexList.size() - 1).getBtreeIndex().getEndKey());
    blockletIndex.setBtreeIndex(blockletBTreeIndex);
    byte[][] currentMinValue = blockletIndexList.get(0).getMinMaxIndex().getMinValues().clone();
    byte[][] currentMaxValue = blockletIndexList.get(0).getMinMaxIndex().getMaxValues().clone();
    byte[][] minValue = null;
    byte[][] maxValue = null;
    for (int i = 1; i < blockletIndexList.size(); i++) {
        minValue = blockletIndexList.get(i).getMinMaxIndex().getMinValues();
        maxValue = blockletIndexList.get(i).getMinMaxIndex().getMaxValues();
        for (int j = 0; j < maxValue.length; j++) {
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMinValue[j], minValue[j]) > 0) {
                currentMinValue[j] = minValue[j].clone();
            }
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMaxValue[j], maxValue[j]) < 0) {
                currentMaxValue[j] = maxValue[j].clone();
            }
        }
    }
    BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
    minMax.setMaxValues(currentMaxValue);
    minMax.setMinValues(currentMinValue);
    blockletIndex.setMinMaxIndex(minMax);
    return blockletIndex;
}

Example 2 with BlockletMinMaxIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex in project carbondata by apache.

the class AbstractDataFileFooterConverter method getBlockletIndex.

/**
   * Below method will be used to convert the blocklet index of thrift to
   * wrapper
   *
   * @param blockletIndexThrift
   * @return blocklet index wrapper
   */
protected BlockletIndex getBlockletIndex(org.apache.carbondata.format.BlockletIndex blockletIndexThrift) {
    org.apache.carbondata.format.BlockletBTreeIndex btreeIndex = blockletIndexThrift.getB_tree_index();
    org.apache.carbondata.format.BlockletMinMaxIndex minMaxIndex = blockletIndexThrift.getMin_max_index();
    return new BlockletIndex(new BlockletBTreeIndex(btreeIndex.getStart_key(), btreeIndex.getEnd_key()), new BlockletMinMaxIndex(minMaxIndex.getMin_values(), minMaxIndex.getMax_values()));
}

Example 3 with BlockletMinMaxIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex in project carbondata by apache.

the class BTreeBlockFinderTest method getFileMatadataWithOnlyNoDictionaryKey.

private DataFileFooter getFileMatadataWithOnlyNoDictionaryKey(byte[] startKey, byte[] endKey, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) {
    DataFileFooter footer = new DataFileFooter();
    BlockletIndex index = new BlockletIndex();
    BlockletBTreeIndex btreeIndex = new BlockletBTreeIndex();
    ByteBuffer buffer = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryStartKey.length);
    buffer.putInt(0);
    buffer.putInt(noDictionaryStartKey.length);
    buffer.put(noDictionaryStartKey);
    buffer.rewind();
    btreeIndex.setStartKey(buffer.array());
    ByteBuffer buffer1 = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryEndKey.length);
    buffer1.putInt(0);
    buffer1.putInt(noDictionaryEndKey.length);
    buffer1.put(noDictionaryEndKey);
    buffer1.rewind();
    btreeIndex.setEndKey(buffer1.array());
    BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
    minMax.setMaxValues(new byte[][] { endKey, noDictionaryEndKey });
    minMax.setMinValues(new byte[][] { startKey, noDictionaryStartKey });
    index.setBtreeIndex(btreeIndex);
    index.setMinMaxIndex(minMax);
    footer.setBlockletIndex(index);
    return footer;
}

Example 4 with BlockletMinMaxIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex in project carbondata by apache.

the class BTreeBlockFinderTest method getFileFooterWithOnlyDictionaryKey.

private DataFileFooter getFileFooterWithOnlyDictionaryKey(byte[] startKey, byte[] endKey, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) {
    DataFileFooter footer = new DataFileFooter();
    BlockletIndex index = new BlockletIndex();
    BlockletBTreeIndex btreeIndex = new BlockletBTreeIndex();
    ByteBuffer buffer = ByteBuffer.allocate(4 + startKey.length + 4 + 0);
    buffer.putInt(startKey.length);
    buffer.putInt(0);
    buffer.put(startKey);
    buffer.rewind();
    btreeIndex.setStartKey(buffer.array());
    ByteBuffer buffer1 = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryEndKey.length);
    buffer1.putInt(endKey.length);
    buffer1.putInt(0);
    buffer1.put(endKey);
    buffer1.rewind();
    btreeIndex.setEndKey(buffer1.array());
    BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
    minMax.setMaxValues(new byte[][] { endKey });
    minMax.setMinValues(new byte[][] { startKey });
    index.setBtreeIndex(btreeIndex);
    index.setMinMaxIndex(minMax);
    footer.setBlockletIndex(index);
    return footer;
}

Example 5 with BlockletMinMaxIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletMinMaxIndex in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method fillBlockIndexInfoDetails.

/**
   * Below method will be used to fill the block info details
   *
   * @param numberOfRows    number of rows in file
   * @param carbonDataFileName The name of carbonData file
   * @param currentPosition current offset
   */
protected void fillBlockIndexInfoDetails(long numberOfRows, String carbonDataFileName, long currentPosition) {
    byte[][] currentMinValue = new byte[blockletIndex.get(0).min_max_index.max_values.size()][];
    byte[][] currentMaxValue = new byte[blockletIndex.get(0).min_max_index.max_values.size()][];
    for (int i = 0; i < currentMaxValue.length; i++) {
        currentMinValue[i] = blockletIndex.get(0).min_max_index.getMin_values().get(i).array();
        currentMaxValue[i] = blockletIndex.get(0).min_max_index.getMax_values().get(i).array();
    }
    byte[] minValue = null;
    byte[] maxValue = null;
    int measureStartIndex = currentMinValue.length - dataWriterVo.getMeasureCount();
    for (int i = 1; i < blockletIndex.size(); i++) {
        for (int j = 0; j < measureStartIndex; j++) {
            minValue = blockletIndex.get(i).min_max_index.getMin_values().get(j).array();
            maxValue = blockletIndex.get(i).min_max_index.getMax_values().get(j).array();
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMinValue[j], minValue) > 0) {
                currentMinValue[j] = minValue.clone();
            }
            if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMaxValue[j], maxValue) < 0) {
                currentMaxValue[j] = maxValue.clone();
            }
        }
        int measureIndex = 0;
        for (int j = measureStartIndex; j < currentMinValue.length; j++) {
            minValue = blockletIndex.get(i).min_max_index.getMin_values().get(j).array();
            maxValue = blockletIndex.get(i).min_max_index.getMax_values().get(j).array();
            if (CarbonMetadataUtil.compareMeasureData(currentMinValue[j], minValue, dataWriterVo.getSegmentProperties().getMeasures().get(measureIndex).getDataType()) > 0) {
                currentMinValue[j] = minValue.clone();
            }
            if (CarbonMetadataUtil.compareMeasureData(currentMaxValue[j], maxValue, dataWriterVo.getSegmentProperties().getMeasures().get(measureIndex).getDataType()) < 0) {
                currentMaxValue[j] = maxValue.clone();
            }
            measureIndex++;
        }
    }
    BlockletBTreeIndex btree = new BlockletBTreeIndex(blockletIndex.get(0).b_tree_index.getStart_key(), blockletIndex.get(blockletIndex.size() - 1).b_tree_index.getEnd_key());
    BlockletMinMaxIndex minmax = new BlockletMinMaxIndex();
    minmax.setMinValues(currentMinValue);
    minmax.setMaxValues(currentMaxValue);
    org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex blockletIndex = new org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex(btree, minmax);
    BlockIndexInfo blockIndexInfo = new BlockIndexInfo(numberOfRows, carbonDataFileName, currentPosition, blockletIndex);
    blockIndexInfoList.add(blockIndexInfo);
}