Examples with BlockletIndex org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex used on opensource projects

Example 6 with BlockletIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex in project carbondata by apache.

the class DataFileFooterConverterV3 method readDataFileFooter.

/**
   * Below method will be used to convert thrift file meta to wrapper file meta
   * This method will read the footer from footer offset present in the data file
   * 1. It will read the header from carbon data file, header starts from 0 offset
   * 2. It will set the stream offset
   * 3. It will read the footer data from file
   * 4. parse the footer to thrift object
   * 5. convert to wrapper object
   *
   * @param tableBlockInfo
   *        table block info
   * @return data file footer
   */
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
    DataFileFooter dataFileFooter = new DataFileFooter();
    CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(tableBlockInfo.getFilePath());
    FileHeader fileHeader = carbonHeaderReader.readHeader();
    CarbonFooterReaderV3 reader = new CarbonFooterReaderV3(tableBlockInfo.getFilePath(), tableBlockInfo.getBlockOffset());
    FileFooter3 footer = reader.readFooterVersion3();
    dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) fileHeader.getVersion()));
    dataFileFooter.setNumberOfRows(footer.getNum_rows());
    dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
    dataFileFooter.setSchemaUpdatedTimeStamp(fileHeader.getTime_stamp());
    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
    List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema();
    for (int i = 0; i < table_columns.size(); i++) {
        columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
    }
    dataFileFooter.setColumnInTable(columnSchemaList);
    List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
    List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
    for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
        BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
        blockletIndexList.add(blockletIndex);
    }
    List<org.apache.carbondata.format.BlockletInfo3> leaf_node_infos_Thrift = footer.getBlocklet_info_list3();
    List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
    for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
        BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i), getNumberOfDimensionColumns(columnSchemaList));
        blockletInfo.setBlockletIndex(blockletIndexList.get(i));
        blockletInfoList.add(blockletInfo);
    }
    dataFileFooter.setBlockletList(blockletInfoList);
    dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
    return dataFileFooter;
}

Example 7 with BlockletIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex in project carbondata by apache.

the class BTreeBlockFinderTest method getFileMatadataWithOnlyNoDictionaryKey.

private DataFileFooter getFileMatadataWithOnlyNoDictionaryKey(byte[] startKey, byte[] endKey, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) {
    DataFileFooter footer = new DataFileFooter();
    BlockletIndex index = new BlockletIndex();
    BlockletBTreeIndex btreeIndex = new BlockletBTreeIndex();
    ByteBuffer buffer = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryStartKey.length);
    buffer.putInt(0);
    buffer.putInt(noDictionaryStartKey.length);
    buffer.put(noDictionaryStartKey);
    buffer.rewind();
    btreeIndex.setStartKey(buffer.array());
    ByteBuffer buffer1 = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryEndKey.length);
    buffer1.putInt(0);
    buffer1.putInt(noDictionaryEndKey.length);
    buffer1.put(noDictionaryEndKey);
    buffer1.rewind();
    btreeIndex.setEndKey(buffer1.array());
    BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
    minMax.setMaxValues(new byte[][] { endKey, noDictionaryEndKey });
    minMax.setMinValues(new byte[][] { startKey, noDictionaryStartKey });
    index.setBtreeIndex(btreeIndex);
    index.setMinMaxIndex(minMax);
    footer.setBlockletIndex(index);
    return footer;
}

Example 8 with BlockletIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex in project carbondata by apache.

the class BTreeBlockFinderTest method getFileFooterWithOnlyDictionaryKey.

private DataFileFooter getFileFooterWithOnlyDictionaryKey(byte[] startKey, byte[] endKey, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) {
    DataFileFooter footer = new DataFileFooter();
    BlockletIndex index = new BlockletIndex();
    BlockletBTreeIndex btreeIndex = new BlockletBTreeIndex();
    ByteBuffer buffer = ByteBuffer.allocate(4 + startKey.length + 4 + 0);
    buffer.putInt(startKey.length);
    buffer.putInt(0);
    buffer.put(startKey);
    buffer.rewind();
    btreeIndex.setStartKey(buffer.array());
    ByteBuffer buffer1 = ByteBuffer.allocate(4 + 0 + 4 + noDictionaryEndKey.length);
    buffer1.putInt(endKey.length);
    buffer1.putInt(0);
    buffer1.put(endKey);
    buffer1.rewind();
    btreeIndex.setEndKey(buffer1.array());
    BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
    minMax.setMaxValues(new byte[][] { endKey });
    minMax.setMinValues(new byte[][] { startKey });
    index.setBtreeIndex(btreeIndex);
    index.setMinMaxIndex(minMax);
    footer.setBlockletIndex(index);
    return footer;
}

Example 9 with BlockletIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex in project carbondata by apache.

the class SegmentTaskIndexTest method setUp.

@BeforeClass
public static void setUp() {
    segmentInfo = new SegmentInfo();
    footer = new DataFileFooter();
    columnSchema = new ColumnSchema();
    blockletInfo = new BlockletInfo();
    blockletIndex = new BlockletIndex();
}

Example 10 with BlockletIndex

use of org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex in project carbondata by apache.

the class BlockIndexTest method setUp.

@BeforeClass
public static void setUp() {
    segmentInfo = new SegmentInfo();
    footer = new DataFileFooter();
    columnSchema = new ColumnSchema();
    blockletInfo = new BlockletInfo();
    blockletIndex = new BlockletIndex();
}