Examples with DataFileFooter org.apache.carbondata.core.metadata.blocklet.DataFileFooter used on opensource projects

Example 1 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class DataFileFooterConverter method readDataFileFooter.

/**
   * Below method will be used to convert thrift file meta to wrapper file meta
   */
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
    DataFileFooter dataFileFooter = new DataFileFooter();
    FileHolder fileReader = null;
    try {
        long completeBlockLength = tableBlockInfo.getBlockLength();
        long footerPointer = completeBlockLength - 8;
        fileReader = FileFactory.getFileHolder(FileFactory.getFileType(tableBlockInfo.getFilePath()));
        long actualFooterOffset = fileReader.readLong(tableBlockInfo.getFilePath(), footerPointer);
        CarbonFooterReader reader = new CarbonFooterReader(tableBlockInfo.getFilePath(), actualFooterOffset);
        FileFooter footer = reader.readFooter();
        dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) footer.getVersion()));
        dataFileFooter.setNumberOfRows(footer.getNum_rows());
        dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
        List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
        List<org.apache.carbondata.format.ColumnSchema> table_columns = footer.getTable_columns();
        for (int i = 0; i < table_columns.size(); i++) {
            columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
        }
        dataFileFooter.setColumnInTable(columnSchemaList);
        List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
        List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
        for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
            BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
            blockletIndexList.add(blockletIndex);
        }
        List<org.apache.carbondata.format.BlockletInfo> leaf_node_infos_Thrift = footer.getBlocklet_info_list();
        List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
        for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
            BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i));
            blockletInfo.setBlockletIndex(blockletIndexList.get(i));
            blockletInfoList.add(blockletInfo);
        }
        dataFileFooter.setBlockletList(blockletInfoList);
        dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
    } finally {
        if (null != fileReader) {
            fileReader.finish();
        }
    }
    return dataFileFooter;
}

Example 2 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class DataFileFooterConverter2 method readDataFileFooter.

/**
   * Below method will be used to convert thrift file meta to wrapper file meta
   */
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
    DataFileFooter dataFileFooter = new DataFileFooter();
    CarbonFooterReader reader = new CarbonFooterReader(tableBlockInfo.getFilePath(), tableBlockInfo.getBlockOffset());
    FileFooter footer = reader.readFooter();
    dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) footer.getVersion()));
    dataFileFooter.setNumberOfRows(footer.getNum_rows());
    dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
    List<org.apache.carbondata.format.ColumnSchema> table_columns = footer.getTable_columns();
    for (int i = 0; i < table_columns.size(); i++) {
        columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
    }
    dataFileFooter.setColumnInTable(columnSchemaList);
    List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
    List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
    for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
        BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
        blockletIndexList.add(blockletIndex);
    }
    List<org.apache.carbondata.format.BlockletInfo2> leaf_node_infos_Thrift = footer.getBlocklet_info_list2();
    List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
    for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
        BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i), getNumberOfDimensionColumns(columnSchemaList));
        blockletInfo.setBlockletIndex(blockletIndexList.get(i));
        blockletInfoList.add(blockletInfo);
    }
    dataFileFooter.setBlockletList(blockletInfoList);
    dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
    return dataFileFooter;
}

Example 3 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class DataFileFooterConverterV3 method readDataFileFooter.

/**
   * Below method will be used to convert thrift file meta to wrapper file meta
   * This method will read the footer from footer offset present in the data file
   * 1. It will read the header from carbon data file, header starts from 0 offset
   * 2. It will set the stream offset
   * 3. It will read the footer data from file
   * 4. parse the footer to thrift object
   * 5. convert to wrapper object
   *
   * @param tableBlockInfo
   *        table block info
   * @return data file footer
   */
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
    DataFileFooter dataFileFooter = new DataFileFooter();
    CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(tableBlockInfo.getFilePath());
    FileHeader fileHeader = carbonHeaderReader.readHeader();
    CarbonFooterReaderV3 reader = new CarbonFooterReaderV3(tableBlockInfo.getFilePath(), tableBlockInfo.getBlockOffset());
    FileFooter3 footer = reader.readFooterVersion3();
    dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) fileHeader.getVersion()));
    dataFileFooter.setNumberOfRows(footer.getNum_rows());
    dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
    dataFileFooter.setSchemaUpdatedTimeStamp(fileHeader.getTime_stamp());
    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
    List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema();
    for (int i = 0; i < table_columns.size(); i++) {
        columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
    }
    dataFileFooter.setColumnInTable(columnSchemaList);
    List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
    List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
    for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
        BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
        blockletIndexList.add(blockletIndex);
    }
    List<org.apache.carbondata.format.BlockletInfo3> leaf_node_infos_Thrift = footer.getBlocklet_info_list3();
    List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
    for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
        BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i), getNumberOfDimensionColumns(columnSchemaList));
        blockletInfo.setBlockletIndex(blockletIndexList.get(i));
        blockletInfoList.add(blockletInfo);
    }
    dataFileFooter.setBlockletList(blockletInfoList);
    dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
    return dataFileFooter;
}

Example 4 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class SegmentTaskIndexStore method loadBlocks.

/**
   * Below method will be used to load the blocks
   *
   * @param tableBlockInfoList
   * @return loaded segment
   * @throws IOException
   */
private AbstractIndex loadBlocks(TaskBucketHolder taskBucketHolder, List<TableBlockInfo> tableBlockInfoList, AbsoluteTableIdentifier tableIdentifier) throws IOException {
    // all the block of one task id will be loaded together
    // so creating a list which will have all the data file meta data to of one task
    List<DataFileFooter> footerList = CarbonUtil.readCarbonIndexFile(taskBucketHolder.taskNo, taskBucketHolder.bucketNumber, tableBlockInfoList, tableIdentifier);
    // Reuse SegmentProperties object if tableIdentifier, columnsInTable and columnCardinality are
    // the same.
    List<ColumnSchema> columnsInTable = footerList.get(0).getColumnInTable();
    int[] columnCardinality = footerList.get(0).getSegmentInfo().getColumnCardinality();
    SegmentPropertiesWrapper segmentPropertiesWrapper = new SegmentPropertiesWrapper(tableIdentifier, columnsInTable, columnCardinality);
    SegmentProperties segmentProperties;
    if (this.segmentProperties.containsKey(segmentPropertiesWrapper)) {
        segmentProperties = this.segmentProperties.get(segmentPropertiesWrapper);
    } else {
        // create a metadata details
        // this will be useful in query handling
        // all the data file metadata will have common segment properties we
        // can use first one to get create the segment properties
        segmentProperties = new SegmentProperties(columnsInTable, columnCardinality);
        this.segmentProperties.put(segmentPropertiesWrapper, segmentProperties);
    }
    AbstractIndex segment = new SegmentTaskIndex(segmentProperties);
    // file path of only first block is passed as it all table block info path of
    // same task id will be same
    segment.buildIndex(footerList);
    return segment;
}

Example 5 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey1.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithDictionaryKey1() throws KeyGenException {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { 2, 2 }, 2, 0);
    int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
    KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    IndexKey key = new IndexKey(multiDimKeyVarLengthGenerator.generateKey(new int[] { 1, 1 }), null);
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(0, findFirstBlock.nodeNumber());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(0, findLastBlock.nodeNumber());
}