Examples with DataFileFooter org.apache.carbondata.core.metadata.blocklet.DataFileFooter used on opensource projects

Example 16 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class SegmentTaskIndexTest method setUp.

@BeforeClass
public static void setUp() {
    segmentInfo = new SegmentInfo();
    footer = new DataFileFooter();
    columnSchema = new ColumnSchema();
    blockletInfo = new BlockletInfo();
    blockletIndex = new BlockletIndex();
}

Example 17 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class BlockIndexTest method setUp.

@BeforeClass
public static void setUp() {
    segmentInfo = new SegmentInfo();
    footer = new DataFileFooter();
    columnSchema = new ColumnSchema();
    blockletInfo = new BlockletInfo();
    blockletIndex = new BlockletIndex();
}

Example 18 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary1.

@Test
public void testBtreeSearchIsWorkingAndGivingPorperBlockletWithNoDictionary1() {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getFileFooterListWithOnlyNoDictionaryKey();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
    DataRefNode dataBlock = builder.get();
    assertTrue(dataBlock != null);
    DataRefNodeFinder finder = new BTreeDataRefNodeFinder(new int[] { -1 }, 1, 1);
    ByteBuffer buffer = ByteBuffer.allocate(4 + 2);
    buffer.rewind();
    buffer.putShort((short) 1);
    buffer.putInt(12);
    buffer.array();
    IndexKey key = new IndexKey(null, buffer.array());
    DataRefNode findFirstBlock = finder.findFirstDataBlock(dataBlock, key);
    assertEquals(1, findFirstBlock.nodeNumber());
    DataRefNode findLastBlock = finder.findLastDataBlock(dataBlock, key);
    assertEquals(1, findLastBlock.nodeNumber());
}

Example 19 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class CarbonCompactionUtil method createDataFileFooterMappingForSegments.

/**
   * To create a mapping of Segment Id and DataFileFooter.
   *
   * @param tableBlockInfoList
   * @return
   */
public static Map<String, List<DataFileFooter>> createDataFileFooterMappingForSegments(List<TableBlockInfo> tableBlockInfoList) throws IOException {
    Map<String, List<DataFileFooter>> segmentBlockInfoMapping = new HashMap<>();
    for (TableBlockInfo blockInfo : tableBlockInfoList) {
        List<DataFileFooter> eachSegmentBlocks = new ArrayList<>();
        String segId = blockInfo.getSegmentId();
        DataFileFooter dataFileMatadata = null;
        // check if segId is already present in map
        List<DataFileFooter> metadataList = segmentBlockInfoMapping.get(segId);
        dataFileMatadata = CarbonUtil.readMetadatFile(blockInfo);
        if (null == metadataList) {
            // if it is not present
            eachSegmentBlocks.add(dataFileMatadata);
            segmentBlockInfoMapping.put(segId, eachSegmentBlocks);
        } else {
            // if its already present then update the list.
            metadataList.add(dataFileMatadata);
        }
    }
    return segmentBlockInfoMapping;
}

Example 20 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class AbstractDataFileFooterConverter method getIndexInfo.

/**
   * Below method will be used to get the index info from index file
   *
   * @param filePath           file path of the index file
   * @param tableBlockInfoList table block index
   * @return list of index info
   * @throws IOException problem while reading the index file
   */
public List<DataFileFooter> getIndexInfo(String filePath, List<TableBlockInfo> tableBlockInfoList) throws IOException {
    CarbonIndexFileReader indexReader = new CarbonIndexFileReader();
    List<DataFileFooter> dataFileFooters = new ArrayList<DataFileFooter>();
    try {
        // open the reader
        indexReader.openThriftReader(filePath);
        // get the index header
        org.apache.carbondata.format.IndexHeader readIndexHeader = indexReader.readIndexHeader();
        List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
        List<org.apache.carbondata.format.ColumnSchema> table_columns = readIndexHeader.getTable_columns();
        for (int i = 0; i < table_columns.size(); i++) {
            columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
        }
        // get the segment info
        SegmentInfo segmentInfo = getSegmentInfo(readIndexHeader.getSegment_info());
        BlockletIndex blockletIndex = null;
        int counter = 0;
        int index = 0;
        DataFileFooter dataFileFooter = null;
        // read the block info from file
        while (indexReader.hasNext()) {
            BlockIndex readBlockIndexInfo = indexReader.readBlockIndexInfo();
            blockletIndex = getBlockletIndex(readBlockIndexInfo.getBlock_index());
            dataFileFooter = new DataFileFooter();
            TableBlockInfo tableBlockInfo = tableBlockInfoList.get(index);
            if (Integer.parseInt(CarbonTablePath.DataFileUtil.getPartNo(tableBlockInfo.getFilePath())) == counter++) {
                tableBlockInfo.setBlockOffset(readBlockIndexInfo.getOffset());
                tableBlockInfo.setVersion(ColumnarFormatVersion.valueOf((short) readIndexHeader.getVersion()));
                int blockletSize = getBlockletSize(readBlockIndexInfo);
                tableBlockInfo.getBlockletInfos().setNoOfBlockLets(blockletSize);
                dataFileFooter.setBlockletIndex(blockletIndex);
                dataFileFooter.setColumnInTable(columnSchemaList);
                dataFileFooter.setNumberOfRows(readBlockIndexInfo.getNum_rows());
                dataFileFooter.setBlockInfo(new BlockInfo(tableBlockInfo));
                dataFileFooter.setSegmentInfo(segmentInfo);
                dataFileFooters.add(dataFileFooter);
                if (++index == tableBlockInfoList.size()) {
                    break;
                }
            }
        }
    } finally {
        indexReader.closeThriftReader();
    }
    return dataFileFooters;
}