use of org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex in project carbondata by apache.
the class AbstractDataFileFooterConverter method getBlockletIndexForDataFileFooter.
/**
* Below method will be used to get blocklet index for data file meta
*
* @param blockletIndexList
* @return blocklet index
*/
protected BlockletIndex getBlockletIndexForDataFileFooter(List<BlockletIndex> blockletIndexList) {
BlockletIndex blockletIndex = new BlockletIndex();
BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
blockletBTreeIndex.setStartKey(blockletIndexList.get(0).getBtreeIndex().getStartKey());
blockletBTreeIndex.setEndKey(blockletIndexList.get(blockletIndexList.size() - 1).getBtreeIndex().getEndKey());
blockletIndex.setBtreeIndex(blockletBTreeIndex);
byte[][] currentMinValue = blockletIndexList.get(0).getMinMaxIndex().getMinValues().clone();
byte[][] currentMaxValue = blockletIndexList.get(0).getMinMaxIndex().getMaxValues().clone();
byte[][] minValue = null;
byte[][] maxValue = null;
for (int i = 1; i < blockletIndexList.size(); i++) {
minValue = blockletIndexList.get(i).getMinMaxIndex().getMinValues();
maxValue = blockletIndexList.get(i).getMinMaxIndex().getMaxValues();
for (int j = 0; j < maxValue.length; j++) {
if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMinValue[j], minValue[j]) > 0) {
currentMinValue[j] = minValue[j].clone();
}
if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(currentMaxValue[j], maxValue[j]) < 0) {
currentMaxValue[j] = maxValue[j].clone();
}
}
}
BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
minMax.setMaxValues(currentMaxValue);
minMax.setMinValues(currentMinValue);
blockletIndex.setMinMaxIndex(minMax);
return blockletIndex;
}
use of org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex in project carbondata by apache.
the class AbstractDataFileFooterConverter method getBlockletIndex.
/**
* Below method will be used to convert the blocklet index of thrift to
* wrapper
*
* @param blockletIndexThrift
* @return blocklet index wrapper
*/
protected BlockletIndex getBlockletIndex(org.apache.carbondata.format.BlockletIndex blockletIndexThrift) {
org.apache.carbondata.format.BlockletBTreeIndex btreeIndex = blockletIndexThrift.getB_tree_index();
org.apache.carbondata.format.BlockletMinMaxIndex minMaxIndex = blockletIndexThrift.getMin_max_index();
return new BlockletIndex(new BlockletBTreeIndex(btreeIndex.getStart_key(), btreeIndex.getEnd_key()), new BlockletMinMaxIndex(minMaxIndex.getMin_values(), minMaxIndex.getMax_values()));
}
use of org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex in project carbondata by apache.
the class DataFileFooterConverter method readDataFileFooter.
/**
* Below method will be used to convert thrift file meta to wrapper file meta
*/
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
DataFileFooter dataFileFooter = new DataFileFooter();
FileHolder fileReader = null;
try {
long completeBlockLength = tableBlockInfo.getBlockLength();
long footerPointer = completeBlockLength - 8;
fileReader = FileFactory.getFileHolder(FileFactory.getFileType(tableBlockInfo.getFilePath()));
long actualFooterOffset = fileReader.readLong(tableBlockInfo.getFilePath(), footerPointer);
CarbonFooterReader reader = new CarbonFooterReader(tableBlockInfo.getFilePath(), actualFooterOffset);
FileFooter footer = reader.readFooter();
dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) footer.getVersion()));
dataFileFooter.setNumberOfRows(footer.getNum_rows());
dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
List<org.apache.carbondata.format.ColumnSchema> table_columns = footer.getTable_columns();
for (int i = 0; i < table_columns.size(); i++) {
columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
}
dataFileFooter.setColumnInTable(columnSchemaList);
List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
blockletIndexList.add(blockletIndex);
}
List<org.apache.carbondata.format.BlockletInfo> leaf_node_infos_Thrift = footer.getBlocklet_info_list();
List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i));
blockletInfo.setBlockletIndex(blockletIndexList.get(i));
blockletInfoList.add(blockletInfo);
}
dataFileFooter.setBlockletList(blockletInfoList);
dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
} finally {
if (null != fileReader) {
fileReader.finish();
}
}
return dataFileFooter;
}
use of org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex in project carbondata by apache.
the class DataFileFooterConverter2 method readDataFileFooter.
/**
* Below method will be used to convert thrift file meta to wrapper file meta
*/
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
DataFileFooter dataFileFooter = new DataFileFooter();
CarbonFooterReader reader = new CarbonFooterReader(tableBlockInfo.getFilePath(), tableBlockInfo.getBlockOffset());
FileFooter footer = reader.readFooter();
dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) footer.getVersion()));
dataFileFooter.setNumberOfRows(footer.getNum_rows());
dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
List<org.apache.carbondata.format.ColumnSchema> table_columns = footer.getTable_columns();
for (int i = 0; i < table_columns.size(); i++) {
columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
}
dataFileFooter.setColumnInTable(columnSchemaList);
List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
blockletIndexList.add(blockletIndex);
}
List<org.apache.carbondata.format.BlockletInfo2> leaf_node_infos_Thrift = footer.getBlocklet_info_list2();
List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i), getNumberOfDimensionColumns(columnSchemaList));
blockletInfo.setBlockletIndex(blockletIndexList.get(i));
blockletInfoList.add(blockletInfo);
}
dataFileFooter.setBlockletList(blockletInfoList);
dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
return dataFileFooter;
}
use of org.apache.carbondata.core.metadata.blocklet.index.BlockletIndex in project carbondata by apache.
the class DataFileFooterConverterV3 method readDataFileFooter.
/**
* Below method will be used to convert thrift file meta to wrapper file meta
* This method will read the footer from footer offset present in the data file
* 1. It will read the header from carbon data file, header starts from 0 offset
* 2. It will set the stream offset
* 3. It will read the footer data from file
* 4. parse the footer to thrift object
* 5. convert to wrapper object
*
* @param tableBlockInfo
* table block info
* @return data file footer
*/
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
DataFileFooter dataFileFooter = new DataFileFooter();
CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(tableBlockInfo.getFilePath());
FileHeader fileHeader = carbonHeaderReader.readHeader();
CarbonFooterReaderV3 reader = new CarbonFooterReaderV3(tableBlockInfo.getFilePath(), tableBlockInfo.getBlockOffset());
FileFooter3 footer = reader.readFooterVersion3();
dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) fileHeader.getVersion()));
dataFileFooter.setNumberOfRows(footer.getNum_rows());
dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
dataFileFooter.setSchemaUpdatedTimeStamp(fileHeader.getTime_stamp());
List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema();
for (int i = 0; i < table_columns.size(); i++) {
columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
}
dataFileFooter.setColumnInTable(columnSchemaList);
List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
blockletIndexList.add(blockletIndex);
}
List<org.apache.carbondata.format.BlockletInfo3> leaf_node_infos_Thrift = footer.getBlocklet_info_list3();
List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i), getNumberOfDimensionColumns(columnSchemaList));
blockletInfo.setBlockletIndex(blockletIndexList.get(i));
blockletInfoList.add(blockletInfo);
}
dataFileFooter.setBlockletList(blockletInfoList);
dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
return dataFileFooter;
}
Aggregations