Examples with FileFooter3 org.apache.carbondata.format.FileFooter3 used on opensource projects

Example 1 with FileFooter3

use of org.apache.carbondata.format.FileFooter3 in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method writeBlockletInfoToFile.

@Override
protected void writeBlockletInfoToFile(FileChannel channel, String filePath) throws CarbonDataWriterException {
    try {
        // get the current file position
        long currentPosition = channel.size();
        // get thrift file footer instance
        FileFooter3 convertFileMeta = CarbonMetadataUtil.convertFileFooterVersion3(blockletMetadata, blockletIndex, localCardinality, thriftColumnSchemaList.size(), dataWriterVo.getSegmentProperties());
        // fill the carbon index details
        fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFileName, currentPosition);
        // write the footer
        byte[] byteArray = CarbonUtil.getByteArray(convertFileMeta);
        ByteBuffer buffer = ByteBuffer.allocate(byteArray.length + CarbonCommonConstants.LONG_SIZE_IN_BYTE);
        buffer.put(byteArray);
        buffer.putLong(currentPosition);
        buffer.flip();
        channel.write(buffer);
    } catch (IOException e) {
        throw new CarbonDataWriterException("Problem while writing the carbon file: ", e);
    }
}

Example 2 with FileFooter3

use of org.apache.carbondata.format.FileFooter3 in project carbondata by apache.

the class CarbonFooterReaderV3 method readFooterVersion3.

/**
 * It reads the metadata in FileFooter thrift object format.
 *
 * @return
 * @throws IOException
 */
public FileFooter3 readFooterVersion3() throws IOException {
    ThriftReader thriftReader = openThriftReader(filePath);
    thriftReader.open();
    // Set the offset from where it should read
    thriftReader.setReadOffset(footerOffset);
    FileFooter3 footer = (FileFooter3) thriftReader.read();
    thriftReader.close();
    return footer;
}

Example 3 with FileFooter3

use of org.apache.carbondata.format.FileFooter3 in project carbondata by apache.

the class DataFileFooterConverterV3 method readDataFileFooter.

/**
 * Below method will be used to convert thrift file meta to wrapper file meta
 * This method will read the footer from footer offset present in the data file
 * 1. It will read the header from carbon data file, header starts from 0 offset
 * 2. It will set the stream offset
 * 3. It will read the footer data from file
 * 4. parse the footer to thrift object
 * 5. convert to wrapper object
 *
 * @param tableBlockInfo
 *        table block info
 * @return data file footer
 */
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
    DataFileFooter dataFileFooter = new DataFileFooter();
    CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(tableBlockInfo.getFilePath());
    FileHeader fileHeader = carbonHeaderReader.readHeader();
    CarbonFooterReaderV3 reader = new CarbonFooterReaderV3(tableBlockInfo.getFilePath(), tableBlockInfo.getBlockOffset());
    FileFooter3 footer = reader.readFooterVersion3();
    dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) fileHeader.getVersion()));
    dataFileFooter.setNumberOfRows(footer.getNum_rows());
    dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
    dataFileFooter.setSchemaUpdatedTimeStamp(fileHeader.getTime_stamp());
    List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
    List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema();
    for (int i = 0; i < table_columns.size(); i++) {
        columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
    }
    dataFileFooter.setColumnInTable(columnSchemaList);
    List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
    List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
    for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
        BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
        blockletIndexList.add(blockletIndex);
    }
    List<org.apache.carbondata.format.BlockletInfo3> leaf_node_infos_Thrift = footer.getBlocklet_info_list3();
    List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
    for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
        BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i), CarbonUtil.getNumberOfDimensionColumns(columnSchemaList));
        blockletInfo.setBlockletIndex(blockletIndexList.get(i));
        blockletInfoList.add(blockletInfo);
    }
    dataFileFooter.setBlockletList(blockletInfoList);
    dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
    return dataFileFooter;
}

Example 4 with FileFooter3

use of org.apache.carbondata.format.FileFooter3 in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method writeBlockletInfoToFile.

@Override
protected void writeBlockletInfoToFile() throws CarbonDataWriterException {
    try {
        // get the current file position
        long currentPosition = currentOffsetInFile;
        // get thrift file footer instance
        FileFooter3 convertFileMeta = CarbonMetadataUtil.convertFileFooterVersion3(blockletMetadata, blockletIndex, localCardinality, thriftColumnSchemaList.size());
        // fill the carbon index details
        fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFileName, currentPosition);
        // write the footer
        byte[] byteArray = CarbonUtil.getByteArray(convertFileMeta);
        ByteBuffer buffer = ByteBuffer.allocate(byteArray.length + CarbonCommonConstants.LONG_SIZE_IN_BYTE);
        buffer.put(byteArray);
        buffer.putLong(currentPosition);
        buffer.flip();
        currentOffsetInFile += fileChannel.write(buffer);
    } catch (IOException e) {
        LOGGER.error(e, "Problem while writing the carbon file");
        throw new CarbonDataWriterException("Problem while writing the carbon file: ", e);
    }
}

Example 5 with FileFooter3

use of org.apache.carbondata.format.FileFooter3 in project carbondata by apache.

the class CarbonMetadataUtil method getFileFooter3.

/**
 * Below method will be used to get the file footer object
 *
 * @param infoList         blocklet info
 * @param blockletIndexs
 * @param cardinalities    cardinlaity of dimension columns
 * @param numberOfColumns
 * @return file footer
 */
private static FileFooter3 getFileFooter3(List<BlockletInfo3> infoList, List<BlockletIndex> blockletIndexs, int[] cardinalities, int numberOfColumns) {
    SegmentInfo segmentInfo = new SegmentInfo();
    segmentInfo.setNum_cols(numberOfColumns);
    segmentInfo.setColumn_cardinalities(CarbonUtil.convertToIntegerList(cardinalities));
    FileFooter3 footer = new FileFooter3();
    footer.setNum_rows(getNumberOfRowForFooter(infoList));
    footer.setSegment_info(segmentInfo);
    for (BlockletIndex info : blockletIndexs) {
        footer.addToBlocklet_index_list(info);
    }
    return footer;
}