Examples with FileFooter org.apache.carbondata.format.FileFooter used on opensource projects

Example 1 with FileFooter

use of org.apache.carbondata.format.FileFooter in project carbondata by apache.

the class CarbonMetadataUtil method getFileFooter.

/**
   * Below method will be used to get the file footer object
   *
   * @param infoList         blocklet info
   * @param cardinalities    cardinlaity of dimension columns
   * @param columnSchemaList column schema list
   * @return file footer
   */
private static FileFooter getFileFooter(List<BlockletInfoColumnar> infoList, int[] cardinalities, List<ColumnSchema> columnSchemaList) {
    SegmentInfo segmentInfo = new SegmentInfo();
    segmentInfo.setNum_cols(columnSchemaList.size());
    segmentInfo.setColumn_cardinalities(CarbonUtil.convertToIntegerList(cardinalities));
    ColumnarFormatVersion version = CarbonProperties.getInstance().getFormatVersion();
    FileFooter footer = new FileFooter();
    footer.setVersion(version.number());
    footer.setNum_rows(getTotalNumberOfRows(infoList));
    footer.setSegment_info(segmentInfo);
    footer.setTable_columns(columnSchemaList);
    for (BlockletInfoColumnar info : infoList) {
        footer.addToBlocklet_index_list(getBlockletIndex(info));
    }
    return footer;
}

Example 2 with FileFooter

use of org.apache.carbondata.format.FileFooter in project carbondata by apache.

the class CarbonMetadataUtil method convertFilterFooter2.

/**
   * Below method will be used to get the file footer object for
   *
   * @param infoList         blocklet info
   * @param cardinalities    cardinality of each column
   * @param columnSchemaList column schema list
   * @param dataChunksOffset data chunks offsets
   * @param dataChunksLength data chunks length
   * @return filefooter thrift object
   */
public static FileFooter convertFilterFooter2(List<BlockletInfoColumnar> infoList, int[] cardinalities, List<ColumnSchema> columnSchemaList, List<List<Long>> dataChunksOffset, List<List<Short>> dataChunksLength) {
    FileFooter footer = getFileFooter(infoList, cardinalities, columnSchemaList);
    int index = 0;
    for (BlockletInfoColumnar info : infoList) {
        footer.addToBlocklet_info_list2(getBlockletInfo2(info, dataChunksOffset.get(index), dataChunksLength.get(index)));
        index++;
    }
    return footer;
}

Example 3 with FileFooter

use of org.apache.carbondata.format.FileFooter in project carbondata by apache.

the class CarbonFactDataWriterImplV2 method writeBlockletInfoToFile.

/**
   * This method will write metadata at the end of file file format in thrift format
   */
protected void writeBlockletInfoToFile(FileChannel channel, String filePath) throws CarbonDataWriterException {
    try {
        // get the current file position
        long currentPosition = channel.size();
        CarbonFooterWriter writer = new CarbonFooterWriter(filePath);
        // get thrift file footer instance
        FileFooter convertFileMeta = CarbonMetadataUtil.convertFilterFooter2(blockletInfoList, localCardinality, thriftColumnSchemaList, dataChunksOffsets, dataChunksLength);
        // fill the carbon index details
        fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFileName, currentPosition);
        // write the footer
        writer.writeFooter(convertFileMeta, currentPosition);
    } catch (IOException e) {
        throw new CarbonDataWriterException("Problem while writing the carbon file: ", e);
    }
}

Example 4 with FileFooter

use of org.apache.carbondata.format.FileFooter in project carbondata by apache.

the class DataFileFooterConverterTest method testReadDataFileFooter.

@Test
public void testReadDataFileFooter() throws Exception {
    DataFileFooterConverter dataFileFooterConverter = new DataFileFooterConverter();
    DataFileFooter dataFileFooter = new DataFileFooter();
    List<Integer> column_cardinalities = new ArrayList<>();
    column_cardinalities.add(new Integer("1"));
    column_cardinalities.add(new Integer("2"));
    column_cardinalities.add(new Integer("3"));
    org.apache.carbondata.format.SegmentInfo segmentInfo1 = new org.apache.carbondata.format.SegmentInfo(3, column_cardinalities);
    List<Encoding> encoders = new ArrayList<>();
    encoders.add(Encoding.INVERTED_INDEX);
    encoders.add(Encoding.BIT_PACKED);
    encoders.add(Encoding.DELTA);
    encoders.add(Encoding.DICTIONARY);
    encoders.add(Encoding.DIRECT_DICTIONARY);
    encoders.add(Encoding.RLE);
    ColumnSchema columnSchema = new ColumnSchema(DataType.INT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema1 = new ColumnSchema(DataType.ARRAY, "column", "3", true, encoders, true);
    ColumnSchema columnSchema2 = new ColumnSchema(DataType.DECIMAL, "column", "3", true, encoders, true);
    ColumnSchema columnSchema3 = new ColumnSchema(DataType.DOUBLE, "column", "3", true, encoders, true);
    ColumnSchema columnSchema4 = new ColumnSchema(DataType.LONG, "column", "3", true, encoders, true);
    ColumnSchema columnSchema5 = new ColumnSchema(DataType.SHORT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema6 = new ColumnSchema(DataType.STRUCT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema7 = new ColumnSchema(DataType.STRING, "column", "3", true, encoders, true);
    final List<ColumnSchema> columnSchemas = new ArrayList<>();
    columnSchemas.add(columnSchema);
    columnSchemas.add(columnSchema1);
    columnSchemas.add(columnSchema2);
    columnSchemas.add(columnSchema3);
    columnSchemas.add(columnSchema4);
    columnSchemas.add(columnSchema5);
    columnSchemas.add(columnSchema6);
    columnSchemas.add(columnSchema7);
    org.apache.carbondata.format.BlockletIndex blockletIndex1 = new org.apache.carbondata.format.BlockletIndex();
    List<org.apache.carbondata.format.BlockletIndex> blockletIndexArrayList = new ArrayList<>();
    blockletIndexArrayList.add(blockletIndex1);
    org.apache.carbondata.format.BlockletInfo blockletInfo = new org.apache.carbondata.format.BlockletInfo();
    List<org.apache.carbondata.format.BlockletInfo> blockletInfoArrayList = new ArrayList<>();
    blockletInfoArrayList.add(blockletInfo);
    final FileFooter fileFooter = new FileFooter(1, 3, columnSchemas, segmentInfo1, blockletIndexArrayList);
    fileFooter.setBlocklet_info_list(blockletInfoArrayList);
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    blockletBTreeIndex.setStart_key("1".getBytes());
    blockletBTreeIndex.setEnd_key("3".getBytes());
    blockletIndex1.setB_tree_index(blockletBTreeIndex);
    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
    blockletMinMaxIndex.setMax_values(Arrays.asList(ByteBuffer.allocate(1).put((byte) 2)));
    blockletMinMaxIndex.setMin_values(Arrays.asList(ByteBuffer.allocate(1).put((byte) 1)));
    blockletIndex1.setMin_max_index(blockletMinMaxIndex);
    new MockUp<FileFactory>() {

        @SuppressWarnings("unused")
        @Mock
        public FileFactory.FileType getFileType(String path) {
            return FileFactory.FileType.LOCAL;
        }

        @SuppressWarnings("unused")
        @Mock
        public FileReader getFileHolder(FileFactory.FileType fileType) {
            return new FileReaderImpl();
        }
    };
    new MockUp<FileReaderImpl>() {

        @SuppressWarnings("unused")
        @Mock
        public long readLong(String filePath, long offset) {
            return 1;
        }
    };
    new MockUp<CarbonFooterReader>() {

        @SuppressWarnings("unused")
        @Mock
        public FileFooter readFooter() throws IOException {
            return fileFooter;
        }
    };
    SegmentInfo segmentInfo = new SegmentInfo();
    int[] arr = { 1, 2, 3 };
    segmentInfo.setColumnCardinality(arr);
    dataFileFooter.setNumberOfRows(3);
    dataFileFooter.setSegmentInfo(segmentInfo);
    TableBlockInfo info = new TableBlockInfo("/file.carbondata", 1, "0", new String[0], 1, ColumnarFormatVersion.V1, null);
    DataFileFooter result = dataFileFooterConverter.readDataFileFooter(info);
    assertEquals(result.getNumberOfRows(), 3);
}

Example 5 with FileFooter

use of org.apache.carbondata.format.FileFooter in project carbondata by apache.

the class DataFileFooterConverter method readDataFileFooter.

/**
 * Below method will be used to convert thrift file meta to wrapper file meta
 */
@Override
public DataFileFooter readDataFileFooter(TableBlockInfo tableBlockInfo) throws IOException {
    DataFileFooter dataFileFooter = new DataFileFooter();
    FileReader fileReader = null;
    try {
        long completeBlockLength = tableBlockInfo.getBlockLength();
        long footerPointer = completeBlockLength - 8;
        fileReader = FileFactory.getFileHolder(FileFactory.getFileType(tableBlockInfo.getFilePath()));
        long actualFooterOffset = fileReader.readLong(tableBlockInfo.getFilePath(), footerPointer);
        CarbonFooterReader reader = new CarbonFooterReader(tableBlockInfo.getFilePath(), actualFooterOffset);
        FileFooter footer = reader.readFooter();
        dataFileFooter.setVersionId(ColumnarFormatVersion.valueOf((short) footer.getVersion()));
        dataFileFooter.setNumberOfRows(footer.getNum_rows());
        dataFileFooter.setSegmentInfo(getSegmentInfo(footer.getSegment_info()));
        List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
        List<org.apache.carbondata.format.ColumnSchema> table_columns = footer.getTable_columns();
        for (int i = 0; i < table_columns.size(); i++) {
            columnSchemaList.add(thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i)));
        }
        dataFileFooter.setColumnInTable(columnSchemaList);
        List<org.apache.carbondata.format.BlockletIndex> leaf_node_indices_Thrift = footer.getBlocklet_index_list();
        List<BlockletIndex> blockletIndexList = new ArrayList<BlockletIndex>();
        for (int i = 0; i < leaf_node_indices_Thrift.size(); i++) {
            BlockletIndex blockletIndex = getBlockletIndex(leaf_node_indices_Thrift.get(i));
            blockletIndexList.add(blockletIndex);
        }
        List<org.apache.carbondata.format.BlockletInfo> leaf_node_infos_Thrift = footer.getBlocklet_info_list();
        List<BlockletInfo> blockletInfoList = new ArrayList<BlockletInfo>();
        for (int i = 0; i < leaf_node_infos_Thrift.size(); i++) {
            BlockletInfo blockletInfo = getBlockletInfo(leaf_node_infos_Thrift.get(i));
            blockletInfo.setBlockletIndex(blockletIndexList.get(i));
            blockletInfoList.add(blockletInfo);
        }
        dataFileFooter.setBlockletList(blockletInfoList);
        dataFileFooter.setBlockletIndex(getBlockletIndexForDataFileFooter(blockletIndexList));
    } finally {
        if (null != fileReader) {
            fileReader.finish();
        }
    }
    return dataFileFooter;
}