Examples with ValueEncoderMeta org.apache.carbondata.core.metadata.ValueEncoderMeta used on opensource projects

Example 6 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV2 method convertToMeasureChunk.

public MeasureColumnDataChunk convertToMeasureChunk(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException {
    MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
    DataChunk2 measureColumnChunk = null;
    int copyPoint = measureRawColumnChunk.getOffSet();
    int blockIndex = measureRawColumnChunk.getBlockletId();
    ByteBuffer rawData = measureRawColumnChunk.getRawData();
    if (measureColumnChunkOffsets.size() - 1 == blockIndex) {
        measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint, measureColumnChunkLength.get(blockIndex));
        synchronized (measureRawColumnChunk.getFileReader()) {
            rawData = measureRawColumnChunk.getFileReader().readByteBuffer(filePath, measureColumnChunkOffsets.get(blockIndex) + measureColumnChunkLength.get(blockIndex), measureColumnChunk.data_page_length);
        }
    } else {
        measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint, measureColumnChunkLength.get(blockIndex));
        copyPoint += measureColumnChunkLength.get(blockIndex);
    }
    List<ValueEncoderMeta> valueEncodeMeta = new ArrayList<>();
    for (int i = 0; i < measureColumnChunk.getEncoder_meta().size(); i++) {
        valueEncodeMeta.add(CarbonUtil.deserializeEncoderMeta(measureColumnChunk.getEncoder_meta().get(i).array()));
    }
    WriterCompressModel compressionModel = CarbonUtil.getValueCompressionModel(valueEncodeMeta);
    ValueCompressionHolder values = compressionModel.getValueCompressionHolder()[0];
    // uncompress
    values.uncompress(compressionModel.getConvertedDataType()[0], rawData.array(), copyPoint, measureColumnChunk.data_page_length, compressionModel.getMantissa()[0], compressionModel.getMaxValue()[0], numberOfRows);
    CarbonReadDataHolder measureDataHolder = new CarbonReadDataHolder(values);
    // set the data chunk
    datChunk.setMeasureDataHolder(measureDataHolder);
    // set the enun value indexes
    datChunk.setNullValueIndexHolder(getPresenceMeta(measureColumnChunk.presence));
    return datChunk;
}

Example 7 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV3 method convertToMeasureChunk.

/**
   * Below method will be used to convert the compressed measure chunk raw data to actual data
   *
   * @param measureRawColumnChunk measure raw chunk
   * @param pageNumber            number
   * @return DimensionColumnDataChunk
   */
@Override
public MeasureColumnDataChunk convertToMeasureChunk(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException {
    MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
    // data chunk of blocklet column
    DataChunk3 dataChunk3 = measureRawColumnChunk.getDataChunkV3();
    // data chunk of page
    DataChunk2 measureColumnChunk = dataChunk3.getData_chunk_list().get(pageNumber);
    // calculating the start point of data
    // as buffer can contain multiple column data, start point will be datachunkoffset +
    // data chunk length + page offset
    int copyPoint = measureRawColumnChunk.getOffSet() + measureColumnChunkLength.get(measureRawColumnChunk.getBlockletId()) + dataChunk3.getPage_offset().get(pageNumber);
    List<ValueEncoderMeta> valueEncodeMeta = new ArrayList<>();
    for (int i = 0; i < measureColumnChunk.getEncoder_meta().size(); i++) {
        valueEncodeMeta.add(CarbonUtil.deserializeEncoderMetaNew(measureColumnChunk.getEncoder_meta().get(i).array()));
    }
    WriterCompressModel compressionModel = CarbonUtil.getValueCompressionModel(valueEncodeMeta);
    ValueCompressionHolder values = compressionModel.getValueCompressionHolder()[0];
    // uncompress
    ByteBuffer rawData = measureRawColumnChunk.getRawData();
    values.uncompress(compressionModel.getConvertedDataType()[0], rawData.array(), copyPoint, measureColumnChunk.data_page_length, compressionModel.getMantissa()[0], compressionModel.getMaxValue()[0], measureRawColumnChunk.getRowCount()[pageNumber]);
    CarbonReadDataHolder measureDataHolder = new CarbonReadDataHolder(values);
    // set the data chunk
    datChunk.setMeasureDataHolder(measureDataHolder);
    // set the null value indexes
    datChunk.setNullValueIndexHolder(getPresenceMeta(measureColumnChunk.presence));
    return datChunk;
}

Example 8 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CarbonUtil method deserializeEncoderMeta.

/**
   * Below method will be used to convert the encode metadata to
   * ValueEncoderMeta object
   *
   * @param encoderMeta
   * @return ValueEncoderMeta object
   */
public static ValueEncoderMeta deserializeEncoderMeta(byte[] encoderMeta) {
    // TODO : should remove the unnecessary fields.
    ByteArrayInputStream aos = null;
    ObjectInputStream objStream = null;
    ValueEncoderMeta meta = null;
    try {
        aos = new ByteArrayInputStream(encoderMeta);
        objStream = new ObjectInputStream(aos);
        meta = (ValueEncoderMeta) objStream.readObject();
    } catch (ClassNotFoundException e) {
        LOGGER.error(e);
    } catch (IOException e) {
        CarbonUtil.closeStreams(objStream);
    }
    return meta;
}

Example 9 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CarbonMetadataUtil method convertBlockletInfo.

/**
   * It converts FileFooter thrift object to list of BlockletInfoColumnar
   * objects
   *
   * @param footer
   * @return
   */
public static List<BlockletInfoColumnar> convertBlockletInfo(FileFooter footer) throws IOException {
    List<BlockletInfoColumnar> listOfNodeInfo = new ArrayList<BlockletInfoColumnar>(CarbonCommonConstants.CONSTANT_SIZE_TEN);
    for (BlockletInfo blockletInfo : footer.getBlocklet_info_list()) {
        BlockletInfoColumnar blockletInfoColumnar = new BlockletInfoColumnar();
        blockletInfoColumnar.setNumberOfKeys(blockletInfo.getNum_rows());
        List<DataChunk> columnChunks = blockletInfo.getColumn_data_chunks();
        List<DataChunk> dictChunks = new ArrayList<DataChunk>();
        List<DataChunk> nonDictColChunks = new ArrayList<DataChunk>();
        for (DataChunk dataChunk : columnChunks) {
            if (dataChunk.getEncoders().get(0).equals(Encoding.DICTIONARY)) {
                dictChunks.add(dataChunk);
            } else {
                nonDictColChunks.add(dataChunk);
            }
        }
        int[] keyLengths = new int[dictChunks.size()];
        long[] keyOffSets = new long[dictChunks.size()];
        long[] keyBlockIndexOffsets = new long[dictChunks.size()];
        int[] keyBlockIndexLens = new int[dictChunks.size()];
        long[] indexMapOffsets = new long[dictChunks.size()];
        int[] indexMapLens = new int[dictChunks.size()];
        boolean[] sortState = new boolean[dictChunks.size()];
        int i = 0;
        for (DataChunk dataChunk : dictChunks) {
            keyLengths[i] = dataChunk.getData_page_length();
            keyOffSets[i] = dataChunk.getData_page_offset();
            keyBlockIndexOffsets[i] = dataChunk.getRowid_page_offset();
            keyBlockIndexLens[i] = dataChunk.getRowid_page_length();
            indexMapOffsets[i] = dataChunk.getRle_page_offset();
            indexMapLens[i] = dataChunk.getRle_page_length();
            sortState[i] = dataChunk.getSort_state().equals(SortState.SORT_EXPLICIT);
            i++;
        }
        blockletInfoColumnar.setKeyLengths(keyLengths);
        blockletInfoColumnar.setKeyOffSets(keyOffSets);
        blockletInfoColumnar.setKeyBlockIndexOffSets(keyBlockIndexOffsets);
        blockletInfoColumnar.setKeyBlockIndexLength(keyBlockIndexLens);
        blockletInfoColumnar.setDataIndexMapOffsets(indexMapOffsets);
        blockletInfoColumnar.setDataIndexMapLength(indexMapLens);
        blockletInfoColumnar.setIsSortedKeyColumn(sortState);
        int[] msrLens = new int[nonDictColChunks.size()];
        long[] msrOffsets = new long[nonDictColChunks.size()];
        ValueEncoderMeta[] encoderMetas = new ValueEncoderMeta[nonDictColChunks.size()];
        i = 0;
        for (DataChunk msrChunk : nonDictColChunks) {
            msrLens[i] = msrChunk.getData_page_length();
            msrOffsets[i] = msrChunk.getData_page_offset();
            encoderMetas[i] = deserializeValueEncoderMeta(msrChunk.getEncoder_meta().get(0));
            i++;
        }
        blockletInfoColumnar.setMeasureLength(msrLens);
        blockletInfoColumnar.setMeasureOffset(msrOffsets);
        blockletInfoColumnar.setCompressionModel(getValueCompressionModel(encoderMetas));
        listOfNodeInfo.add(blockletInfoColumnar);
    }
    setBlockletIndex(footer, listOfNodeInfo);
    return listOfNodeInfo;
}

Example 10 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV1 method convertToMeasureChunk.

@Override
public MeasureColumnDataChunk convertToMeasureChunk(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException {
    int blockIndex = measureRawColumnChunk.getBlockletId();
    DataChunk dataChunk = measureColumnChunks.get(blockIndex);
    ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0);
    ReaderCompressModel compressModel = ValueCompressionUtil.getReaderCompressModel(meta);
    ValueCompressionHolder values = compressModel.getValueCompressionHolder();
    ByteBuffer rawData = measureRawColumnChunk.getRawData();
    // unCompress data
    values.uncompress(compressModel.getConvertedDataType(), rawData.array(), measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength(), compressModel.getMantissa(), compressModel.getMaxValue(), numberOfRows);
    CarbonReadDataHolder measureDataHolder = new CarbonReadDataHolder(values);
    // create and set the data chunk
    MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
    datChunk.setMeasureDataHolder(measureDataHolder);
    // set the enun value indexes
    datChunk.setNullValueIndexHolder(dataChunk.getNullValueIndexForColumn());
    return datChunk;
}