Examples with ColumnPageDecoder org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder used on opensource projects

Example 1 with ColumnPageDecoder

use of org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV2 method decodeMeasure.

protected ColumnPage decodeMeasure(MeasureRawColumnChunk measureRawColumnChunk, DataChunk2 measureColumnChunk, int copyPoint) throws MemoryException, IOException {
    assert (measureColumnChunk.getEncoder_meta().size() > 0);
    List<ByteBuffer> encoder_meta = measureColumnChunk.getEncoder_meta();
    byte[] encodedMeta = encoder_meta.get(0).array();
    ValueEncoderMeta meta = CarbonUtil.deserializeEncoderMetaV2(encodedMeta);
    ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta);
    byte[] rawData = measureRawColumnChunk.getRawData().array();
    return codec.decode(rawData, copyPoint, measureColumnChunk.data_page_length);
}

Example 2 with ColumnPageDecoder

use of org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder in project carbondata by apache.

the class DimensionRawColumnChunk method getDictionary.

/**
 * Below method will be used to get the local dictionary for a blocklet
 * @param localDictionaryChunk
 * local dictionary chunk thrift object
 * @return local dictionary
 * @throws IOException
 */
public static CarbonDictionary getDictionary(LocalDictionaryChunk localDictionaryChunk, Compressor compressor) throws IOException {
    if (null != localDictionaryChunk) {
        List<Encoding> encodings = localDictionaryChunk.getDictionary_meta().getEncoders();
        List<ByteBuffer> encoderMetas = localDictionaryChunk.getDictionary_meta().getEncoder_meta();
        ColumnPageDecoder decoder = DefaultEncodingFactory.getInstance().createDecoder(encodings, encoderMetas, compressor.getName());
        ColumnPage decode = decoder.decode(localDictionaryChunk.getDictionary_data(), 0, localDictionaryChunk.getDictionary_data().length);
        BitSet usedDictionary = BitSet.valueOf(compressor.unCompressByte(localDictionaryChunk.getDictionary_values()));
        int length = usedDictionary.length();
        int index = 0;
        byte[][] dictionary = new byte[length][];
        for (int i = 0; i < length; i++) {
            if (usedDictionary.get(i)) {
                dictionary[i] = decode.getBytes(index++);
            } else {
                dictionary[i] = null;
            }
        }
        decode.freeMemory();
        if (!usedDictionary.isEmpty()) {
            // as dictionary values starts from 1 setting null default value
            dictionary[1] = CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY;
        }
        return new CarbonDictionaryImpl(dictionary, usedDictionary.cardinality());
    }
    return null;
}

Example 3 with ColumnPageDecoder

use of org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder in project carbondata by apache.

the class AdaptiveDeltaIntegralCodec method createDecoder.

@Override
public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) {
    return new ColumnPageDecoder() {

        @Override
        public ColumnPage decode(byte[] input, int offset, int length) {
            ColumnPage page = null;
            if (DataTypes.isDecimal(meta.getSchemaDataType())) {
                page = ColumnPage.decompressDecimalPage(meta, input, offset, length);
            } else {
                page = ColumnPage.decompress(meta, input, offset, length, false, false);
            }
            return LazyColumnPage.newPage(page, converter);
        }

        @Override
        public void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) {
            Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName());
            byte[] unCompressData;
            if (null != reusableDataBuffer && compressor.supportReusableBuffer()) {
                int uncompressedLength = compressor.unCompressedLength(input, offset, length);
                unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength);
                compressor.rawUncompress(input, offset, length, unCompressData);
            } else {
                unCompressData = compressor.unCompressByte(input, offset, length);
            }
            if (DataTypes.isDecimal(meta.getSchemaDataType())) {
                TableSpec.ColumnSpec columnSpec = meta.getColumnSpec();
                vectorInfo.decimalConverter = DecimalConverterFactory.INSTANCE.getDecimalConverter(columnSpec.getPrecision(), columnSpec.getScale());
            }
            converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize);
        }

        @Override
        public ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded) {
            return decode(input, offset, length);
        }
    };
}

Example 4 with ColumnPageDecoder

use of org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder in project carbondata by apache.

the class AdaptiveDeltaFloatingCodec method createDecoder.

@Override
public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) {
    return new ColumnPageDecoder() {

        @Override
        public ColumnPage decode(byte[] input, int offset, int length) {
            ColumnPage page = ColumnPage.decompress(meta, input, offset, length, false, false);
            return LazyColumnPage.newPage(page, converter);
        }

        @Override
        public void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) {
            Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName());
            byte[] unCompressData;
            if (null != reusableDataBuffer && compressor.supportReusableBuffer()) {
                int uncompressedLength = compressor.unCompressedLength(input, offset, length);
                unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength);
                compressor.rawUncompress(input, offset, length, unCompressData);
            } else {
                unCompressData = compressor.unCompressByte(input, offset, length);
            }
            converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize);
        }

        @Override
        public ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded) {
            return decode(input, offset, length);
        }
    };
}

Example 5 with ColumnPageDecoder

use of org.apache.carbondata.core.datastore.page.encoding.ColumnPageDecoder in project carbondata by apache.

the class AdaptiveFloatingCodec method createDecoder.

@Override
public ColumnPageDecoder createDecoder(final ColumnPageEncoderMeta meta) {
    return new ColumnPageDecoder() {

        @Override
        public ColumnPage decode(byte[] input, int offset, int length) {
            ColumnPage page = ColumnPage.decompress(meta, input, offset, length, false, false);
            return LazyColumnPage.newPage(page, converter);
        }

        @Override
        public void decodeAndFillVector(byte[] input, int offset, int length, ColumnVectorInfo vectorInfo, BitSet nullBits, boolean isLVEncoded, int pageSize, ReusableDataBuffer reusableDataBuffer) {
            Compressor compressor = CompressorFactory.getInstance().getCompressor(meta.getCompressorName());
            byte[] unCompressData;
            if (null != reusableDataBuffer && compressor.supportReusableBuffer()) {
                int uncompressedLength = compressor.unCompressedLength(input, offset, length);
                unCompressData = reusableDataBuffer.getDataBuffer(uncompressedLength);
                compressor.rawUncompress(input, offset, length, unCompressData);
            } else {
                unCompressData = compressor.unCompressByte(input, offset, length);
            }
            converter.decodeAndFillVector(unCompressData, vectorInfo, nullBits, meta.getStoreDataType(), pageSize);
        }

        @Override
        public ColumnPage decode(byte[] input, int offset, int length, boolean isLVEncoded) {
            return decode(input, offset, length);
        }
    };
}