Examples with ColumnPage org.apache.carbondata.core.datastore.page.ColumnPage used on opensource projects

Example 6 with ColumnPage

use of org.apache.carbondata.core.datastore.page.ColumnPage in project carbondata by apache.

the class AdaptiveIntegralCodec method createEncoder.

@Override
public ColumnPageEncoder createEncoder(Map<String, String> parameter) {
    final Compressor compressor = CompressorFactory.getInstance().getCompressor();
    return new ColumnPageEncoder() {

        @Override
        protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException {
            if (encodedPage != null) {
                throw new IllegalStateException("already encoded");
            }
            encodedPage = ColumnPage.newPage(input.getColumnSpec(), targetDataType, input.getPageSize());
            input.convertValue(converter);
            byte[] result = encodedPage.compress(compressor);
            encodedPage.freeMemory();
            return result;
        }

        @Override
        protected List<Encoding> getEncodingList() {
            List<Encoding> encodings = new ArrayList<Encoding>();
            encodings.add(Encoding.ADAPTIVE_INTEGRAL);
            return encodings;
        }

        @Override
        protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) {
            return new ColumnPageEncoderMeta(inputPage.getColumnSpec(), targetDataType, stats, compressor.getName());
        }
    };
}

Example 7 with ColumnPage

use of org.apache.carbondata.core.datastore.page.ColumnPage in project carbondata by apache.

the class ComplexDimensionIndexCodec method createEncoder.

@Override
public ColumnPageEncoder createEncoder(Map<String, String> parameter) {
    return new IndexStorageEncoder() {

        @Override
        void encodeIndexStorage(ColumnPage inputPage) {
            IndexStorage indexStorage = new BlockIndexerStorageForShort(inputPage.getByteArrayPage(), false, false, false);
            byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage());
            byte[] compressed = compressor.compressByte(flattened);
            super.indexStorage = indexStorage;
            super.compressedDataPage = compressed;
        }

        @Override
        protected List<Encoding> getEncodingList() {
            List<Encoding> encodings = new ArrayList<>();
            encodings.add(Encoding.DICTIONARY);
            encodings.add(Encoding.INVERTED_INDEX);
            return encodings;
        }
    };
}

Example 8 with ColumnPage

use of org.apache.carbondata.core.datastore.page.ColumnPage in project carbondata by apache.

the class DictDimensionIndexCodec method createEncoder.

@Override
public ColumnPageEncoder createEncoder(Map<String, String> parameter) {
    return new IndexStorageEncoder() {

        @Override
        void encodeIndexStorage(ColumnPage inputPage) {
            IndexStorage indexStorage;
            byte[][] data = inputPage.getByteArrayPage();
            if (isInvertedIndex) {
                indexStorage = new BlockIndexerStorageForShort(data, true, false, isSort);
            } else {
                indexStorage = new BlockIndexerStorageForNoInvertedIndexForShort(data, false);
            }
            byte[] flattened = ByteUtil.flatten(indexStorage.getDataPage());
            super.compressedDataPage = compressor.compressByte(flattened);
            super.indexStorage = indexStorage;
        }

        @Override
        protected List<Encoding> getEncodingList() {
            List<Encoding> encodings = new ArrayList<>();
            encodings.add(Encoding.DICTIONARY);
            encodings.add(Encoding.RLE);
            if (isInvertedIndex) {
                encodings.add(Encoding.INVERTED_INDEX);
            }
            return encodings;
        }
    };
}

Example 9 with ColumnPage

use of org.apache.carbondata.core.datastore.page.ColumnPage in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV1 method decodeColumnPage.

@Override
public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException, MemoryException {
    int blockIndex = measureRawColumnChunk.getColumnIndex();
    DataChunk dataChunk = measureColumnChunks.get(blockIndex);
    ValueEncoderMeta meta = dataChunk.getValueEncoderMeta().get(0);
    ColumnPageDecoder codec = encodingFactory.createDecoderLegacy(meta);
    ColumnPage decodedPage = codec.decode(measureRawColumnChunk.getRawData().array(), (int) measureRawColumnChunk.getOffSet(), dataChunk.getDataPageLength());
    decodedPage.setNullBits(dataChunk.getNullValueIndexForColumn());
    return decodedPage;
}

Example 10 with ColumnPage

use of org.apache.carbondata.core.datastore.page.ColumnPage in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV2 method decodeColumnPage.

public ColumnPage decodeColumnPage(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException, MemoryException {
    int copyPoint = (int) measureRawColumnChunk.getOffSet();
    int blockIndex = measureRawColumnChunk.getColumnIndex();
    ByteBuffer rawData = measureRawColumnChunk.getRawData();
    DataChunk2 measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint, measureColumnChunkLength.get(blockIndex));
    copyPoint += measureColumnChunkLength.get(blockIndex);
    ColumnPage page = decodeMeasure(measureRawColumnChunk, measureColumnChunk, copyPoint);
    page.setNullBits(getNullBitSet(measureColumnChunk.presence));
    return page;
}