Search in sources :

Example 1 with RLECodec

use of org.apache.carbondata.core.datastore.page.encoding.rle.RLECodec in project carbondata by apache.

the class RLECodecSuite method testBytePageEncode.

private void testBytePageEncode(ColumnPage inputPage, byte[] expectedEncodedBytes) throws IOException, MemoryException {
    RLECodec codec = new RLECodec();
    ColumnPageEncoder encoder = codec.createEncoder(null);
    EncodedColumnPage result = encoder.encode(inputPage);
    byte[] encoded = result.getEncodedData().array();
    assertEquals(expectedEncodedBytes.length, encoded.length);
    for (int i = 0; i < encoded.length; i++) {
        assertEquals(expectedEncodedBytes[i], encoded[i]);
    }
}
Also used : RLECodec(org.apache.carbondata.core.datastore.page.encoding.rle.RLECodec)

Example 2 with RLECodec

use of org.apache.carbondata.core.datastore.page.encoding.rle.RLECodec in project carbondata by apache.

the class RLECodecSuite method testBytePageDecode.

private void testBytePageDecode(byte[] inputBytes, byte[] expectedDecodedBytes) throws IOException, MemoryException {
    RLECodec codec = new RLECodec();
    RLEEncoderMeta meta = new RLEEncoderMeta(TableSpec.ColumnSpec.newInstance("test", DataTypes.BYTE, ColumnType.MEASURE), DataTypes.BYTE, expectedDecodedBytes.length, null);
    ColumnPageDecoder decoder = codec.createDecoder(meta);
    ColumnPage page = decoder.decode(inputBytes, 0, inputBytes.length);
    byte[] decoded = page.getBytePage();
    assertEquals(expectedDecodedBytes.length, decoded.length);
    for (int i = 0; i < decoded.length; i++) {
        assertEquals(expectedDecodedBytes[i], decoded[i]);
    }
}
Also used : ColumnPage(org.apache.carbondata.core.datastore.page.ColumnPage) RLECodec(org.apache.carbondata.core.datastore.page.encoding.rle.RLECodec) RLEEncoderMeta(org.apache.carbondata.core.datastore.page.encoding.rle.RLEEncoderMeta)

Example 3 with RLECodec

use of org.apache.carbondata.core.datastore.page.encoding.rle.RLECodec in project carbondata by apache.

the class EncodingFactory method createDecoder.

/**
 * Return new decoder based on encoder metadata read from file
 */
public ColumnPageDecoder createDecoder(List<Encoding> encodings, List<ByteBuffer> encoderMetas) throws IOException {
    assert (encodings.size() == 1);
    assert (encoderMetas.size() == 1);
    Encoding encoding = encodings.get(0);
    byte[] encoderMeta = encoderMetas.get(0).array();
    ByteArrayInputStream stream = new ByteArrayInputStream(encoderMeta);
    DataInputStream in = new DataInputStream(stream);
    if (encoding == DIRECT_COMPRESS) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        return new DirectCompressCodec(metadata.getStoreDataType()).createDecoder(metadata);
    } else if (encoding == ADAPTIVE_INTEGRAL) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveIntegralCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats).createDecoder(metadata);
    } else if (encoding == ADAPTIVE_DELTA_INTEGRAL) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveDeltaIntegralCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats).createDecoder(metadata);
    } else if (encoding == ADAPTIVE_FLOATING) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveFloatingCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats).createDecoder(metadata);
    } else if (encoding == ADAPTIVE_DELTA_FLOATING) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveDeltaFloatingCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats).createDecoder(metadata);
    } else if (encoding == RLE_INTEGRAL) {
        RLEEncoderMeta metadata = new RLEEncoderMeta();
        metadata.readFields(in);
        return new RLECodec().createDecoder(metadata);
    } else if (encoding == BOOL_BYTE) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        return new DirectCompressCodec(metadata.getStoreDataType()).createDecoder(metadata);
    } else {
        // for backward compatibility
        ValueEncoderMeta metadata = CarbonUtil.deserializeEncoderMetaV3(encoderMeta);
        return createDecoderLegacy(metadata);
    }
}
Also used : RLEEncoderMeta(org.apache.carbondata.core.datastore.page.encoding.rle.RLEEncoderMeta) Encoding(org.apache.carbondata.format.Encoding) DataInputStream(java.io.DataInputStream) AdaptiveDeltaIntegralCodec(org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveDeltaIntegralCodec) AdaptiveFloatingCodec(org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveFloatingCodec) SimpleStatsResult(org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult) AdaptiveIntegralCodec(org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveIntegralCodec) ByteArrayInputStream(java.io.ByteArrayInputStream) RLECodec(org.apache.carbondata.core.datastore.page.encoding.rle.RLECodec) DirectCompressCodec(org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec) AdaptiveDeltaFloatingCodec(org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveDeltaFloatingCodec) ValueEncoderMeta(org.apache.carbondata.core.metadata.ValueEncoderMeta)

Aggregations

RLECodec (org.apache.carbondata.core.datastore.page.encoding.rle.RLECodec)3 RLEEncoderMeta (org.apache.carbondata.core.datastore.page.encoding.rle.RLEEncoderMeta)2 ByteArrayInputStream (java.io.ByteArrayInputStream)1 DataInputStream (java.io.DataInputStream)1 ColumnPage (org.apache.carbondata.core.datastore.page.ColumnPage)1 AdaptiveDeltaFloatingCodec (org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveDeltaFloatingCodec)1 AdaptiveDeltaIntegralCodec (org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveDeltaIntegralCodec)1 AdaptiveFloatingCodec (org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveFloatingCodec)1 AdaptiveIntegralCodec (org.apache.carbondata.core.datastore.page.encoding.adaptive.AdaptiveIntegralCodec)1 DirectCompressCodec (org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec)1 SimpleStatsResult (org.apache.carbondata.core.datastore.page.statistics.SimpleStatsResult)1 ValueEncoderMeta (org.apache.carbondata.core.metadata.ValueEncoderMeta)1 Encoding (org.apache.carbondata.format.Encoding)1