Examples with ValueEncoderMeta org.apache.carbondata.core.metadata.ValueEncoderMeta used on opensource projects

Example 16 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class EncodingFactory method createDecoder.

/**
 * Return new decoder based on encoder metadata read from file
 * @param encodings encodings used to decode the page
 * @param encoderMetas metadata of encodings to decode the data
 * @param compressor Compressor name which will be used to decode data.
 * @param fullVectorFill whether the flow should go to fill the given vector completely while
 *                       decoding the data itself.
 * @return decoder to decode page.
 * @throws IOException
 */
public ColumnPageDecoder createDecoder(List<Encoding> encodings, List<ByteBuffer> encoderMetas, String compressor, boolean fullVectorFill) throws IOException {
    assert (encodings.size() >= 1);
    assert (encoderMetas.size() == 1);
    boolean isComplexPrimitiveIntLengthEncoding = encodings.contains(Encoding.INT_LENGTH_COMPLEX_CHILD_BYTE_ARRAY);
    Encoding encoding = encodings.get(0);
    byte[] encoderMeta = encoderMetas.get(0).array();
    ByteArrayInputStream stream = new ByteArrayInputStream(encoderMeta);
    DataInputStream in = new DataInputStream(stream);
    if (encoding == DIRECT_COMPRESS || encoding == DIRECT_COMPRESS_VARCHAR) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.setFillCompleteVector(fullVectorFill);
        metadata.readFields(in);
        DirectCompressCodec directCompressCodec = new DirectCompressCodec(metadata.getStoreDataType());
        directCompressCodec.setComplexPrimitiveIntLengthEncoding(isComplexPrimitiveIntLengthEncoding);
        return directCompressCodec.createDecoder(metadata);
    } else if (encoding == ADAPTIVE_INTEGRAL) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.setFillCompleteVector(fullVectorFill);
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveIntegralCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats, encodings.contains(Encoding.INVERTED_INDEX)).createDecoder(metadata);
    } else if (encoding == ADAPTIVE_DELTA_INTEGRAL) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.setFillCompleteVector(fullVectorFill);
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveDeltaIntegralCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats, encodings.contains(Encoding.INVERTED_INDEX)).createDecoder(metadata);
    } else if (encoding == ADAPTIVE_FLOATING) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.setFillCompleteVector(fullVectorFill);
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveFloatingCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats, encodings.contains(Encoding.INVERTED_INDEX)).createDecoder(metadata);
    } else if (encoding == ADAPTIVE_DELTA_FLOATING) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.setFillCompleteVector(fullVectorFill);
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveDeltaFloatingCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats, encodings.contains(Encoding.INVERTED_INDEX)).createDecoder(metadata);
    } else if (encoding == RLE_INTEGRAL) {
        RLEEncoderMeta metadata = new RLEEncoderMeta();
        metadata.readFields(in);
        return new RLECodec().createDecoder(metadata);
    } else if (encoding == BOOL_BYTE) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.setFillCompleteVector(fullVectorFill);
        metadata.readFields(in);
        return new DirectCompressCodec(metadata.getStoreDataType()).createDecoder(metadata);
    } else {
        // for backward compatibility
        ValueEncoderMeta metadata = CarbonUtil.deserializeEncoderMetaV3(encoderMeta);
        return createDecoderLegacy(metadata, compressor, fullVectorFill);
    }
}

Example 17 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CarbonMetadataUtilTest method setUp.

@BeforeClass
public static void setUp() {
    objMaxArr = new Long[6];
    objMaxArr[0] = new Long("111111");
    objMaxArr[1] = new Long("121111");
    objMaxArr[2] = new Long("131111");
    objMaxArr[3] = new Long("141111");
    objMaxArr[4] = new Long("151111");
    objMaxArr[5] = new Long("161111");
    objMinArr = new Long[6];
    objMinArr[0] = new Long("119");
    objMinArr[1] = new Long("121");
    objMinArr[2] = new Long("131");
    objMinArr[3] = new Long("141");
    objMinArr[4] = new Long("151");
    objMinArr[5] = new Long("161");
    objDecimal = new int[] { 0, 0, 0, 0, 0, 0 };
    columnSchemaList = new ArrayList<>();
    List<Encoding> encodingList = new ArrayList<>();
    encodingList.add(Encoding.BIT_PACKED);
    encodingList.add(Encoding.DELTA);
    encodingList.add(Encoding.INVERTED_INDEX);
    encodingList.add(Encoding.DIRECT_DICTIONARY);
    byteArr = "412111".getBytes();
    byte[] byteArr1 = "321".getBytes();
    byte[] byteArr2 = "356".getBytes();
    byteBufferList = new ArrayList<>();
    ByteBuffer bb = ByteBuffer.allocate(byteArr.length);
    bb.put(byteArr);
    ByteBuffer bb1 = ByteBuffer.allocate(byteArr1.length);
    bb1.put(byteArr1);
    ByteBuffer bb2 = ByteBuffer.allocate(byteArr2.length);
    bb2.put(byteArr2);
    byteBufferList.add(bb);
    byteBufferList.add(bb1);
    byteBufferList.add(bb2);
    DataChunk dataChunk = new DataChunk();
    dataChunk.setEncoders(encodingList);
    dataChunk.setEncoder_meta(byteBufferList);
    List<DataChunk> dataChunkList = new ArrayList<>();
    dataChunkList.add(dataChunk);
    dataChunkList.add(dataChunk);
    BlockletInfo blockletInfo = new BlockletInfo();
    blockletInfo.setColumn_data_chunks(dataChunkList);
    blockletInfoList = new ArrayList<>();
    blockletInfoList.add(blockletInfo);
    blockletInfoList.add(blockletInfo);
    ValueEncoderMeta meta = CarbonTestUtil.createValueEncoderMeta();
    meta.setDecimal(5);
    meta.setMinValue(objMinArr);
    meta.setMaxValue(objMaxArr);
    meta.setType(org.apache.carbondata.core.metadata.datatype.DataType.DOUBLE_MEASURE_CHAR);
    List<Encoding> encoders = new ArrayList<>();
    encoders.add(Encoding.INVERTED_INDEX);
    encoders.add(Encoding.BIT_PACKED);
    encoders.add(Encoding.DELTA);
    encoders.add(Encoding.DICTIONARY);
    encoders.add(Encoding.DIRECT_DICTIONARY);
    encoders.add(Encoding.RLE);
    ColumnSchema columnSchema = new ColumnSchema(DataType.INT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema1 = new ColumnSchema(DataType.ARRAY, "column", "3", true, encoders, true);
    ColumnSchema columnSchema2 = new ColumnSchema(DataType.DECIMAL, "column", "3", true, encoders, true);
    ColumnSchema columnSchema3 = new ColumnSchema(DataType.DOUBLE, "column", "3", true, encoders, true);
    ColumnSchema columnSchema4 = new ColumnSchema(DataType.LONG, "column", "3", true, encoders, true);
    ColumnSchema columnSchema5 = new ColumnSchema(DataType.SHORT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema6 = new ColumnSchema(DataType.STRUCT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema7 = new ColumnSchema(DataType.STRING, "column", "3", true, encoders, true);
    columnSchemas = new ArrayList<>();
    columnSchemas.add(columnSchema);
    columnSchemas.add(columnSchema1);
    columnSchemas.add(columnSchema2);
    columnSchemas.add(columnSchema3);
    columnSchemas.add(columnSchema4);
    columnSchemas.add(columnSchema5);
    columnSchemas.add(columnSchema6);
    columnSchemas.add(columnSchema7);
}

Example 18 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CarbonUtil method deserializeEncoderMetaV3.

public static ValueEncoderMeta deserializeEncoderMetaV3(byte[] encodeMeta) {
    ByteBuffer buffer = ByteBuffer.wrap(encodeMeta);
    char measureType = buffer.getChar();
    ValueEncoderMeta valueEncoderMeta = new ValueEncoderMeta();
    valueEncoderMeta.setType(measureType);
    switch(measureType) {
        case DataType.DOUBLE_MEASURE_CHAR:
            valueEncoderMeta.setMaxValue(buffer.getDouble());
            valueEncoderMeta.setMinValue(buffer.getDouble());
            valueEncoderMeta.setUniqueValue(buffer.getDouble());
            break;
        case DataType.BIG_DECIMAL_MEASURE_CHAR:
            valueEncoderMeta.setMaxValue(BigDecimal.valueOf(Long.MAX_VALUE));
            valueEncoderMeta.setMinValue(BigDecimal.valueOf(Long.MIN_VALUE));
            valueEncoderMeta.setUniqueValue(BigDecimal.valueOf(Long.MIN_VALUE));
            break;
        case DataType.BIG_INT_MEASURE_CHAR:
            valueEncoderMeta.setMaxValue(buffer.getLong());
            valueEncoderMeta.setMinValue(buffer.getLong());
            valueEncoderMeta.setUniqueValue(buffer.getLong());
            break;
        default:
            throw new IllegalArgumentException("invalid measure type: " + measureType);
    }
    valueEncoderMeta.setDecimal(buffer.getInt());
    valueEncoderMeta.setDataTypeSelected(buffer.get());
    return valueEncoderMeta;
}