Examples with ValueEncoderMeta org.apache.carbondata.core.metadata.ValueEncoderMeta used on opensource projects

Example 1 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CarbonUtil method getValueCompressionModel.

/**
   * Below method will be used to get the value compression model of the
   * measure data chunk
   *
   * @return value compression model
   */
public static WriterCompressModel getValueCompressionModel(List<ValueEncoderMeta> encodeMetaList) {
    Object[] maxValue = new Object[encodeMetaList.size()];
    Object[] minValue = new Object[encodeMetaList.size()];
    Object[] uniqueValue = new Object[encodeMetaList.size()];
    int[] decimal = new int[encodeMetaList.size()];
    DataType[] type = new DataType[encodeMetaList.size()];
    byte[] dataTypeSelected = new byte[encodeMetaList.size()];
    /*
     * to fill the meta data required for value compression model
     */
    for (int i = 0; i < dataTypeSelected.length; i++) {
        // always 1
        ValueEncoderMeta valueEncoderMeta = encodeMetaList.get(i);
        maxValue[i] = valueEncoderMeta.getMaxValue();
        minValue[i] = valueEncoderMeta.getMinValue();
        uniqueValue[i] = valueEncoderMeta.getUniqueValue();
        decimal[i] = valueEncoderMeta.getDecimal();
        type[i] = valueEncoderMeta.getType();
        dataTypeSelected[i] = valueEncoderMeta.getDataTypeSelected();
    }
    MeasureMetaDataModel measureMetadataModel = new MeasureMetaDataModel(minValue, maxValue, decimal, dataTypeSelected.length, uniqueValue, type, dataTypeSelected);
    return ValueCompressionUtil.getWriterCompressModel(measureMetadataModel);
}

Example 2 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CarbonMetadataUtil method deserializeValueEncoderMeta.

private static ValueEncoderMeta deserializeValueEncoderMeta(ByteBuffer byteBuffer) throws IOException {
    ByteArrayInputStream bis = new ByteArrayInputStream(byteBuffer.array());
    ObjectInputStream objStream = new ObjectInputStream(bis);
    ValueEncoderMeta encoderMeta = null;
    try {
        encoderMeta = (ValueEncoderMeta) objStream.readObject();
    } catch (ClassNotFoundException e) {
        LOGGER.error("Error while reading ValueEncoderMeta");
    }
    return encoderMeta;
}

Example 3 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CarbonUtil method deserializeEncoderMetaNew.

public static ValueEncoderMeta deserializeEncoderMetaNew(byte[] encodeMeta) {
    ByteBuffer buffer = ByteBuffer.wrap(encodeMeta);
    char measureType = buffer.getChar();
    ValueEncoderMeta valueEncoderMeta = new ValueEncoderMeta();
    valueEncoderMeta.setType(measureType);
    switch(measureType) {
        case CarbonCommonConstants.DOUBLE_MEASURE:
            valueEncoderMeta.setMaxValue(buffer.getDouble());
            valueEncoderMeta.setMinValue(buffer.getDouble());
            valueEncoderMeta.setUniqueValue(buffer.getDouble());
            break;
        case CarbonCommonConstants.BIG_DECIMAL_MEASURE:
            valueEncoderMeta.setMaxValue(0.0);
            valueEncoderMeta.setMinValue(0.0);
            valueEncoderMeta.setUniqueValue(0.0);
            break;
        case CarbonCommonConstants.BIG_INT_MEASURE:
            valueEncoderMeta.setMaxValue(buffer.getLong());
            valueEncoderMeta.setMinValue(buffer.getLong());
            valueEncoderMeta.setUniqueValue(buffer.getLong());
            break;
        default:
            throw new IllegalArgumentException("invalid measure type");
    }
    valueEncoderMeta.setDecimal(buffer.getInt());
    valueEncoderMeta.setDataTypeSelected(buffer.get());
    return valueEncoderMeta;
}

Example 4 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CarbonMetadataUtil method createValueEncoderMeta.

private static ValueEncoderMeta createValueEncoderMeta(WriterCompressModel compressionModel, int index) {
    ValueEncoderMeta encoderMeta = new ValueEncoderMeta();
    encoderMeta.setMaxValue(compressionModel.getMaxValue()[index]);
    encoderMeta.setMinValue(compressionModel.getMinValue()[index]);
    encoderMeta.setDataTypeSelected(compressionModel.getDataTypeSelected()[index]);
    encoderMeta.setDecimal(compressionModel.getMantissa()[index]);
    encoderMeta.setType(compressionModel.getType()[index]);
    encoderMeta.setUniqueValue(compressionModel.getUniqueValue()[index]);
    return encoderMeta;
}

Example 5 with ValueEncoderMeta

use of org.apache.carbondata.core.metadata.ValueEncoderMeta in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV2 method convertToMeasureChunk.

public MeasureColumnDataChunk convertToMeasureChunk(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException {
    MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
    DataChunk2 measureColumnChunk = null;
    int copyPoint = measureRawColumnChunk.getOffSet();
    int blockIndex = measureRawColumnChunk.getBlockletId();
    ByteBuffer rawData = measureRawColumnChunk.getRawData();
    if (measureColumnChunkOffsets.size() - 1 == blockIndex) {
        measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint, measureColumnChunkLength.get(blockIndex));
        synchronized (measureRawColumnChunk.getFileReader()) {
            rawData = measureRawColumnChunk.getFileReader().readByteBuffer(filePath, measureColumnChunkOffsets.get(blockIndex) + measureColumnChunkLength.get(blockIndex), measureColumnChunk.data_page_length);
        }
    } else {
        measureColumnChunk = CarbonUtil.readDataChunk(rawData, copyPoint, measureColumnChunkLength.get(blockIndex));
        copyPoint += measureColumnChunkLength.get(blockIndex);
    }
    List<ValueEncoderMeta> valueEncodeMeta = new ArrayList<>();
    for (int i = 0; i < measureColumnChunk.getEncoder_meta().size(); i++) {
        valueEncodeMeta.add(CarbonUtil.deserializeEncoderMeta(measureColumnChunk.getEncoder_meta().get(i).array()));
    }
    WriterCompressModel compressionModel = CarbonUtil.getValueCompressionModel(valueEncodeMeta);
    ValueCompressionHolder values = compressionModel.getValueCompressionHolder()[0];
    // uncompress
    values.uncompress(compressionModel.getConvertedDataType()[0], rawData.array(), copyPoint, measureColumnChunk.data_page_length, compressionModel.getMantissa()[0], compressionModel.getMaxValue()[0], numberOfRows);
    CarbonReadDataHolder measureDataHolder = new CarbonReadDataHolder(values);
    // set the data chunk
    datChunk.setMeasureDataHolder(measureDataHolder);
    // set the enun value indexes
    datChunk.setNullValueIndexHolder(getPresenceMeta(measureColumnChunk.presence));
    return datChunk;
}