Examples with WriterCompressModel org.apache.carbondata.core.datastore.compression.WriterCompressModel used on opensource projects

Example 6 with WriterCompressModel

use of org.apache.carbondata.core.datastore.compression.WriterCompressModel in project carbondata by apache.

the class CompressedMeasureChunkFileBasedReaderV3 method convertToMeasureChunk.

/**
   * Below method will be used to convert the compressed measure chunk raw data to actual data
   *
   * @param measureRawColumnChunk measure raw chunk
   * @param pageNumber            number
   * @return DimensionColumnDataChunk
   */
@Override
public MeasureColumnDataChunk convertToMeasureChunk(MeasureRawColumnChunk measureRawColumnChunk, int pageNumber) throws IOException {
    MeasureColumnDataChunk datChunk = new MeasureColumnDataChunk();
    // data chunk of blocklet column
    DataChunk3 dataChunk3 = measureRawColumnChunk.getDataChunkV3();
    // data chunk of page
    DataChunk2 measureColumnChunk = dataChunk3.getData_chunk_list().get(pageNumber);
    // calculating the start point of data
    // as buffer can contain multiple column data, start point will be datachunkoffset +
    // data chunk length + page offset
    int copyPoint = measureRawColumnChunk.getOffSet() + measureColumnChunkLength.get(measureRawColumnChunk.getBlockletId()) + dataChunk3.getPage_offset().get(pageNumber);
    List<ValueEncoderMeta> valueEncodeMeta = new ArrayList<>();
    for (int i = 0; i < measureColumnChunk.getEncoder_meta().size(); i++) {
        valueEncodeMeta.add(CarbonUtil.deserializeEncoderMetaNew(measureColumnChunk.getEncoder_meta().get(i).array()));
    }
    WriterCompressModel compressionModel = CarbonUtil.getValueCompressionModel(valueEncodeMeta);
    ValueCompressionHolder values = compressionModel.getValueCompressionHolder()[0];
    // uncompress
    ByteBuffer rawData = measureRawColumnChunk.getRawData();
    values.uncompress(compressionModel.getConvertedDataType()[0], rawData.array(), copyPoint, measureColumnChunk.data_page_length, compressionModel.getMantissa()[0], compressionModel.getMaxValue()[0], measureRawColumnChunk.getRowCount()[pageNumber]);
    CarbonReadDataHolder measureDataHolder = new CarbonReadDataHolder(values);
    // set the data chunk
    datChunk.setMeasureDataHolder(measureDataHolder);
    // set the null value indexes
    datChunk.setNullValueIndexHolder(getPresenceMeta(measureColumnChunk.presence));
    return datChunk;
}

Example 7 with WriterCompressModel

use of org.apache.carbondata.core.datastore.compression.WriterCompressModel in project carbondata by apache.

the class ValueCompressionUtil method getWriterCompressModel.

/**
   * Create Value compression model for write path
   */
public static WriterCompressModel getWriterCompressModel(MeasureMetaDataModel measureMDMdl) {
    int measureCount = measureMDMdl.getMeasureCount();
    Object[] minValue = measureMDMdl.getMinValue();
    Object[] maxValue = measureMDMdl.getMaxValue();
    Object[] uniqueValue = measureMDMdl.getUniqueValue();
    int[] mantissa = measureMDMdl.getMantissa();
    DataType[] type = measureMDMdl.getType();
    byte[] dataTypeSelected = measureMDMdl.getDataTypeSelected();
    WriterCompressModel compressionModel = new WriterCompressModel();
    DataType[] actualType = new DataType[measureCount];
    DataType[] convertedType = new DataType[measureCount];
    CompressionFinder[] compressionFinders = new CompressionFinder[measureCount];
    for (int i = 0; i < measureCount; i++) {
        CompressionFinder compresssionFinder = ValueCompressionUtil.getCompressionFinder(maxValue[i], minValue[i], mantissa[i], type[i], dataTypeSelected[i]);
        compressionFinders[i] = compresssionFinder;
        actualType[i] = compresssionFinder.getActualDataType();
        convertedType[i] = compresssionFinder.getConvertedDataType();
    }
    compressionModel.setCompressionFinders(compressionFinders);
    compressionModel.setMaxValue(maxValue);
    compressionModel.setMantissa(mantissa);
    compressionModel.setConvertedDataType(convertedType);
    compressionModel.setActualDataType(actualType);
    compressionModel.setMinValue(minValue);
    compressionModel.setUniqueValue(uniqueValue);
    compressionModel.setType(type);
    compressionModel.setDataTypeSelected(dataTypeSelected);
    ValueCompressionHolder[] values = ValueCompressionUtil.getValueCompressionHolder(compressionFinders);
    compressionModel.setValueCompressionHolder(values);
    return compressionModel;
}

Example 8 with WriterCompressModel

use of org.apache.carbondata.core.datastore.compression.WriterCompressModel in project carbondata by apache.

the class CarbonMetadataUtilTest method testConvertFileFooter.

@Test
public void testConvertFileFooter() throws Exception {
    int[] intArr = { 1, 2, 3, 4, 5 };
    boolean[] boolArr = { true, true, true, true, true };
    long[] longArr = { 1, 2, 3, 4, 5 };
    byte[][] maxByteArr = { { 1, 2 }, { 3, 4 }, { 5, 6 }, { 2, 4 }, { 1, 2 } };
    int[] cardinality = { 1, 2, 3, 4, 5 };
    org.apache.carbondata.core.metadata.datatype.DataType[] dataType = { org.apache.carbondata.core.metadata.datatype.DataType.INT, org.apache.carbondata.core.metadata.datatype.DataType.INT, org.apache.carbondata.core.metadata.datatype.DataType.INT, org.apache.carbondata.core.metadata.datatype.DataType.INT, org.apache.carbondata.core.metadata.datatype.DataType.INT };
    org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema colSchema = new org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema();
    org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema colSchema1 = new org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema();
    List<org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema> columnSchemaList = new ArrayList<>();
    columnSchemaList.add(colSchema);
    columnSchemaList.add(colSchema1);
    SegmentProperties segmentProperties = new SegmentProperties(columnSchemaList, cardinality);
    final List<Integer> integerList = new ArrayList<>();
    integerList.add(new Integer("1"));
    integerList.add(new Integer("2"));
    WriterCompressModel writerCompressModel = new WriterCompressModel();
    writerCompressModel.setMaxValue(objMaxArr);
    writerCompressModel.setMinValue(objMinArr);
    writerCompressModel.setDataTypeSelected(byteArr);
    writerCompressModel.setMantissa(intArr);
    writerCompressModel.setType(dataType);
    writerCompressModel.setUniqueValue(objMinArr);
    BlockletInfoColumnar blockletInfoColumnar = new BlockletInfoColumnar();
    BitSet[] bitSetArr = new BitSet[6];
    bitSetArr[0] = new BitSet();
    bitSetArr[1] = new BitSet();
    bitSetArr[2] = new BitSet();
    bitSetArr[3] = new BitSet();
    bitSetArr[4] = new BitSet();
    bitSetArr[5] = new BitSet();
    blockletInfoColumnar.setColumnMaxData(maxByteArr);
    blockletInfoColumnar.setColumnMinData(maxByteArr);
    blockletInfoColumnar.setKeyLengths(intArr);
    blockletInfoColumnar.setColGrpBlocks(boolArr);
    blockletInfoColumnar.setKeyOffSets(longArr);
    blockletInfoColumnar.setDataIndexMapOffsets(longArr);
    blockletInfoColumnar.setAggKeyBlock(boolArr);
    blockletInfoColumnar.setDataIndexMapLength(intArr);
    blockletInfoColumnar.setIsSortedKeyColumn(boolArr);
    blockletInfoColumnar.setKeyOffSets(longArr);
    blockletInfoColumnar.setMeasureLength(intArr);
    blockletInfoColumnar.setMeasureOffset(longArr);
    blockletInfoColumnar.setMeasureNullValueIndex(bitSetArr);
    blockletInfoColumnar.setCompressionModel(writerCompressModel);
    BlockletInfoColumnar blockletInfoColumnar1 = new BlockletInfoColumnar();
    blockletInfoColumnar1.setColumnMaxData(maxByteArr);
    blockletInfoColumnar1.setColumnMinData(maxByteArr);
    blockletInfoColumnar1.setKeyLengths(intArr);
    blockletInfoColumnar1.setKeyOffSets(longArr);
    blockletInfoColumnar1.setDataIndexMapOffsets(longArr);
    blockletInfoColumnar1.setAggKeyBlock(boolArr);
    blockletInfoColumnar1.setDataIndexMapLength(intArr);
    blockletInfoColumnar1.setIsSortedKeyColumn(boolArr);
    blockletInfoColumnar1.setColGrpBlocks(boolArr);
    blockletInfoColumnar1.setKeyOffSets(longArr);
    blockletInfoColumnar1.setMeasureLength(intArr);
    blockletInfoColumnar1.setMeasureOffset(longArr);
    blockletInfoColumnar1.setMeasureNullValueIndex(bitSetArr);
    blockletInfoColumnar1.setCompressionModel(writerCompressModel);
    blockletInfoColumnar1.setColGrpBlocks(boolArr);
    List<BlockletInfoColumnar> blockletInfoColumnarList = new ArrayList<>();
    blockletInfoColumnarList.add(blockletInfoColumnar);
    blockletInfoColumnarList.add(blockletInfoColumnar1);
    new MockUp<CarbonUtil>() {

        @SuppressWarnings("unused")
        @Mock
        public List<Integer> convertToIntegerList(int[] array) {
            return integerList;
        }
    };
    final Set<Integer> integerSet = new HashSet<>();
    integerSet.add(new Integer("1"));
    integerSet.add(new Integer("2"));
    new MockUp<SegmentProperties>() {

        @SuppressWarnings("unused")
        @Mock
        public Set<Integer> getDimensionOrdinalForBlock(int blockIndex) {
            return integerSet;
        }
    };
    SegmentInfo segmentInfo = new SegmentInfo();
    segmentInfo.setNum_cols(4);
    segmentInfo.setColumn_cardinalities(integerList);
    FileFooter fileFooter = new FileFooter();
    fileFooter.setNum_rows(4);
    fileFooter.setSegment_info(segmentInfo);
    byte[] byteMaxArr = "1".getBytes();
    byte[] byteMinArr = "2".getBytes();
    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
    blockletMinMaxIndex.addToMax_values(ByteBuffer.wrap(byteMaxArr));
    blockletMinMaxIndex.addToMin_values(ByteBuffer.wrap(byteMinArr));
    FileFooter result = convertFileFooter(blockletInfoColumnarList, 4, cardinality, columnSchemas, segmentProperties);
    assertEquals(result.getTable_columns(), columnSchemas);
}

Example 9 with WriterCompressModel

use of org.apache.carbondata.core.datastore.compression.WriterCompressModel in project carbondata by apache.

the class CarbonUtilTest method testToGetValueCompressionModel.

@Test
public void testToGetValueCompressionModel() {
    List<DataChunk> dataChunkList = new ArrayList<>();
    DataChunk dataChunk = new DataChunk();
    List<Encoding> encodingList = new ArrayList<>();
    encodingList.add(Encoding.DELTA);
    dataChunk.setEncodingList(encodingList);
    List<ValueEncoderMeta> valueEncoderMetas = new ArrayList<>();
    ValueEncoderMeta valueEncoderMeta = new ValueEncoderMeta();
    valueEncoderMeta.setMaxValue(5.0);
    valueEncoderMeta.setMinValue(1.0);
    valueEncoderMeta.setUniqueValue(2.0);
    valueEncoderMeta.setType('n');
    valueEncoderMeta.setDataTypeSelected((byte) 'v');
    valueEncoderMetas.add(valueEncoderMeta);
    dataChunk.setValueEncoderMeta(valueEncoderMetas);
    dataChunkList.add(dataChunk);
    WriterCompressModel writerCompressModel = CarbonUtil.getValueCompressionModel(dataChunkList.get(0).getValueEncoderMeta());
    assertEquals(1, writerCompressModel.getMaxValue().length);
}

Example 10 with WriterCompressModel

use of org.apache.carbondata.core.datastore.compression.WriterCompressModel in project carbondata by apache.

the class ValueCompressionUtilTest method testToGetValueCompressionModelForByteAndIntAndDataTypeSelected0.

@Test
public void testToGetValueCompressionModelForByteAndIntAndDataTypeSelected0() {
    Object[] maxValues = { -32766.00 };
    Object[] minValues = { 32744.0 };
    int[] decimalLength = { 1 };
    Object[] uniqueValues = { 5 };
    DataType[] types = { DataType.DOUBLE };
    byte[] dataTypeSelected = { 0 };
    MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected);
    WriterCompressModel writerCompressModel = ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
    assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.BIGINT, writerCompressModel.getCompType(0));
}