Examples with DirectCompressCodec org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec used on opensource projects

Example 16 with DirectCompressCodec

use of org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec in project carbondata by apache.

the class TestEncodingFactory method testSelectProperDeltaType.

@Test
public void testSelectProperDeltaType() {
    PrimitivePageStatsCollector primitivePageStatsCollector = PrimitivePageStatsCollector.newInstance(DataTypes.LONG);
    // for Byte
    primitivePageStatsCollector.update((long) Byte.MAX_VALUE);
    ColumnPageCodec columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof AdaptiveIntegralCodec);
    assert (DataTypes.BYTE == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
    // for Short
    primitivePageStatsCollector.update((long) Short.MAX_VALUE);
    columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof AdaptiveIntegralCodec);
    assert (DataTypes.SHORT == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
    // for int
    primitivePageStatsCollector.update((long) Integer.MAX_VALUE);
    columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof AdaptiveIntegralCodec);
    assert (DataTypes.INT == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
    // for long
    primitivePageStatsCollector.update(Long.MAX_VALUE);
    columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof DirectCompressCodec);
    assert ("DirectCompressCodec".equals(columnPageCodec.getName()));
}

Example 17 with DirectCompressCodec

use of org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec in project carbondata by apache.

the class PageLevelDictionary method getLocalDictionaryChunkForBlocklet.

/**
 * Below method will be used to get the local dictionary chunk for writing
 * @TODO Support for numeric data type dictionary exclude columns
 * @return encoded local dictionary chunk
 * @throws IOException
 * in case of problem in encoding
 */
public LocalDictionaryChunk getLocalDictionaryChunkForBlocklet() throws IOException {
    // TODO support for actual data type dictionary ColumnSPEC
    ColumnType columnType = ColumnType.PLAIN_VALUE;
    boolean isVarcharType = false;
    int lvSize = CarbonCommonConstants.SHORT_SIZE_IN_BYTE;
    if (DataTypes.VARCHAR == dataType) {
        columnType = ColumnType.PLAIN_LONG_VALUE;
        lvSize = CarbonCommonConstants.INT_SIZE_IN_BYTE;
        isVarcharType = true;
    }
    TableSpec.ColumnSpec spec = TableSpec.ColumnSpec.newInstance(columnName, DataTypes.BYTE_ARRAY, columnType);
    ColumnPage dictionaryColumnPage = ColumnPage.newPage(new ColumnPageEncoderMeta(spec, DataTypes.BYTE_ARRAY, columnCompressor), usedDictionaryValues.cardinality());
    // TODO support data type specific stats collector for numeric data types
    dictionaryColumnPage.setStatsCollector(new DummyStatsCollector());
    int rowId = 0;
    ByteBuffer byteBuffer = null;
    for (int i = usedDictionaryValues.nextSetBit(0); i >= 0; i = usedDictionaryValues.nextSetBit(i + 1)) {
        if (!isComplexTypePrimitive) {
            dictionaryColumnPage.putData(rowId++, localDictionaryGenerator.getDictionaryKeyBasedOnValue(i));
        } else {
            byte[] dictionaryKeyBasedOnValue = localDictionaryGenerator.getDictionaryKeyBasedOnValue(i);
            byteBuffer = ByteBuffer.allocate(lvSize + dictionaryKeyBasedOnValue.length);
            if (!isVarcharType) {
                byteBuffer.putShort((short) dictionaryKeyBasedOnValue.length);
            } else {
                byteBuffer.putInt(dictionaryKeyBasedOnValue.length);
            }
            byteBuffer.put(dictionaryKeyBasedOnValue);
            dictionaryColumnPage.putData(rowId++, byteBuffer.array());
        }
    }
    // creating a encoder
    ColumnPageEncoder encoder = new DirectCompressCodec(DataTypes.BYTE_ARRAY).createEncoder(null);
    // get encoded dictionary values
    LocalDictionaryChunk localDictionaryChunk = encoder.encodeDictionary(dictionaryColumnPage);
    // set compressed dictionary values
    localDictionaryChunk.setDictionary_values(CompressorFactory.getInstance().getCompressor(columnCompressor).compressByte(usedDictionaryValues.toByteArray()));
    // free the dictionary page memory
    dictionaryColumnPage.freeMemory();
    return localDictionaryChunk;
}