Examples with DirectCompressCodec org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec used on opensource projects

Example 6 with DirectCompressCodec

use of org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec in project carbondata by apache.

the class EncodingFactory method createDecoderLegacy.

/**
 * Old way of creating decoder, based on algorithm
 */
private ColumnPageDecoder createDecoderLegacy(ValueEncoderMeta metadata, String compressor, boolean fullVectorFill) {
    if (null == metadata) {
        throw new RuntimeException("internal error");
    }
    SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
    TableSpec.ColumnSpec spec = TableSpec.ColumnSpec.newInstanceLegacy("legacy", stats.getDataType(), ColumnType.MEASURE);
    DataType dataType = DataType.getDataType(metadata.getType());
    if (dataType == DataTypes.BYTE || dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) {
        // create the codec based on algorithm and create decoder by recovering the metadata
        ColumnPageCodec codec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(stats, false, spec);
        if (codec instanceof AdaptiveIntegralCodec) {
            AdaptiveIntegralCodec adaptiveCodec = (AdaptiveIntegralCodec) codec;
            ColumnPageEncoderMeta meta = new ColumnPageEncoderMeta(spec, adaptiveCodec.getTargetDataType(), stats, compressor);
            meta.setFillCompleteVector(fullVectorFill);
            return codec.createDecoder(meta);
        } else if (codec instanceof AdaptiveDeltaIntegralCodec) {
            AdaptiveDeltaIntegralCodec adaptiveCodec = (AdaptiveDeltaIntegralCodec) codec;
            ColumnPageEncoderMeta meta = new ColumnPageEncoderMeta(spec, adaptiveCodec.getTargetDataType(), stats, compressor);
            meta.setFillCompleteVector(fullVectorFill);
            return codec.createDecoder(meta);
        } else if (codec instanceof DirectCompressCodec) {
            ColumnPageEncoderMeta meta = new ColumnPageEncoderMeta(spec, DataType.getDataType(metadata.getType()), stats, compressor);
            meta.setFillCompleteVector(fullVectorFill);
            return codec.createDecoder(meta);
        } else {
            throw new RuntimeException("internal error");
        }
    } else if (dataType == DataTypes.FLOAT || dataType == DataTypes.DOUBLE) {
        // create the codec based on algorithm and create decoder by recovering the metadata
        ColumnPageCodec codec = DefaultEncodingFactory.selectCodecByAlgorithmForFloating(stats, false, spec);
        if (codec instanceof AdaptiveFloatingCodec) {
            AdaptiveFloatingCodec adaptiveCodec = (AdaptiveFloatingCodec) codec;
            ColumnPageEncoderMeta meta = new ColumnPageEncoderMeta(spec, adaptiveCodec.getTargetDataType(), stats, compressor);
            meta.setFillCompleteVector(fullVectorFill);
            return codec.createDecoder(meta);
        } else if (codec instanceof DirectCompressCodec) {
            ColumnPageEncoderMeta meta = new ColumnPageEncoderMeta(spec, DataType.getDataType(metadata.getType()), stats, compressor);
            meta.setFillCompleteVector(fullVectorFill);
            return codec.createDecoder(meta);
        } else if (codec instanceof AdaptiveDeltaFloatingCodec) {
            AdaptiveDeltaFloatingCodec adaptiveCodec = (AdaptiveDeltaFloatingCodec) codec;
            ColumnPageEncoderMeta meta = new ColumnPageEncoderMeta(spec, adaptiveCodec.getTargetDataType(), stats, compressor);
            meta.setFillCompleteVector(fullVectorFill);
            return codec.createDecoder(meta);
        } else {
            throw new RuntimeException("internal error");
        }
    } else if (DataTypes.isDecimal(dataType) || dataType == DataTypes.BYTE_ARRAY) {
        // no dictionary dimension
        ColumnPageEncoderMeta meta = new ColumnPageEncoderMeta(spec, stats.getDataType(), stats, compressor);
        meta.setFillCompleteVector(fullVectorFill);
        return new DirectCompressCodec(stats.getDataType()).createDecoder(meta);
    } else {
        throw new RuntimeException("unsupported data type: " + stats.getDataType());
    }
}

Example 7 with DirectCompressCodec

use of org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec in project carbondata by apache.

the class ColumnPageEncoder method encodeChildColumn.

private static EncodedColumnPage encodeChildColumn(byte[][] data) throws IOException, MemoryException {
    TableSpec.ColumnSpec spec = TableSpec.ColumnSpec.newInstance("complex_inner_column", DataTypes.BYTE_ARRAY, ColumnType.COMPLEX);
    ColumnPage page = ColumnPage.wrapByteArrayPage(spec, data);
    ColumnPageEncoder encoder = new DirectCompressCodec(DataTypes.BYTE_ARRAY).createEncoder(null);
    return encoder.encode(page);
}

Example 8 with DirectCompressCodec

use of org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec in project carbondata by apache.

the class DefaultEncodingFactory method selectCodecByAlgorithmForIntegral.

/**
 * choose between adaptive encoder or delta adaptive encoder, based on whose target data type
 * size is smaller
 */
static ColumnPageCodec selectCodecByAlgorithmForIntegral(SimpleStatsResult stats) {
    DataType srcDataType = stats.getDataType();
    DataType adaptiveDataType = fitMinMax(stats.getDataType(), stats.getMax(), stats.getMin());
    DataType deltaDataType;
    if (adaptiveDataType == DataTypes.LONG) {
        deltaDataType = DataTypes.LONG;
    } else {
        deltaDataType = fitDelta(stats.getDataType(), stats.getMax(), stats.getMin());
    }
    // that get size in bytes
    if (Math.min(adaptiveDataType.getSizeInBytes(), deltaDataType.getSizeInBytes()) == srcDataType.getSizeInBytes()) {
        // no effect to use adaptive or delta, use compression only
        return new DirectCompressCodec(stats.getDataType());
    }
    if (adaptiveDataType.getSizeInBytes() <= deltaDataType.getSizeInBytes()) {
        // choose adaptive encoding
        return new AdaptiveIntegralCodec(stats.getDataType(), adaptiveDataType, stats);
    } else {
        // choose delta adaptive encoding
        return new AdaptiveDeltaIntegralCodec(stats.getDataType(), deltaDataType, stats);
    }
}

Example 9 with DirectCompressCodec

use of org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec in project carbondata by apache.

the class EncodingFactory method createDecoder.

/**
 * Return new decoder based on encoder metadata read from file
 */
public ColumnPageDecoder createDecoder(List<Encoding> encodings, List<ByteBuffer> encoderMetas) throws IOException {
    assert (encodings.size() == 1);
    assert (encoderMetas.size() == 1);
    Encoding encoding = encodings.get(0);
    byte[] encoderMeta = encoderMetas.get(0).array();
    ByteArrayInputStream stream = new ByteArrayInputStream(encoderMeta);
    DataInputStream in = new DataInputStream(stream);
    if (encoding == DIRECT_COMPRESS) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        return new DirectCompressCodec(metadata.getStoreDataType()).createDecoder(metadata);
    } else if (encoding == ADAPTIVE_INTEGRAL) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveIntegralCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats).createDecoder(metadata);
    } else if (encoding == ADAPTIVE_DELTA_INTEGRAL) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveDeltaIntegralCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats).createDecoder(metadata);
    } else if (encoding == ADAPTIVE_FLOATING) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveFloatingCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats).createDecoder(metadata);
    } else if (encoding == ADAPTIVE_DELTA_FLOATING) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        SimpleStatsResult stats = PrimitivePageStatsCollector.newInstance(metadata);
        return new AdaptiveDeltaFloatingCodec(metadata.getSchemaDataType(), metadata.getStoreDataType(), stats).createDecoder(metadata);
    } else if (encoding == RLE_INTEGRAL) {
        RLEEncoderMeta metadata = new RLEEncoderMeta();
        metadata.readFields(in);
        return new RLECodec().createDecoder(metadata);
    } else if (encoding == BOOL_BYTE) {
        ColumnPageEncoderMeta metadata = new ColumnPageEncoderMeta();
        metadata.readFields(in);
        return new DirectCompressCodec(metadata.getStoreDataType()).createDecoder(metadata);
    } else {
        // for backward compatibility
        ValueEncoderMeta metadata = CarbonUtil.deserializeEncoderMetaV3(encoderMeta);
        return createDecoderLegacy(metadata);
    }
}

Example 10 with DirectCompressCodec

use of org.apache.carbondata.core.datastore.page.encoding.compress.DirectCompressCodec in project carbondata by apache.

the class DefaultEncodingFactory method getColumnPageCodec.

private static ColumnPageCodec getColumnPageCodec(SimpleStatsResult stats, boolean isComplexPrimitive, TableSpec.ColumnSpec columnSpec, DataType srcDataType, double maxValue, double minValue, int decimalCount, double absMaxValue) {
    // If absMaxValue exceeds LONG.MAX_VALUE, then go for direct compression
    if ((Math.pow(10, decimalCount) * absMaxValue) > Long.MAX_VALUE) {
        return new DirectCompressCodec(DataTypes.DOUBLE);
    } else {
        long max = (long) (Math.pow(10, decimalCount) * absMaxValue);
        DataType adaptiveDataType = fitLongMinMax(max, 0);
        DataType deltaDataType = compareMinMaxAndSelectDataType((long) (Math.pow(10, decimalCount) * (maxValue - minValue)));
        if (adaptiveDataType.getSizeInBytes() > deltaDataType.getSizeInBytes()) {
            return new AdaptiveDeltaFloatingCodec(srcDataType, deltaDataType, stats, isInvertedIndex(isComplexPrimitive, columnSpec));
        } else if (adaptiveDataType.getSizeInBytes() < DataTypes.DOUBLE.getSizeInBytes() || ((isComplexPrimitive) && (adaptiveDataType.getSizeInBytes() == DataTypes.DOUBLE.getSizeInBytes()))) {
            return new AdaptiveFloatingCodec(srcDataType, adaptiveDataType, stats, isInvertedIndex(isComplexPrimitive, columnSpec));
        } else {
            return new DirectCompressCodec(DataTypes.DOUBLE);
        }
    }
}