Examples with PrimitivePageStatsCollector org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector used on opensource projects

Example 1 with PrimitivePageStatsCollector

use of org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector in project carbondata by apache.

the class TestEncodingFactory method testSelectProperDeltaType2.

@Test
public void testSelectProperDeltaType2() {
    PrimitivePageStatsCollector primitivePageStatsCollector = PrimitivePageStatsCollector.newInstance(DataTypes.LONG);
    // for Byte
    primitivePageStatsCollector.update((long) 200);
    ColumnPageCodec columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof AdaptiveDeltaIntegralCodec);
    assert (DataTypes.BYTE == ((AdaptiveDeltaIntegralCodec) columnPageCodec).getTargetDataType());
    // for Short
    primitivePageStatsCollector.update((long) 634767);
    columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof AdaptiveIntegralCodec);
    assert (DataTypes.SHORT_INT == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
    // for int
    primitivePageStatsCollector.update((long) (Integer.MAX_VALUE + 200));
    columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof AdaptiveIntegralCodec);
    assert (DataTypes.INT == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
    // for int
    primitivePageStatsCollector.update(Long.MAX_VALUE);
    columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof DirectCompressCodec);
    assert ("DirectCompressCodec".equals(columnPageCodec.getName()));
}

Example 2 with PrimitivePageStatsCollector

use of org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector in project carbondata by apache.

the class DefaultEncodingFactory method selectCodecByAlgorithmForFloating.

// choose between upscale adaptive encoder or upscale delta adaptive encoder,
// based on whose target data type size is smaller
static ColumnPageCodec selectCodecByAlgorithmForFloating(SimpleStatsResult stats, boolean isComplexPrimitive, TableSpec.ColumnSpec columnSpec) {
    DataType srcDataType = stats.getDataType();
    double maxValue;
    double minValue;
    if (srcDataType == DataTypes.FLOAT) {
        maxValue = (float) stats.getMax();
        minValue = (float) stats.getMin();
    } else {
        maxValue = (double) stats.getMax();
        minValue = (double) stats.getMin();
    }
    int decimalCount = stats.getDecimalCount();
    // to do that decimal count should be actual count instead of -1.
    if (isComplexPrimitive && decimalCount == -1 && stats instanceof PrimitivePageStatsCollector) {
        decimalCount = ((PrimitivePageStatsCollector) stats).getDecimalForComplexPrimitive();
    }
    // Here we should use the Max abs as max to getDatatype, let's say -1 and -10000000, -1 is max,
    // but we can't use -1 to getDatatype, we should use -10000000.
    double absMaxValue = Math.max(Math.abs(maxValue), Math.abs(minValue));
    if (srcDataType == DataTypes.FLOAT && decimalCount == 0) {
        return getColumnPageCodec(stats, isComplexPrimitive, columnSpec, srcDataType, maxValue, minValue, decimalCount, absMaxValue);
    } else if (decimalCount == 0) {
        // short, int, long
        return selectCodecByAlgorithmForIntegral(stats, false, columnSpec);
    } else if (decimalCount < 0 && !isComplexPrimitive) {
        return new DirectCompressCodec(DataTypes.DOUBLE);
    } else {
        return getColumnPageCodec(stats, isComplexPrimitive, columnSpec, srcDataType, maxValue, minValue, decimalCount, absMaxValue);
    }
}

Example 3 with PrimitivePageStatsCollector

use of org.apache.carbondata.core.datastore.page.statistics.PrimitivePageStatsCollector in project carbondata by apache.

the class TestEncodingFactory method testSelectProperDeltaType.

@Test
public void testSelectProperDeltaType() {
    PrimitivePageStatsCollector primitivePageStatsCollector = PrimitivePageStatsCollector.newInstance(DataTypes.LONG);
    // for Byte
    primitivePageStatsCollector.update((long) Byte.MAX_VALUE);
    ColumnPageCodec columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof AdaptiveIntegralCodec);
    assert (DataTypes.BYTE == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
    // for Short
    primitivePageStatsCollector.update((long) Short.MAX_VALUE);
    columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof AdaptiveIntegralCodec);
    assert (DataTypes.SHORT == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
    // for int
    primitivePageStatsCollector.update((long) Integer.MAX_VALUE);
    columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof AdaptiveIntegralCodec);
    assert (DataTypes.INT == ((AdaptiveIntegralCodec) columnPageCodec).getTargetDataType());
    // for long
    primitivePageStatsCollector.update(Long.MAX_VALUE);
    columnPageCodec = DefaultEncodingFactory.selectCodecByAlgorithmForIntegral(primitivePageStatsCollector, false, null);
    assert (columnPageCodec instanceof DirectCompressCodec);
    assert ("DirectCompressCodec".equals(columnPageCodec.getName()));
}