Examples with ColumnPageEncoder org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder used on opensource projects

Example 1 with ColumnPageEncoder

use of org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder in project carbondata by apache.

the class AdaptiveDeltaFloatingCodec method createEncoder.

@Override
public ColumnPageEncoder createEncoder(Map<String, String> parameter) {
    final Compressor compressor = CompressorFactory.getInstance().getCompressor();
    return new ColumnPageEncoder() {

        @Override
        protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException {
            if (encodedPage != null) {
                throw new IllegalStateException("already encoded");
            }
            encodedPage = ColumnPage.newPage(input.getColumnSpec(), targetDataType, input.getPageSize());
            input.convertValue(converter);
            byte[] result = encodedPage.compress(compressor);
            encodedPage.freeMemory();
            return result;
        }

        @Override
        protected List<Encoding> getEncodingList() {
            List<Encoding> encodings = new ArrayList<Encoding>();
            encodings.add(Encoding.ADAPTIVE_DELTA_FLOATING);
            return encodings;
        }

        @Override
        protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) {
            return new ColumnPageEncoderMeta(inputPage.getColumnSpec(), targetDataType, stats, compressor.getName());
        }
    };
}

Example 2 with ColumnPageEncoder

use of org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder in project carbondata by apache.

the class AdaptiveIntegralCodec method createEncoder.

@Override
public ColumnPageEncoder createEncoder(Map<String, String> parameter) {
    final Compressor compressor = CompressorFactory.getInstance().getCompressor();
    return new ColumnPageEncoder() {

        @Override
        protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException {
            if (encodedPage != null) {
                throw new IllegalStateException("already encoded");
            }
            encodedPage = ColumnPage.newPage(input.getColumnSpec(), targetDataType, input.getPageSize());
            input.convertValue(converter);
            byte[] result = encodedPage.compress(compressor);
            encodedPage.freeMemory();
            return result;
        }

        @Override
        protected List<Encoding> getEncodingList() {
            List<Encoding> encodings = new ArrayList<Encoding>();
            encodings.add(Encoding.ADAPTIVE_INTEGRAL);
            return encodings;
        }

        @Override
        protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) {
            return new ColumnPageEncoderMeta(inputPage.getColumnSpec(), targetDataType, stats, compressor.getName());
        }
    };
}

Example 3 with ColumnPageEncoder

use of org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder in project carbondata by apache.

the class TablePage method encodeAndCompressDimensions.

// apply and compress each dimension, set encoded data in `encodedData`
private EncodedColumnPage[] encodeAndCompressDimensions() throws KeyGenException, IOException, MemoryException {
    List<EncodedColumnPage> encodedDimensions = new ArrayList<>();
    List<EncodedColumnPage> encodedComplexDimenions = new ArrayList<>();
    TableSpec tableSpec = model.getTableSpec();
    int dictIndex = 0;
    int noDictIndex = 0;
    int complexDimIndex = 0;
    int numDimensions = tableSpec.getNumDimensions();
    for (int i = 0; i < numDimensions; i++) {
        ColumnPageEncoder columnPageEncoder;
        EncodedColumnPage encodedPage;
        TableSpec.DimensionSpec spec = tableSpec.getDimensionSpec(i);
        switch(spec.getColumnType()) {
            case GLOBAL_DICTIONARY:
            case DIRECT_DICTIONARY:
                columnPageEncoder = encodingFactory.createEncoder(spec, dictDimensionPages[dictIndex]);
                encodedPage = columnPageEncoder.encode(dictDimensionPages[dictIndex++]);
                encodedDimensions.add(encodedPage);
                break;
            case PLAIN_VALUE:
                columnPageEncoder = encodingFactory.createEncoder(spec, noDictDimensionPages[noDictIndex]);
                encodedPage = columnPageEncoder.encode(noDictDimensionPages[noDictIndex++]);
                encodedDimensions.add(encodedPage);
                break;
            case COMPLEX:
                EncodedColumnPage[] encodedPages = ColumnPageEncoder.encodeComplexColumn(complexDimensionPages[complexDimIndex++]);
                encodedComplexDimenions.addAll(Arrays.asList(encodedPages));
                break;
            default:
                throw new IllegalArgumentException("unsupported dimension type:" + spec.getColumnType());
        }
    }
    encodedDimensions.addAll(encodedComplexDimenions);
    return encodedDimensions.toArray(new EncodedColumnPage[encodedDimensions.size()]);
}

Example 4 with ColumnPageEncoder

use of org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder in project carbondata by apache.

the class TablePage method encodeAndCompressMeasures.

// apply measure and set encodedData in `encodedData`
private EncodedColumnPage[] encodeAndCompressMeasures() throws MemoryException, IOException {
    EncodedColumnPage[] encodedMeasures = new EncodedColumnPage[measurePages.length];
    for (int i = 0; i < measurePages.length; i++) {
        ColumnPageEncoder encoder = encodingFactory.createEncoder(model.getTableSpec().getMeasureSpec(i), measurePages[i]);
        encodedMeasures[i] = encoder.encode(measurePages[i]);
    }
    return encodedMeasures;
}

Example 5 with ColumnPageEncoder

use of org.apache.carbondata.core.datastore.page.encoding.ColumnPageEncoder in project carbondata by apache.

the class AdaptiveFloatingCodec method createEncoder.

@Override
public ColumnPageEncoder createEncoder(Map<String, String> parameter) {
    final Compressor compressor = CompressorFactory.getInstance().getCompressor();
    return new ColumnPageEncoder() {

        @Override
        protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException {
            if (encodedPage != null) {
                throw new IllegalStateException("already encoded");
            }
            encodedPage = ColumnPage.newPage(input.getColumnSpec(), targetDataType, input.getPageSize());
            input.convertValue(converter);
            byte[] result = encodedPage.compress(compressor);
            encodedPage.freeMemory();
            return result;
        }

        @Override
        protected List<Encoding> getEncodingList() {
            List<Encoding> encodings = new ArrayList<Encoding>();
            encodings.add(Encoding.ADAPTIVE_FLOATING);
            return encodings;
        }

        @Override
        protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) {
            return new ColumnPageEncoderMeta(inputPage.getColumnSpec(), targetDataType, stats, compressor.getName());
        }
    };
}