Examples with ColumnPage org.apache.carbondata.core.datastore.page.ColumnPage used on opensource projects

Example 1 with ColumnPage

use of org.apache.carbondata.core.datastore.page.ColumnPage in project carbondata by apache.

the class RowLevelRangeGrtThanFiterExecuterImpl method applyFilter.

@Override
public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, boolean useBitsetPipeLine) throws IOException {
    // select all rows if dimension does not exists in the current block
    if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
        int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
        return FilterUtil.createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(), numberOfRows, true);
    }
    if (isDimensionPresentInCurrentBlock[0]) {
        int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
        if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
            rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readDimensionChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
        }
        DimensionRawColumnChunk rawColumnChunk = rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
        BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
        for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
            if (rawColumnChunk.getMaxValues() != null) {
                if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues)) {
                    int compare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
                    if (compare < 0) {
                        BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
                        bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
                        bitSetGroup.setBitSet(bitSet, i);
                    } else {
                        BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]);
                        bitSetGroup.setBitSet(bitSet, i);
                    }
                }
            } else {
                BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]);
                bitSetGroup.setBitSet(bitSet, i);
            }
        }
        return bitSetGroup;
    } else if (isMeasurePresentInCurrentBlock[0]) {
        int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]);
        if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
            rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
        }
        MeasureRawColumnChunk rawColumnChunk = rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
        BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
        for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
            if (rawColumnChunk.getMaxValues() != null) {
                if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType())) {
                    int compare = comparator.compare(msrFilterRangeValues[0], DataTypeUtil.getMeasureObjectFromDataType(rawColumnChunk.getMinValues()[i], msrColEvalutorInfoList.get(0).getType()));
                    ColumnPage columnPage = rawColumnChunk.decodeColumnPage(i);
                    if (compare < 0 && columnPage.getNullBits().isEmpty()) {
                        BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
                        bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
                        bitSetGroup.setBitSet(bitSet, i);
                    } else {
                        BitSet bitSet = getFilteredIndexesForMeasures(columnPage, rawColumnChunk.getRowCount()[i]);
                        bitSetGroup.setBitSet(bitSet, i);
                    }
                }
            } else {
                BitSet bitSet = getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]);
                bitSetGroup.setBitSet(bitSet, i);
            }
        }
        return bitSetGroup;
    }
    return null;
}

Example 2 with ColumnPage

use of org.apache.carbondata.core.datastore.page.ColumnPage in project carbondata by apache.

the class RowLevelRangeGrtrThanEquaToFilterExecuterImpl method applyFilter.

@Override
public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks, boolean useBitsetPipeLine) throws IOException {
    // select all rows if dimension does not exists in the current block
    if (!isDimensionPresentInCurrentBlock[0] && !isMeasurePresentInCurrentBlock[0]) {
        int numberOfRows = rawBlockletColumnChunks.getDataBlock().numRows();
        return FilterUtil.createBitSetGroupWithDefaultValue(rawBlockletColumnChunks.getDataBlock().numberOfPages(), numberOfRows, true);
    }
    if (isDimensionPresentInCurrentBlock[0]) {
        int chunkIndex = segmentProperties.getDimensionOrdinalToChunkMapping().get(dimensionChunkIndex[0]);
        if (null == rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex]) {
            rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readDimensionChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
        }
        DimensionRawColumnChunk rawColumnChunk = rawBlockletColumnChunks.getDimensionRawColumnChunks()[chunkIndex];
        BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
        for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
            if (rawColumnChunk.getMaxValues() != null) {
                if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.filterRangeValues)) {
                    int compare = ByteUtil.UnsafeComparer.INSTANCE.compareTo(filterRangeValues[0], rawColumnChunk.getMinValues()[i]);
                    if (compare <= 0) {
                        BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
                        bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
                        bitSetGroup.setBitSet(bitSet, i);
                    } else {
                        BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]);
                        bitSetGroup.setBitSet(bitSet, i);
                    }
                }
            } else {
                BitSet bitSet = getFilteredIndexes(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]);
                bitSetGroup.setBitSet(bitSet, i);
            }
        }
        return bitSetGroup;
    } else {
        int chunkIndex = segmentProperties.getMeasuresOrdinalToChunkMapping().get(measureChunkIndex[0]);
        if (null == rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex]) {
            rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex] = rawBlockletColumnChunks.getDataBlock().readMeasureChunk(rawBlockletColumnChunks.getFileReader(), chunkIndex);
        }
        MeasureRawColumnChunk rawColumnChunk = rawBlockletColumnChunks.getMeasureRawColumnChunks()[chunkIndex];
        BitSetGroup bitSetGroup = new BitSetGroup(rawColumnChunk.getPagesCount());
        for (int i = 0; i < rawColumnChunk.getPagesCount(); i++) {
            if (rawColumnChunk.getMaxValues() != null) {
                if (isScanRequired(rawColumnChunk.getMaxValues()[i], this.msrFilterRangeValues, msrColEvalutorInfoList.get(0).getType())) {
                    int compare = comparator.compare(msrFilterRangeValues[0], DataTypeUtil.getMeasureObjectFromDataType(rawColumnChunk.getMinValues()[i], msrColEvalutorInfoList.get(0).getType()));
                    ColumnPage columnPage = rawColumnChunk.decodeColumnPage(i);
                    if (compare <= 0 && columnPage.getNullBits().isEmpty()) {
                        BitSet bitSet = new BitSet(rawColumnChunk.getRowCount()[i]);
                        bitSet.flip(0, rawColumnChunk.getRowCount()[i]);
                        bitSetGroup.setBitSet(bitSet, i);
                    } else {
                        BitSet bitSet = getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]);
                        bitSetGroup.setBitSet(bitSet, i);
                    }
                }
            } else {
                BitSet bitSet = getFilteredIndexesForMeasures(rawColumnChunk.decodeColumnPage(i), rawColumnChunk.getRowCount()[i]);
                bitSetGroup.setBitSet(bitSet, i);
            }
        }
        return bitSetGroup;
    }
}

Example 3 with ColumnPage

use of org.apache.carbondata.core.datastore.page.ColumnPage in project carbondata by apache.

the class CompressedMsrChunkFileBasedPageLevelReaderV3 method decodeColumnPage.

/**
 * Below method will be used to convert the compressed measure chunk raw data to actual data
 *
 * @param rawColumnPage measure raw chunk
 * @param pageNumber            number
 * @return DimensionColumnDataChunk
 */
@Override
public ColumnPage decodeColumnPage(MeasureRawColumnChunk rawColumnPage, int pageNumber) throws IOException, MemoryException {
    // data chunk of blocklet column
    DataChunk3 dataChunk3 = rawColumnPage.getDataChunkV3();
    // data chunk of page
    DataChunk2 pageMetadata = dataChunk3.getData_chunk_list().get(pageNumber);
    // calculating the start point of data
    // as buffer can contain multiple column data, start point will be datachunkoffset +
    // data chunk length + page offset
    long offset = rawColumnPage.getOffSet() + measureColumnChunkLength.get(rawColumnPage.getColumnIndex()) + dataChunk3.getPage_offset().get(pageNumber);
    ByteBuffer buffer = rawColumnPage.getFileReader().readByteBuffer(filePath, offset, pageMetadata.data_page_length);
    ColumnPage decodedPage = decodeMeasure(pageMetadata, buffer, 0);
    decodedPage.setNullBits(getNullBitSet(pageMetadata.presence));
    return decodedPage;
}

Example 4 with ColumnPage

use of org.apache.carbondata.core.datastore.page.ColumnPage in project carbondata by apache.

the class RLECodecSuite method testBytePageDecode.

private void testBytePageDecode(byte[] inputBytes, byte[] expectedDecodedBytes) throws IOException, MemoryException {
    RLECodec codec = new RLECodec();
    RLEEncoderMeta meta = new RLEEncoderMeta(TableSpec.ColumnSpec.newInstance("test", DataTypes.BYTE, ColumnType.MEASURE), DataTypes.BYTE, expectedDecodedBytes.length, null);
    ColumnPageDecoder decoder = codec.createDecoder(meta);
    ColumnPage page = decoder.decode(inputBytes, 0, inputBytes.length);
    byte[] decoded = page.getBytePage();
    assertEquals(expectedDecodedBytes.length, decoded.length);
    for (int i = 0; i < decoded.length; i++) {
        assertEquals(expectedDecodedBytes[i], decoded[i]);
    }
}

Example 5 with ColumnPage

use of org.apache.carbondata.core.datastore.page.ColumnPage in project carbondata by apache.

the class AdaptiveDeltaFloatingCodec method createEncoder.

@Override
public ColumnPageEncoder createEncoder(Map<String, String> parameter) {
    final Compressor compressor = CompressorFactory.getInstance().getCompressor();
    return new ColumnPageEncoder() {

        @Override
        protected byte[] encodeData(ColumnPage input) throws MemoryException, IOException {
            if (encodedPage != null) {
                throw new IllegalStateException("already encoded");
            }
            encodedPage = ColumnPage.newPage(input.getColumnSpec(), targetDataType, input.getPageSize());
            input.convertValue(converter);
            byte[] result = encodedPage.compress(compressor);
            encodedPage.freeMemory();
            return result;
        }

        @Override
        protected List<Encoding> getEncodingList() {
            List<Encoding> encodings = new ArrayList<Encoding>();
            encodings.add(Encoding.ADAPTIVE_DELTA_FLOATING);
            return encodings;
        }

        @Override
        protected ColumnPageEncoderMeta getEncoderMeta(ColumnPage inputPage) {
            return new ColumnPageEncoderMeta(inputPage.getColumnSpec(), targetDataType, stats, compressor.getName());
        }
    };
}