Examples with FixedLengthDimensionColumnPage org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage used on opensource projects

Example 1 with FixedLengthDimensionColumnPage

use of org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage in project carbondata by apache.

the class CompressedDimensionChunkFileBasedReaderV1 method decodeColumnPage.

@Override
public DimensionColumnPage decodeColumnPage(DimensionRawColumnChunk dimensionRawColumnChunk, int pageNumber) throws IOException {
    int blockIndex = dimensionRawColumnChunk.getColumnIndex();
    byte[] dataPage = null;
    int[] invertedIndexes = null;
    int[] invertedIndexesReverse = null;
    int[] rlePage = null;
    FileReader fileReader = dimensionRawColumnChunk.getFileReader();
    ByteBuffer rawData = dimensionRawColumnChunk.getRawData();
    dataPage = COMPRESSOR.unCompressByte(rawData.array(), (int) dimensionRawColumnChunk.getOffSet(), dimensionRawColumnChunk.getLength());
    // if row id block is present then read the row id chunk and uncompress it
    DataChunk dataChunk = dimensionColumnChunk.get(blockIndex);
    if (CarbonUtil.hasEncoding(dataChunk.getEncodingList(), Encoding.INVERTED_INDEX)) {
        byte[] columnIndexData;
        synchronized (fileReader) {
            columnIndexData = fileReader.readByteArray(filePath, dataChunk.getRowIdPageOffset(), dataChunk.getRowIdPageLength());
        }
        invertedIndexes = CarbonUtil.getUnCompressColumnIndex(dataChunk.getRowIdPageLength(), columnIndexData, numberComressor, 0);
        // get the reverse index
        invertedIndexesReverse = getInvertedReverseIndex(invertedIndexes);
    }
    // then actual data based on rle block
    if (CarbonUtil.hasEncoding(dataChunk.getEncodingList(), Encoding.RLE)) {
        // read and uncompress the rle block
        byte[] key;
        synchronized (fileReader) {
            key = fileReader.readByteArray(filePath, dataChunk.getRlePageOffset(), dataChunk.getRlePageLength());
        }
        rlePage = numberComressor.unCompress(key, 0, dataChunk.getRlePageLength());
        // uncompress the data with rle indexes
        dataPage = UnBlockIndexer.uncompressData(dataPage, rlePage, eachColumnValueSize[blockIndex]);
        rlePage = null;
    }
    // fill chunk attributes
    DimensionColumnPage columnDataChunk = null;
    if (dataChunk.isRowMajor()) {
        // to store fixed length column chunk values
        columnDataChunk = new ColumnGroupDimensionColumnPage(dataPage, eachColumnValueSize[blockIndex], numberOfRows);
    } else // and set to data chunk instance
    if (!CarbonUtil.hasEncoding(dataChunk.getEncodingList(), Encoding.DICTIONARY)) {
        columnDataChunk = new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, numberOfRows);
    } else {
        // to store fixed length column chunk values
        columnDataChunk = new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, numberOfRows, eachColumnValueSize[blockIndex]);
    }
    return columnDataChunk;
}

Example 2 with FixedLengthDimensionColumnPage

use of org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage in project carbondata by apache.

the class CompressedDimensionChunkFileBasedReaderV3 method decodeDimensionLegacy.

private DimensionColumnPage decodeDimensionLegacy(DimensionRawColumnChunk rawColumnPage, ByteBuffer pageData, DataChunk2 pageMetadata, int offset) {
    byte[] dataPage;
    int[] rlePage;
    int[] invertedIndexes = null;
    int[] invertedIndexesReverse = null;
    dataPage = COMPRESSOR.unCompressByte(pageData.array(), offset, pageMetadata.data_page_length);
    offset += pageMetadata.data_page_length;
    // if row id block is present then read the row id chunk and uncompress it
    if (hasEncoding(pageMetadata.encoders, Encoding.INVERTED_INDEX)) {
        invertedIndexes = CarbonUtil.getUnCompressColumnIndex(pageMetadata.rowid_page_length, pageData, offset);
        offset += pageMetadata.rowid_page_length;
        // get the reverse index
        invertedIndexesReverse = getInvertedReverseIndex(invertedIndexes);
    }
    // then actual data based on rle block
    if (hasEncoding(pageMetadata.encoders, Encoding.RLE)) {
        rlePage = CarbonUtil.getIntArray(pageData, offset, pageMetadata.rle_page_length);
        // uncompress the data with rle indexes
        dataPage = UnBlockIndexer.uncompressData(dataPage, rlePage, eachColumnValueSize[rawColumnPage.getColumnIndex()]);
    }
    DimensionColumnPage columnDataChunk = null;
    // and set to data chunk instance
    if (!hasEncoding(pageMetadata.encoders, Encoding.DICTIONARY)) {
        columnDataChunk = new VariableLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, pageMetadata.getNumberOfRowsInpage());
    } else {
        // to store fixed length column chunk values
        columnDataChunk = new FixedLengthDimensionColumnPage(dataPage, invertedIndexes, invertedIndexesReverse, pageMetadata.getNumberOfRowsInpage(), eachColumnValueSize[rawColumnPage.getColumnIndex()]);
    }
    return columnDataChunk;
}

Example 3 with FixedLengthDimensionColumnPage

use of org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage in project carbondata by apache.

the class CarbonUtilTest method testToGetFirstIndexUsingBinarySearchWithCompareTo1.

@Test
public void testToGetFirstIndexUsingBinarySearchWithCompareTo1() {
    byte[] dataChunks = { 10, 20, 30, 40, 50, 60 };
    byte[] compareValue = { 5 };
    FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
    int result = CarbonUtil.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, false);
    assertEquals(-2, result);
}

Example 4 with FixedLengthDimensionColumnPage

use of org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage in project carbondata by apache.

the class CarbonUtilTest method testToGetFirstIndexUsingBinarySearchWithMatchUpLimitTrue.

@Test
public void testToGetFirstIndexUsingBinarySearchWithMatchUpLimitTrue() {
    byte[] dataChunks = { 10, 10, 10, 40, 50, 60 };
    byte[] compareValue = { 10 };
    FixedLengthDimensionColumnPage fixedLengthDimensionDataChunk = new FixedLengthDimensionColumnPage(dataChunks, null, null, 6, 1);
    int result = CarbonUtil.getFirstIndexUsingBinarySearch(fixedLengthDimensionDataChunk, 1, 3, compareValue, true);
    assertEquals(2, result);
}

Example 5 with FixedLengthDimensionColumnPage

use of org.apache.carbondata.core.datastore.chunk.impl.FixedLengthDimensionColumnPage in project carbondata by apache.

the class CarbonUtilTest method testToGetNextLesserValue.

@Test
public void testToGetNextLesserValue() {
    byte[] dataChunks = { 5, 6, 7, 8, 9 };
    byte[] compareValues = { 7 };
    FixedLengthDimensionColumnPage fixedLengthDataChunk = new FixedLengthDimensionColumnPage(dataChunks, null, null, 5, 1);
    int result = CarbonUtil.nextLesserValueToTarget(2, fixedLengthDataChunk, compareValues);
    assertEquals(result, 1);
}