Examples with ValuesReader org.apache.parquet.column.values.ValuesReader used on opensource projects

Example 6 with ValuesReader

use of org.apache.parquet.column.values.ValuesReader in project parquet-mr by apache.

the class TestDeltaByteArray method testLengths.

@Test
public void testLengths() throws IOException {
    DeltaByteArrayWriter writer = new DeltaByteArrayWriter(64 * 1024, 64 * 1024, new DirectByteBufferAllocator());
    ValuesReader reader = new DeltaBinaryPackingValuesReader();
    Utils.writeData(writer, values);
    ByteBufferInputStream data = writer.getBytes().toInputStream();
    int[] bin = Utils.readInts(reader, data, values.length);
    // test prefix lengths
    Assert.assertEquals(0, bin[0]);
    Assert.assertEquals(7, bin[1]);
    Assert.assertEquals(7, bin[2]);
    reader = new DeltaBinaryPackingValuesReader();
    bin = Utils.readInts(reader, data, values.length);
    // test suffix lengths
    Assert.assertEquals(10, bin[0]);
    Assert.assertEquals(0, bin[1]);
    Assert.assertEquals(7, bin[2]);
}

Example 7 with ValuesReader

use of org.apache.parquet.column.values.ValuesReader in project parquet-mr by apache.

the class ColumnReaderImpl method readPageV1.

private void readPageV1(DataPageV1 page) {
    ValuesReader rlReader = page.getRlEncoding().getValuesReader(path, REPETITION_LEVEL);
    ValuesReader dlReader = page.getDlEncoding().getValuesReader(path, DEFINITION_LEVEL);
    this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader);
    this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader);
    try {
        BytesInput bytes = page.getBytes();
        LOG.debug("page size {} bytes and {} records", bytes.size(), pageValueCount);
        LOG.debug("reading repetition levels at 0");
        ByteBufferInputStream in = bytes.toInputStream();
        rlReader.initFromPage(pageValueCount, in);
        LOG.debug("reading definition levels at {}", in.position());
        dlReader.initFromPage(pageValueCount, in);
        LOG.debug("reading data at {}", in.position());
        initDataReader(page.getValueEncoding(), in, page.getValueCount());
    } catch (IOException e) {
        throw new ParquetDecodingException("could not read page " + page + " in col " + path, e);
    }
}

Example 8 with ValuesReader

use of org.apache.parquet.column.values.ValuesReader in project parquet-mr by apache.

the class BenchmarkReadingRandomIntegers method readingRLE.

@BenchmarkOptions(benchmarkRounds = 20, warmupRounds = 10)
@Test
public void readingRLE() throws IOException {
    for (int j = 0; j < 10; j++) {
        ValuesReader reader = new RunLengthBitPackingHybridValuesReader(32);
        readData(reader, rleBytes);
    }
}

Example 9 with ValuesReader

use of org.apache.parquet.column.values.ValuesReader in project hive by apache.

the class VectorizedPrimitiveColumnReader method readPageV1.

private void readPageV1(DataPageV1 page) {
    ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL);
    ValuesReader dlReader = page.getDlEncoding().getValuesReader(descriptor, DEFINITION_LEVEL);
    this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader);
    this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader);
    try {
        byte[] bytes = page.getBytes().toByteArray();
        LOG.debug("page size " + bytes.length + " bytes and " + pageValueCount + " records");
        LOG.debug("reading repetition levels at 0");
        rlReader.initFromPage(pageValueCount, bytes, 0);
        int next = rlReader.getNextOffset();
        LOG.debug("reading definition levels at " + next);
        dlReader.initFromPage(pageValueCount, bytes, next);
        next = dlReader.getNextOffset();
        LOG.debug("reading data at " + next);
        initDataReader(page.getValueEncoding(), bytes, next, page.getValueCount());
    } catch (IOException e) {
        throw new ParquetDecodingException("could not read page " + page + " in col " + descriptor, e);
    }
}

Example 10 with ValuesReader

use of org.apache.parquet.column.values.ValuesReader in project hive by apache.

the class BaseVectorizedColumnReader method readPageV1.

private void readPageV1(DataPageV1 page) {
    ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL);
    ValuesReader dlReader = page.getDlEncoding().getValuesReader(descriptor, DEFINITION_LEVEL);
    this.repetitionLevelColumn = new ValuesReaderIntIterator(rlReader);
    this.definitionLevelColumn = new ValuesReaderIntIterator(dlReader);
    try {
        byte[] bytes = page.getBytes().toByteArray();
        LOG.debug("page size " + bytes.length + " bytes and " + pageValueCount + " records");
        LOG.debug("reading repetition levels at 0");
        rlReader.initFromPage(pageValueCount, bytes, 0);
        int next = rlReader.getNextOffset();
        LOG.debug("reading definition levels at " + next);
        dlReader.initFromPage(pageValueCount, bytes, next);
        next = dlReader.getNextOffset();
        LOG.debug("reading data at " + next);
        initDataReader(page.getValueEncoding(), bytes, next, page.getValueCount());
    } catch (IOException e) {
        throw new ParquetDecodingException("could not read page " + page + " in col " + descriptor, e);
    }
}