Examples with PlainValuesReader org.apache.parquet.column.values.plain.PlainValuesReader used on opensource projects

Example 1 with PlainValuesReader

use of org.apache.parquet.column.values.plain.PlainValuesReader in project parquet-mr by apache.

the class TestDictionary method testLongDictionaryFallBack.

@Test
public void testLongDictionaryFallBack() throws IOException {
    int slabSize = 100;
    int maxDictionaryByteSize = 50;
    final FallbackValuesWriter<PlainLongDictionaryValuesWriter, PlainValuesWriter> cw = newPlainLongDictionaryValuesWriter(maxDictionaryByteSize, slabSize);
    // Fallbacked to Plain encoding, therefore use PlainValuesReader to read it back
    ValuesReader reader = new PlainValuesReader.LongPlainValuesReader();
    roundTripLong(cw, reader, maxDictionaryByteSize);
    // simulate cutting the page
    cw.reset();
    assertEquals(0, cw.getBufferedSize());
    cw.resetDictionary();
    roundTripLong(cw, reader, maxDictionaryByteSize);
}

Example 2 with PlainValuesReader

use of org.apache.parquet.column.values.plain.PlainValuesReader in project parquet-mr by apache.

the class TestDictionary method testDoubleDictionaryFallBack.

@Test
public void testDoubleDictionaryFallBack() throws IOException {
    int slabSize = 100;
    int maxDictionaryByteSize = 50;
    final FallbackValuesWriter<PlainDoubleDictionaryValuesWriter, PlainValuesWriter> cw = newPlainDoubleDictionaryValuesWriter(maxDictionaryByteSize, slabSize);
    // Fallbacked to Plain encoding, therefore use PlainValuesReader to read it back
    ValuesReader reader = new PlainValuesReader.DoublePlainValuesReader();
    roundTripDouble(cw, reader, maxDictionaryByteSize);
    // simulate cutting the page
    cw.reset();
    assertEquals(0, cw.getBufferedSize());
    cw.resetDictionary();
    roundTripDouble(cw, reader, maxDictionaryByteSize);
}

Example 3 with PlainValuesReader

use of org.apache.parquet.column.values.plain.PlainValuesReader in project parquet-mr by apache.

the class TestDictionary method testIntDictionaryFallBack.

@Test
public void testIntDictionaryFallBack() throws IOException {
    int slabSize = 100;
    int maxDictionaryByteSize = 50;
    final FallbackValuesWriter<PlainIntegerDictionaryValuesWriter, PlainValuesWriter> cw = newPlainIntegerDictionaryValuesWriter(maxDictionaryByteSize, slabSize);
    // Fallbacked to Plain encoding, therefore use PlainValuesReader to read it back
    ValuesReader reader = new PlainValuesReader.IntegerPlainValuesReader();
    roundTripInt(cw, reader, maxDictionaryByteSize);
    // simulate cutting the page
    cw.reset();
    assertEquals(0, cw.getBufferedSize());
    cw.resetDictionary();
    roundTripInt(cw, reader, maxDictionaryByteSize);
}

Example 4 with PlainValuesReader

use of org.apache.parquet.column.values.plain.PlainValuesReader in project parquet-mr by apache.

the class TestDictionary method testFloatDictionaryFallBack.

@Test
public void testFloatDictionaryFallBack() throws IOException {
    int slabSize = 100;
    int maxDictionaryByteSize = 50;
    final FallbackValuesWriter<PlainFloatDictionaryValuesWriter, PlainValuesWriter> cw = newPlainFloatDictionaryValuesWriter(maxDictionaryByteSize, slabSize);
    // Fallbacked to Plain encoding, therefore use PlainValuesReader to read it back
    ValuesReader reader = new PlainValuesReader.FloatPlainValuesReader();
    roundTripFloat(cw, reader, maxDictionaryByteSize);
    // simulate cutting the page
    cw.reset();
    assertEquals(0, cw.getBufferedSize());
    cw.resetDictionary();
    roundTripFloat(cw, reader, maxDictionaryByteSize);
}

Example 5 with PlainValuesReader

use of org.apache.parquet.column.values.plain.PlainValuesReader in project parquet-mr by apache.

the class TestDictionary method testBinaryDictionaryFallBack.

@Test
public void testBinaryDictionaryFallBack() throws IOException {
    int slabSize = 100;
    int maxDictionaryByteSize = 50;
    final ValuesWriter cw = newPlainBinaryDictionaryValuesWriter(maxDictionaryByteSize, slabSize);
    int fallBackThreshold = maxDictionaryByteSize;
    int dataSize = 0;
    for (long i = 0; i < 100; i++) {
        Binary binary = Binary.fromString("str" + i);
        cw.writeBytes(binary);
        dataSize += (binary.length() + 4);
        if (dataSize < fallBackThreshold) {
            assertEquals(PLAIN_DICTIONARY, cw.getEncoding());
        } else {
            assertEquals(PLAIN, cw.getEncoding());
        }
    }
    // Fallbacked to Plain encoding, therefore use PlainValuesReader to read it back
    ValuesReader reader = new BinaryPlainValuesReader();
    reader.initFromPage(100, cw.getBytes().toInputStream());
    for (long i = 0; i < 100; i++) {
        assertEquals(Binary.fromString("str" + i), reader.readBytes());
    }
    // simulate cutting the page
    cw.reset();
    assertEquals(0, cw.getBufferedSize());
}