Examples with PlainValuesWriter org.apache.parquet.column.values.plain.PlainValuesWriter used on opensource projects

Example 1 with PlainValuesWriter

use of org.apache.parquet.column.values.plain.PlainValuesWriter in project parquet-mr by apache.

the class TestDictionary method testLongDictionaryFallBack.

@Test
public void testLongDictionaryFallBack() throws IOException {
    int slabSize = 100;
    int maxDictionaryByteSize = 50;
    final FallbackValuesWriter<PlainLongDictionaryValuesWriter, PlainValuesWriter> cw = newPlainLongDictionaryValuesWriter(maxDictionaryByteSize, slabSize);
    // Fallbacked to Plain encoding, therefore use PlainValuesReader to read it back
    ValuesReader reader = new PlainValuesReader.LongPlainValuesReader();
    roundTripLong(cw, reader, maxDictionaryByteSize);
    // simulate cutting the page
    cw.reset();
    assertEquals(0, cw.getBufferedSize());
    cw.resetDictionary();
    roundTripLong(cw, reader, maxDictionaryByteSize);
}

Example 2 with PlainValuesWriter

use of org.apache.parquet.column.values.plain.PlainValuesWriter in project parquet-mr by apache.

the class TestDictionary method testFloatDictionary.

@Test
public void testFloatDictionary() throws IOException {
    int COUNT = 2000;
    int COUNT2 = 4000;
    final FallbackValuesWriter<PlainFloatDictionaryValuesWriter, PlainValuesWriter> cw = newPlainFloatDictionaryValuesWriter(10000, 10000);
    for (float i = 0; i < COUNT; i++) {
        cw.writeFloat(i % 50);
    }
    BytesInput bytes1 = getBytesAndCheckEncoding(cw, PLAIN_DICTIONARY);
    assertEquals(50, cw.initialWriter.getDictionarySize());
    for (float i = COUNT2; i > 0; i--) {
        cw.writeFloat(i % 50);
    }
    BytesInput bytes2 = getBytesAndCheckEncoding(cw, PLAIN_DICTIONARY);
    assertEquals(50, cw.initialWriter.getDictionarySize());
    DictionaryValuesReader cr = initDicReader(cw, FLOAT);
    cr.initFromPage(COUNT, bytes1.toInputStream());
    for (float i = 0; i < COUNT; i++) {
        float back = cr.readFloat();
        assertEquals(i % 50, back, 0.0f);
    }
    cr.initFromPage(COUNT2, bytes2.toInputStream());
    for (float i = COUNT2; i > 0; i--) {
        float back = cr.readFloat();
        assertEquals(i % 50, back, 0.0f);
    }
}

Example 3 with PlainValuesWriter

use of org.apache.parquet.column.values.plain.PlainValuesWriter in project parquet-mr by apache.

the class TestDictionary method testDoubleDictionaryFallBack.

@Test
public void testDoubleDictionaryFallBack() throws IOException {
    int slabSize = 100;
    int maxDictionaryByteSize = 50;
    final FallbackValuesWriter<PlainDoubleDictionaryValuesWriter, PlainValuesWriter> cw = newPlainDoubleDictionaryValuesWriter(maxDictionaryByteSize, slabSize);
    // Fallbacked to Plain encoding, therefore use PlainValuesReader to read it back
    ValuesReader reader = new PlainValuesReader.DoublePlainValuesReader();
    roundTripDouble(cw, reader, maxDictionaryByteSize);
    // simulate cutting the page
    cw.reset();
    assertEquals(0, cw.getBufferedSize());
    cw.resetDictionary();
    roundTripDouble(cw, reader, maxDictionaryByteSize);
}

Example 4 with PlainValuesWriter

use of org.apache.parquet.column.values.plain.PlainValuesWriter in project parquet-mr by apache.

the class TestDictionary method testIntDictionaryFallBack.

@Test
public void testIntDictionaryFallBack() throws IOException {
    int slabSize = 100;
    int maxDictionaryByteSize = 50;
    final FallbackValuesWriter<PlainIntegerDictionaryValuesWriter, PlainValuesWriter> cw = newPlainIntegerDictionaryValuesWriter(maxDictionaryByteSize, slabSize);
    // Fallbacked to Plain encoding, therefore use PlainValuesReader to read it back
    ValuesReader reader = new PlainValuesReader.IntegerPlainValuesReader();
    roundTripInt(cw, reader, maxDictionaryByteSize);
    // simulate cutting the page
    cw.reset();
    assertEquals(0, cw.getBufferedSize());
    cw.resetDictionary();
    roundTripInt(cw, reader, maxDictionaryByteSize);
}

Example 5 with PlainValuesWriter

use of org.apache.parquet.column.values.plain.PlainValuesWriter in project parquet-mr by apache.

the class TestDictionary method testIntDictionary.

@Test
public void testIntDictionary() throws IOException {
    int COUNT = 2000;
    int COUNT2 = 4000;
    final FallbackValuesWriter<PlainIntegerDictionaryValuesWriter, PlainValuesWriter> cw = newPlainIntegerDictionaryValuesWriter(10000, 10000);
    for (int i = 0; i < COUNT; i++) {
        cw.writeInteger(i % 50);
    }
    BytesInput bytes1 = getBytesAndCheckEncoding(cw, PLAIN_DICTIONARY);
    assertEquals(50, cw.initialWriter.getDictionarySize());
    for (int i = COUNT2; i > 0; i--) {
        cw.writeInteger(i % 50);
    }
    BytesInput bytes2 = getBytesAndCheckEncoding(cw, PLAIN_DICTIONARY);
    assertEquals(50, cw.initialWriter.getDictionarySize());
    DictionaryValuesReader cr = initDicReader(cw, INT32);
    cr.initFromPage(COUNT, bytes1.toInputStream());
    for (int i = 0; i < COUNT; i++) {
        int back = cr.readInteger();
        assertEquals(i % 50, back);
    }
    cr.initFromPage(COUNT2, bytes2.toInputStream());
    for (int i = COUNT2; i > 0; i--) {
        int back = cr.readInteger();
        assertEquals(i % 50, back);
    }
}