Examples with StreamLayoutFactory com.facebook.presto.orc.writer.StreamLayoutFactory used on opensource projects

Example 1 with StreamLayoutFactory

use of com.facebook.presto.orc.writer.StreamLayoutFactory in project presto by prestodb.

the class TestOrcWriterOptions method testToString.

@Test
public void testToString() {
    DataSize stripeMinSize = new DataSize(13, MEGABYTE);
    DataSize stripeMaxSize = new DataSize(27, MEGABYTE);
    int stripeMaxRowCount = 1_100_000;
    int rowGroupMaxRowCount = 15_000;
    DataSize dictionaryMaxMemory = new DataSize(13_000, KILOBYTE);
    DataSize dictionaryMemoryRange = new DataSize(1_000, KILOBYTE);
    int dictionaryUsefulCheckPerChunkFrequency = 9_999;
    DataSize dictionaryUsefulCheckColumnSize = new DataSize(1, MEGABYTE);
    DataSize stringMaxStatisticsLimit = new DataSize(128, BYTE);
    DataSize maxCompressionBufferSize = new DataSize(512, KILOBYTE);
    DataSize dwrfStripeCacheMaxSize = new DataSize(4, MEGABYTE);
    DwrfStripeCacheMode dwrfStripeCacheMode = DwrfStripeCacheMode.INDEX_AND_FOOTER;
    OptionalInt compressionLevel = OptionalInt.of(5);
    StreamLayoutFactory streamLayoutFactory = new ColumnSizeLayoutFactory();
    boolean integerDictionaryEncodingEnabled = false;
    boolean stringDictionarySortingEnabled = true;
    int preserveDirectEncodingStripeCount = 0;
    OrcWriterOptions writerOptions = OrcWriterOptions.builder().withFlushPolicy(DefaultOrcWriterFlushPolicy.builder().withStripeMinSize(stripeMinSize).withStripeMaxSize(stripeMaxSize).withStripeMaxRowCount(stripeMaxRowCount).build()).withRowGroupMaxRowCount(rowGroupMaxRowCount).withDictionaryMaxMemory(dictionaryMaxMemory).withDictionaryMemoryAlmostFullRange(dictionaryMemoryRange).withDictionaryUsefulCheckPerChunkFrequency(dictionaryUsefulCheckPerChunkFrequency).withDictionaryUsefulCheckColumnSize(dictionaryUsefulCheckColumnSize).withMaxStringStatisticsLimit(stringMaxStatisticsLimit).withMaxCompressionBufferSize(maxCompressionBufferSize).withCompressionLevel(compressionLevel).withStreamLayoutFactory(streamLayoutFactory).withIntegerDictionaryEncodingEnabled(integerDictionaryEncodingEnabled).withStringDictionarySortingEnabled(stringDictionarySortingEnabled).withDwrfStripeCacheEnabled(true).withDwrfStripeCacheMaxSize(dwrfStripeCacheMaxSize).withDwrfStripeCacheMode(dwrfStripeCacheMode).withPreserveDirectEncodingStripeCount(preserveDirectEncodingStripeCount).build();
    String expectedString = "OrcWriterOptions{flushPolicy=DefaultOrcWriterFlushPolicy{stripeMaxRowCount=1100000, " + "stripeMinBytes=13631488, stripeMaxBytes=28311552}, rowGroupMaxRowCount=15000, " + "dictionaryMaxMemory=13000kB, dictionaryMemoryAlmostFullRange=1000kB, dictionaryUsefulCheckPerChunkFrequency=9999, " + "dictionaryUsefulCheckColumnSize=1MB, maxStringStatisticsLimit=128B, maxCompressionBufferSize=512kB, " + "compressionLevel=OptionalInt[5], streamLayoutFactory=ColumnSizeLayoutFactory{}, integerDictionaryEncodingEnabled=false, " + "stringDictionarySortingEnabled=true, stringDictionaryEncodingEnabled=true, " + "dwrfWriterOptions=Optional[DwrfStripeCacheOptions{stripeCacheMode=INDEX_AND_FOOTER, stripeCacheMaxSize=4MB}], " + "ignoreDictionaryRowGroupSizes=false, preserveDirectEncodingStripeCount=0}";
    assertEquals(expectedString, writerOptions.toString());
}

Example 2 with StreamLayoutFactory

use of com.facebook.presto.orc.writer.StreamLayoutFactory in project presto by prestodb.

the class TestOrcWriterOptions method testProperties.

@Test
public void testProperties() {
    DataSize stripeMinSize = new DataSize(13, MEGABYTE);
    DataSize stripeMaxSize = new DataSize(27, MEGABYTE);
    int stripeMaxRowCount = 1_100_000;
    int rowGroupMaxRowCount = 15_000;
    DataSize dictionaryMaxMemory = new DataSize(13_000, KILOBYTE);
    DataSize dictionaryMemoryRange = new DataSize(1_000, KILOBYTE);
    int dictionaryUsefulCheckPerChunkFrequency = 9_999;
    DataSize dictionaryUsefulCheckColumnSize = new DataSize(1, MEGABYTE);
    DataSize stringMaxStatisticsLimit = new DataSize(128, BYTE);
    DataSize maxCompressionBufferSize = new DataSize(512, KILOBYTE);
    OptionalInt compressionLevel = OptionalInt.of(5);
    StreamLayoutFactory streamLayoutFactory = new StreamLayoutFactory.StreamSizeLayoutFactory();
    boolean integerDictionaryEncodingEnabled = true;
    boolean stringDictionarySortingEnabled = false;
    boolean stringDictionaryEncodingEnabled = false;
    int preserveDirectEncodingStripeCount = 10;
    OrcWriterOptions.Builder builder = OrcWriterOptions.builder().withFlushPolicy(DefaultOrcWriterFlushPolicy.builder().withStripeMinSize(stripeMinSize).withStripeMaxSize(stripeMaxSize).withStripeMaxRowCount(stripeMaxRowCount).build()).withRowGroupMaxRowCount(rowGroupMaxRowCount).withDictionaryMaxMemory(dictionaryMaxMemory).withDictionaryMemoryAlmostFullRange(dictionaryMemoryRange).withDictionaryUsefulCheckPerChunkFrequency(dictionaryUsefulCheckPerChunkFrequency).withDictionaryUsefulCheckColumnSize(dictionaryUsefulCheckColumnSize).withMaxStringStatisticsLimit(stringMaxStatisticsLimit).withMaxCompressionBufferSize(maxCompressionBufferSize).withCompressionLevel(compressionLevel).withStreamLayoutFactory(streamLayoutFactory).withIntegerDictionaryEncodingEnabled(integerDictionaryEncodingEnabled).withStringDictionarySortingEnabled(stringDictionarySortingEnabled).withStringDictionaryEncodingEnabled(stringDictionaryEncodingEnabled).withPreserveDirectEncodingStripeCount(preserveDirectEncodingStripeCount);
    OrcWriterOptions options = builder.build();
    assertEquals(toIntExact(stripeMinSize.toBytes()), options.getFlushPolicy().getStripeMinBytes());
    assertEquals(toIntExact(stripeMaxSize.toBytes()), options.getFlushPolicy().getStripeMaxBytes());
    assertEquals(stripeMaxRowCount, options.getFlushPolicy().getStripeMaxRowCount());
    assertEquals(rowGroupMaxRowCount, options.getRowGroupMaxRowCount());
    assertEquals(dictionaryMaxMemory, options.getDictionaryMaxMemory());
    assertEquals(dictionaryMemoryRange, options.getDictionaryMemoryAlmostFullRange());
    assertEquals(dictionaryUsefulCheckPerChunkFrequency, options.getDictionaryUsefulCheckPerChunkFrequency());
    assertEquals(dictionaryUsefulCheckColumnSize, options.getDictionaryUsefulCheckColumnSize());
    assertEquals(stringMaxStatisticsLimit, options.getMaxStringStatisticsLimit());
    assertEquals(maxCompressionBufferSize, options.getMaxCompressionBufferSize());
    assertEquals(compressionLevel, options.getCompressionLevel());
    assertEquals(streamLayoutFactory, options.getStreamLayoutFactory());
    assertEquals(integerDictionaryEncodingEnabled, options.isIntegerDictionaryEncodingEnabled());
    assertEquals(stringDictionarySortingEnabled, options.isStringDictionarySortingEnabled());
    assertEquals(stringDictionaryEncodingEnabled, options.isStringDictionaryEncodingEnabled());
    assertEquals(Optional.empty(), options.getDwrfStripeCacheOptions());
    assertEquals(preserveDirectEncodingStripeCount, options.getPreserveDirectEncodingStripeCount());
}