Examples with ColumnSizeLayoutFactory com.facebook.presto.orc.writer.StreamLayoutFactory.ColumnSizeLayoutFactory used on opensource projects

Example 1 with ColumnSizeLayoutFactory

use of com.facebook.presto.orc.writer.StreamLayoutFactory.ColumnSizeLayoutFactory in project presto by prestodb.

the class TestOrcWriterOptions method testToString.

@Test
public void testToString() {
    DataSize stripeMinSize = new DataSize(13, MEGABYTE);
    DataSize stripeMaxSize = new DataSize(27, MEGABYTE);
    int stripeMaxRowCount = 1_100_000;
    int rowGroupMaxRowCount = 15_000;
    DataSize dictionaryMaxMemory = new DataSize(13_000, KILOBYTE);
    DataSize dictionaryMemoryRange = new DataSize(1_000, KILOBYTE);
    int dictionaryUsefulCheckPerChunkFrequency = 9_999;
    DataSize dictionaryUsefulCheckColumnSize = new DataSize(1, MEGABYTE);
    DataSize stringMaxStatisticsLimit = new DataSize(128, BYTE);
    DataSize maxCompressionBufferSize = new DataSize(512, KILOBYTE);
    DataSize dwrfStripeCacheMaxSize = new DataSize(4, MEGABYTE);
    DwrfStripeCacheMode dwrfStripeCacheMode = DwrfStripeCacheMode.INDEX_AND_FOOTER;
    OptionalInt compressionLevel = OptionalInt.of(5);
    StreamLayoutFactory streamLayoutFactory = new ColumnSizeLayoutFactory();
    boolean integerDictionaryEncodingEnabled = false;
    boolean stringDictionarySortingEnabled = true;
    int preserveDirectEncodingStripeCount = 0;
    OrcWriterOptions writerOptions = OrcWriterOptions.builder().withFlushPolicy(DefaultOrcWriterFlushPolicy.builder().withStripeMinSize(stripeMinSize).withStripeMaxSize(stripeMaxSize).withStripeMaxRowCount(stripeMaxRowCount).build()).withRowGroupMaxRowCount(rowGroupMaxRowCount).withDictionaryMaxMemory(dictionaryMaxMemory).withDictionaryMemoryAlmostFullRange(dictionaryMemoryRange).withDictionaryUsefulCheckPerChunkFrequency(dictionaryUsefulCheckPerChunkFrequency).withDictionaryUsefulCheckColumnSize(dictionaryUsefulCheckColumnSize).withMaxStringStatisticsLimit(stringMaxStatisticsLimit).withMaxCompressionBufferSize(maxCompressionBufferSize).withCompressionLevel(compressionLevel).withStreamLayoutFactory(streamLayoutFactory).withIntegerDictionaryEncodingEnabled(integerDictionaryEncodingEnabled).withStringDictionarySortingEnabled(stringDictionarySortingEnabled).withDwrfStripeCacheEnabled(true).withDwrfStripeCacheMaxSize(dwrfStripeCacheMaxSize).withDwrfStripeCacheMode(dwrfStripeCacheMode).withPreserveDirectEncodingStripeCount(preserveDirectEncodingStripeCount).build();
    String expectedString = "OrcWriterOptions{flushPolicy=DefaultOrcWriterFlushPolicy{stripeMaxRowCount=1100000, " + "stripeMinBytes=13631488, stripeMaxBytes=28311552}, rowGroupMaxRowCount=15000, " + "dictionaryMaxMemory=13000kB, dictionaryMemoryAlmostFullRange=1000kB, dictionaryUsefulCheckPerChunkFrequency=9999, " + "dictionaryUsefulCheckColumnSize=1MB, maxStringStatisticsLimit=128B, maxCompressionBufferSize=512kB, " + "compressionLevel=OptionalInt[5], streamLayoutFactory=ColumnSizeLayoutFactory{}, integerDictionaryEncodingEnabled=false, " + "stringDictionarySortingEnabled=true, stringDictionaryEncodingEnabled=true, " + "dwrfWriterOptions=Optional[DwrfStripeCacheOptions{stripeCacheMode=INDEX_AND_FOOTER, stripeCacheMaxSize=4MB}], " + "ignoreDictionaryRowGroupSizes=false, preserveDirectEncodingStripeCount=0}";
    assertEquals(expectedString, writerOptions.toString());
}

Example 2 with ColumnSizeLayoutFactory

use of com.facebook.presto.orc.writer.StreamLayoutFactory.ColumnSizeLayoutFactory in project presto by prestodb.

the class TestOrcFileWriterConfig method testStreamLayoutOption.

@Test
public void testStreamLayoutOption() {
    OrcFileWriterConfig config = new OrcFileWriterConfig();
    config.setStreamLayoutType(BY_STREAM_SIZE);
    OrcWriterOptions options = config.toOrcWriterOptionsBuilder().build();
    assertTrue(options.getStreamLayoutFactory() instanceof StreamSizeLayoutFactory);
    config.setStreamLayoutType(BY_COLUMN_SIZE);
    options = config.toOrcWriterOptionsBuilder().build();
    assertTrue(options.getStreamLayoutFactory() instanceof ColumnSizeLayoutFactory);
}