Examples with DictionaryColumnWriter com.facebook.presto.orc.writer.DictionaryColumnWriter used on opensource projects

Example 1 with DictionaryColumnWriter

use of com.facebook.presto.orc.writer.DictionaryColumnWriter in project presto by prestodb.

the class TestDictionaryColumnWriter method testDictionaryRetainedSizeWithDifferentSettings.

@Test
public void testDictionaryRetainedSizeWithDifferentSettings() {
    DictionaryColumnWriter ignoredRowGroupWriter = getStringDictionaryColumnWriter(true);
    DictionaryColumnWriter withRowGroupWriter = getStringDictionaryColumnWriter(false);
    int numEntries = 10_000;
    int numBlocks = 10;
    BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, numEntries);
    Slice slice = utf8Slice("SomeString");
    for (int i = 0; i < numEntries; i++) {
        VARCHAR.writeSlice(blockBuilder, slice);
    }
    Block block = blockBuilder.build();
    for (int i = 0; i < numBlocks; i++) {
        writeBlock(ignoredRowGroupWriter, block);
        writeBlock(withRowGroupWriter, block);
    }
    long ignoredRowGroupBytes = ignoredRowGroupWriter.getRowGroupRetainedSizeInBytes();
    long withRowGroupBytes = withRowGroupWriter.getRowGroupRetainedSizeInBytes();
    long expectedDictionaryIndexSize = (numBlocks * numEntries * SIZE_OF_BYTE);
    String message = String.format("Ignored bytes %s With bytes %s", ignoredRowGroupBytes, withRowGroupBytes);
    assertTrue(ignoredRowGroupBytes + expectedDictionaryIndexSize <= withRowGroupBytes, message);
}

Example 2 with DictionaryColumnWriter

use of com.facebook.presto.orc.writer.DictionaryColumnWriter in project presto by prestodb.

the class BenchmarkDictionaryWriter method writeDictionaryAndConvert.

@Benchmark
public void writeDictionaryAndConvert(BenchmarkData data) {
    DictionaryColumnWriter columnWriter = getDictionaryColumnWriter(data, true);
    for (Block block : data.getBlocks()) {
        columnWriter.beginRowGroup();
        columnWriter.writeBlock(block);
        columnWriter.finishRowGroup();
    }
    int maxDirectBytes = toIntExact(new DataSize(512, MEGABYTE).toBytes());
    OptionalInt optionalInt = columnWriter.tryConvertToDirect(maxDirectBytes);
    checkState(optionalInt.isPresent(), "Column did not covert to direct");
    columnWriter.close();
    columnWriter.reset();
}

Example 3 with DictionaryColumnWriter

use of com.facebook.presto.orc.writer.DictionaryColumnWriter in project presto by prestodb.

the class BenchmarkDictionaryWriter method getDictionaryColumnWriter.

private DictionaryColumnWriter getDictionaryColumnWriter(BenchmarkData data, boolean sortStringDictionaryKeys) {
    DictionaryColumnWriter columnWriter;
    Type type = data.getType();
    ColumnWriterOptions columnWriterOptions = getColumnWriterOptions(sortStringDictionaryKeys);
    if (type.equals(VARCHAR)) {
        columnWriter = new SliceDictionaryColumnWriter(COLUMN_INDEX, type, columnWriterOptions, Optional.empty(), DWRF, DWRF.createMetadataWriter());
    } else {
        columnWriter = new LongDictionaryColumnWriter(COLUMN_INDEX, type, columnWriterOptions, Optional.empty(), DWRF, DWRF.createMetadataWriter());
    }
    return columnWriter;
}