Examples with DictionaryPage com.facebook.presto.parquet.DictionaryPage used on opensource projects

Example 6 with DictionaryPage

use of com.facebook.presto.parquet.DictionaryPage in project presto by prestodb.

the class TestValuesDecoders method testInt32RLEDictionary.

@Test
public void testInt32RLEDictionary() throws IOException {
    Random random = new Random(83);
    int valueCount = 2048;
    int dictionarySize = 29;
    List<Object> dictionary = new ArrayList<>();
    List<Integer> dictionaryIds = new ArrayList<>();
    byte[] dictionaryPage = generatePlainValuesPage(dictionarySize, 32, random, dictionary);
    byte[] dataPage = generateDictionaryIdPage2048(dictionarySize - 1, random, dictionaryIds);
    List<Object> expectedValues = new ArrayList<>();
    for (Integer dictionaryId : dictionaryIds) {
        expectedValues.add(dictionary.get(dictionaryId));
    }
    IntegerDictionary integerDictionary = new IntegerDictionary(new DictionaryPage(Slices.wrappedBuffer(dictionaryPage), dictionarySize, PLAIN_DICTIONARY));
    int32BatchReadWithSkipHelper(valueCount, 0, valueCount, int32Dictionary(dataPage, dictionarySize, integerDictionary), expectedValues);
    int32BatchReadWithSkipHelper(29, 0, valueCount, int32Dictionary(dataPage, dictionarySize, integerDictionary), expectedValues);
    int32BatchReadWithSkipHelper(89, 0, valueCount, int32Dictionary(dataPage, dictionarySize, integerDictionary), expectedValues);
    int32BatchReadWithSkipHelper(1024, 0, valueCount, int32Dictionary(dataPage, dictionarySize, integerDictionary), expectedValues);
    int32BatchReadWithSkipHelper(256, 29, valueCount, int32Dictionary(dataPage, dictionarySize, integerDictionary), expectedValues);
    int32BatchReadWithSkipHelper(89, 29, valueCount, int32Dictionary(dataPage, dictionarySize, integerDictionary), expectedValues);
    int32BatchReadWithSkipHelper(1024, 1024, valueCount, int32Dictionary(dataPage, dictionarySize, integerDictionary), expectedValues);
}

Example 7 with DictionaryPage

use of com.facebook.presto.parquet.DictionaryPage in project presto by prestodb.

the class TestValuesDecoders method testTimestampRLEDictionary.

@Test
public void testTimestampRLEDictionary() throws IOException {
    Random random = new Random(83);
    int valueCount = 2048;
    int dictionarySize = 29;
    List<Object> dictionary = new ArrayList<>();
    List<Integer> dictionaryIds = new ArrayList<>();
    byte[] dictionaryPage = generatePlainValuesPage(dictionarySize, 96, random, dictionary);
    byte[] dataPage = generateDictionaryIdPage2048(dictionarySize - 1, random, dictionaryIds);
    List<Object> expectedValues = new ArrayList<>();
    for (Integer dictionaryId : dictionaryIds) {
        expectedValues.add(dictionary.get(dictionaryId));
    }
    TimestampDictionary tsDictionary = new TimestampDictionary(new DictionaryPage(Slices.wrappedBuffer(dictionaryPage), dictionarySize, PLAIN_DICTIONARY));
    timestampBatchReadWithSkipHelper(valueCount, 0, valueCount, timestampDictionary(dataPage, dictionarySize, tsDictionary), expectedValues);
    timestampBatchReadWithSkipHelper(29, 0, valueCount, timestampDictionary(dataPage, dictionarySize, tsDictionary), expectedValues);
    timestampBatchReadWithSkipHelper(89, 0, valueCount, timestampDictionary(dataPage, dictionarySize, tsDictionary), expectedValues);
    timestampBatchReadWithSkipHelper(1024, 0, valueCount, timestampDictionary(dataPage, dictionarySize, tsDictionary), expectedValues);
    timestampBatchReadWithSkipHelper(256, 29, valueCount, timestampDictionary(dataPage, dictionarySize, tsDictionary), expectedValues);
    timestampBatchReadWithSkipHelper(89, 29, valueCount, timestampDictionary(dataPage, dictionarySize, tsDictionary), expectedValues);
    timestampBatchReadWithSkipHelper(1024, 1024, valueCount, timestampDictionary(dataPage, dictionarySize, tsDictionary), expectedValues);
}

Example 8 with DictionaryPage

use of com.facebook.presto.parquet.DictionaryPage in project presto by prestodb.

the class Int32FlatBatchReader method init.

@Override
public void init(PageReader pageReader, Field field, RowRanges rowRanges) {
    checkArgument(!isInitialized(), "Parquet batch reader already initialized");
    this.pageReader = requireNonNull(pageReader, "pageReader is null");
    checkArgument(pageReader.getTotalValueCount() > 0, "page is empty");
    this.field = requireNonNull(field, "field is null");
    DictionaryPage dictionaryPage = pageReader.readDictionaryPage();
    if (dictionaryPage != null) {
        dictionary = Dictionaries.createDictionary(columnDescriptor, dictionaryPage);
    }
}

Example 9 with DictionaryPage

use of com.facebook.presto.parquet.DictionaryPage in project presto by prestodb.

the class TupleDomainParquetPredicate method getDomain.

@VisibleForTesting
public static Domain getDomain(Type type, DictionaryDescriptor dictionaryDescriptor) {
    if (dictionaryDescriptor == null) {
        return Domain.all(type);
    }
    ColumnDescriptor columnDescriptor = dictionaryDescriptor.getColumnDescriptor();
    Optional<DictionaryPage> dictionaryPage = dictionaryDescriptor.getDictionaryPage();
    if (!dictionaryPage.isPresent()) {
        return Domain.all(type);
    }
    Dictionary dictionary;
    try {
        dictionary = dictionaryPage.get().getEncoding().initDictionary(columnDescriptor, dictionaryPage.get());
    } catch (Exception e) {
        // OK to ignore exception when reading dictionaries
        return Domain.all(type);
    }
    int dictionarySize = dictionaryPage.get().getDictionarySize();
    DictionaryValueConverter converter = new DictionaryValueConverter(dictionary);
    Function<Integer, Object> convertFunction = converter.getConverter(columnDescriptor.getPrimitiveType());
    List<Object> values = new ArrayList<>();
    for (int i = 0; i < dictionarySize; i++) {
        values.add(convertFunction.apply(i));
    }
    // TODO: when min == max (i.e., singleton ranges, the construction of Domains can be done more efficiently
    return getDomain(columnDescriptor, type, values, values, true);
}

Example 10 with DictionaryPage

use of com.facebook.presto.parquet.DictionaryPage in project presto by prestodb.

the class AbstractNestedBatchReader method init.

@Override
public void init(PageReader pageReader, Field field, RowRanges rowRanges) {
    Preconditions.checkState(!isInitialized(), "already initialized");
    this.pageReader = requireNonNull(pageReader, "pageReader is null");
    checkArgument(pageReader.getTotalValueCount() > 0, "page is empty");
    this.field = requireNonNull(field, "field is null");
    DictionaryPage dictionaryPage = pageReader.readDictionaryPage();
    if (dictionaryPage != null) {
        dictionary = Dictionaries.createDictionary(columnDescriptor, dictionaryPage);
    }
}