Examples with OrcColumnId io.prestosql.orc.metadata.OrcColumnId used on opensource projects

Example 21 with OrcColumnId

use of io.prestosql.orc.metadata.OrcColumnId in project hetu-core by openlookeng.

the class TestAbstractNumbericColumnReader method testTypeCoercionShort.

@Test
public void testTypeCoercionShort() throws OrcCorruptionException {
    OrcColumn column = new OrcColumn("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0", new OrcColumnId(3), "cs_order_number", OrcType.OrcTypeKind.SHORT, new OrcDataSourceId("hdfs://hacluster/user/hive/warehouse/tpcds_orc_hive_1000.db/catalog_sales/cs_sold_date_sk=2452268/000896_0"), ImmutableList.of());
    ColumnReader actualShortColumnReader = ColumnReaders.createColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext(), null);
    ShortColumnReader expectedShortColumnReader = new ShortColumnReader(type, column, AggregatedMemoryContext.newSimpleAggregatedMemoryContext().newLocalMemoryContext(ColumnReaders.class.getSimpleName()));
    assertEquals(actualShortColumnReader.toString(), expectedShortColumnReader.toString());
}

Example 22 with OrcColumnId

use of io.prestosql.orc.metadata.OrcColumnId in project hetu-core by openlookeng.

the class StructColumnWriter method finishRowGroup.

@Override
public Map<OrcColumnId, ColumnStatistics> finishRowGroup() {
    checkState(!closed);
    ColumnStatistics statistics = new ColumnStatistics((long) nonNullValueCount, 0, null, null, null, null, null, null, null, null);
    rowGroupColumnStatistics.add(statistics);
    nonNullValueCount = 0;
    ImmutableMap.Builder<OrcColumnId, ColumnStatistics> columnStatistics = ImmutableMap.builder();
    columnStatistics.put(columnId, statistics);
    structFields.stream().map(ColumnWriter::finishRowGroup).forEach(columnStatistics::putAll);
    return columnStatistics.build();
}

Example 23 with OrcColumnId

use of io.prestosql.orc.metadata.OrcColumnId in project hetu-core by openlookeng.

the class TestReadBloomFilter method testType.

private static <T> void testType(Type type, List<T> uniqueValues, T inBloomFilter, T notInBloomFilter) throws Exception {
    Stream<T> writeValues = newArrayList(limit(cycle(uniqueValues), 30_000)).stream();
    try (TempFile tempFile = new TempFile()) {
        writeOrcColumnHive(tempFile.getFile(), ORC_12, LZ4, type, writeValues.iterator());
        // without predicate a normal block will be created
        try (OrcRecordReader recordReader = createCustomOrcRecordReader(tempFile, OrcPredicate.TRUE, type, MAX_BATCH_SIZE)) {
            assertEquals(recordReader.nextPage().getLoadedPage().getPositionCount(), 1024);
        }
        // predicate for specific value within the min/max range without bloom filter being enabled
        TupleDomainOrcPredicate noBloomFilterPredicate = TupleDomainOrcPredicate.builder().addColumn(new OrcColumnId(1), Domain.singleValue(type, notInBloomFilter)).build();
        try (OrcRecordReader recordReader = createCustomOrcRecordReader(tempFile, noBloomFilterPredicate, type, MAX_BATCH_SIZE)) {
            assertEquals(recordReader.nextPage().getLoadedPage().getPositionCount(), 1024);
        }
        // predicate for specific value within the min/max range with bloom filter enabled, but a value not in the bloom filter
        TupleDomainOrcPredicate notMatchBloomFilterPredicate = TupleDomainOrcPredicate.builder().addColumn(new OrcColumnId(1), Domain.singleValue(type, notInBloomFilter)).setBloomFiltersEnabled(true).build();
        try (OrcRecordReader recordReader = createCustomOrcRecordReader(tempFile, notMatchBloomFilterPredicate, type, MAX_BATCH_SIZE)) {
            assertNull(recordReader.nextPage());
        }
        // predicate for specific value within the min/max range with bloom filter enabled, and a value in the bloom filter
        TupleDomainOrcPredicate matchBloomFilterPredicate = TupleDomainOrcPredicate.builder().addColumn(new OrcColumnId(1), Domain.singleValue(type, inBloomFilter)).setBloomFiltersEnabled(true).build();
        try (OrcRecordReader recordReader = createCustomOrcRecordReader(tempFile, matchBloomFilterPredicate, type, MAX_BATCH_SIZE)) {
            assertEquals(recordReader.nextPage().getLoadedPage().getPositionCount(), 1024);
        }
    }
}