Examples with MetadataReader com.facebook.presto.parquet.cache.MetadataReader used on opensource projects

Example 1 with MetadataReader

use of com.facebook.presto.parquet.cache.MetadataReader in project presto by prestodb.

the class AbstractTestParquetReader method testCaching.

@Test
public void testCaching() throws Exception {
    Cache<ParquetDataSourceId, ParquetFileMetadata> parquetFileMetadataCache = CacheBuilder.newBuilder().maximumWeight(new DataSize(1, MEGABYTE).toBytes()).weigher((id, metadata) -> ((ParquetFileMetadata) metadata).getMetadataSize()).expireAfterAccess(new Duration(10, MINUTES).toMillis(), MILLISECONDS).recordStats().build();
    ParquetMetadataSource parquetMetadataSource = new CachingParquetMetadataSource(parquetFileMetadataCache, new MetadataReader());
    try (ParquetTester.TempFile tempFile = new ParquetTester.TempFile("test", "parquet")) {
        Iterable<Integer> values = intsBetween(0, 10);
        Iterator<?>[] readValues = stream(new Iterable<?>[] { values }).map(Iterable::iterator).toArray(size -> new Iterator<?>[size]);
        List<String> columnNames = singletonList("column1");
        List<Type> columnTypes = singletonList(INTEGER);
        writeParquetFileFromPresto(tempFile.getFile(), columnTypes, columnNames, readValues, 10, CompressionCodecName.GZIP);
        testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
        assertEquals(parquetFileMetadataCache.stats().missCount(), 1);
        assertEquals(parquetFileMetadataCache.stats().hitCount(), 0);
        testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
        assertEquals(parquetFileMetadataCache.stats().missCount(), 1);
        assertEquals(parquetFileMetadataCache.stats().hitCount(), 1);
        testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
        assertEquals(parquetFileMetadataCache.stats().missCount(), 1);
        assertEquals(parquetFileMetadataCache.stats().hitCount(), 2);
        parquetFileMetadataCache.invalidateAll();
        testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
        assertEquals(parquetFileMetadataCache.stats().missCount(), 2);
        assertEquals(parquetFileMetadataCache.stats().hitCount(), 2);
        testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
        assertEquals(parquetFileMetadataCache.stats().missCount(), 2);
        assertEquals(parquetFileMetadataCache.stats().hitCount(), 3);
    }
}

Example 2 with MetadataReader

use of com.facebook.presto.parquet.cache.MetadataReader in project presto by prestodb.

the class HiveTestUtils method getDefaultHiveBatchPageSourceFactories.

public static Set<HiveBatchPageSourceFactory> getDefaultHiveBatchPageSourceFactories(HiveClientConfig hiveClientConfig, MetastoreClientConfig metastoreClientConfig) {
    FileFormatDataSourceStats stats = new FileFormatDataSourceStats();
    HdfsEnvironment testHdfsEnvironment = createTestHdfsEnvironment(hiveClientConfig, metastoreClientConfig);
    return ImmutableSet.<HiveBatchPageSourceFactory>builder().add(new RcFilePageSourceFactory(FUNCTION_AND_TYPE_MANAGER, testHdfsEnvironment, stats)).add(new OrcBatchPageSourceFactory(FUNCTION_AND_TYPE_MANAGER, FUNCTION_RESOLUTION, hiveClientConfig, testHdfsEnvironment, stats, new StorageOrcFileTailSource(), StripeMetadataSourceFactory.of(new StorageStripeMetadataSource()))).add(new DwrfBatchPageSourceFactory(FUNCTION_AND_TYPE_MANAGER, FUNCTION_RESOLUTION, hiveClientConfig, testHdfsEnvironment, stats, new StorageOrcFileTailSource(), StripeMetadataSourceFactory.of(new StorageStripeMetadataSource()), NO_ENCRYPTION)).add(new ParquetPageSourceFactory(FUNCTION_AND_TYPE_MANAGER, FUNCTION_RESOLUTION, testHdfsEnvironment, stats, new MetadataReader())).add(new PageFilePageSourceFactory(testHdfsEnvironment, new BlockEncodingManager())).build();
}

Example 3 with MetadataReader

use of com.facebook.presto.parquet.cache.MetadataReader in project presto by prestodb.

the class HiveClientModule method createParquetMetadataSource.

@Singleton
@Provides
public ParquetMetadataSource createParquetMetadataSource(ParquetCacheConfig parquetCacheConfig, MBeanExporter exporter) {
    ParquetMetadataSource parquetMetadataSource = new MetadataReader();
    if (parquetCacheConfig.isMetadataCacheEnabled()) {
        Cache<ParquetDataSourceId, ParquetFileMetadata> cache = CacheBuilder.newBuilder().maximumWeight(parquetCacheConfig.getMetadataCacheSize().toBytes()).weigher((id, metadata) -> ((ParquetFileMetadata) metadata).getMetadataSize()).expireAfterAccess(parquetCacheConfig.getMetadataCacheTtlSinceLastAccess().toMillis(), MILLISECONDS).recordStats().build();
        CacheStatsMBean cacheStatsMBean = new CacheStatsMBean(cache);
        parquetMetadataSource = new CachingParquetMetadataSource(cache, parquetMetadataSource);
        exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_ParquetMetadata"), cacheStatsMBean);
    }
    return parquetMetadataSource;
}