use of com.facebook.presto.parquet.cache.CachingParquetMetadataSource in project presto by prestodb.
the class AbstractTestParquetReader method testCaching.
@Test
public void testCaching() throws Exception {
Cache<ParquetDataSourceId, ParquetFileMetadata> parquetFileMetadataCache = CacheBuilder.newBuilder().maximumWeight(new DataSize(1, MEGABYTE).toBytes()).weigher((id, metadata) -> ((ParquetFileMetadata) metadata).getMetadataSize()).expireAfterAccess(new Duration(10, MINUTES).toMillis(), MILLISECONDS).recordStats().build();
ParquetMetadataSource parquetMetadataSource = new CachingParquetMetadataSource(parquetFileMetadataCache, new MetadataReader());
try (ParquetTester.TempFile tempFile = new ParquetTester.TempFile("test", "parquet")) {
Iterable<Integer> values = intsBetween(0, 10);
Iterator<?>[] readValues = stream(new Iterable<?>[] { values }).map(Iterable::iterator).toArray(size -> new Iterator<?>[size]);
List<String> columnNames = singletonList("column1");
List<Type> columnTypes = singletonList(INTEGER);
writeParquetFileFromPresto(tempFile.getFile(), columnTypes, columnNames, readValues, 10, CompressionCodecName.GZIP);
testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
assertEquals(parquetFileMetadataCache.stats().missCount(), 1);
assertEquals(parquetFileMetadataCache.stats().hitCount(), 0);
testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
assertEquals(parquetFileMetadataCache.stats().missCount(), 1);
assertEquals(parquetFileMetadataCache.stats().hitCount(), 1);
testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
assertEquals(parquetFileMetadataCache.stats().missCount(), 1);
assertEquals(parquetFileMetadataCache.stats().hitCount(), 2);
parquetFileMetadataCache.invalidateAll();
testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
assertEquals(parquetFileMetadataCache.stats().missCount(), 2);
assertEquals(parquetFileMetadataCache.stats().hitCount(), 2);
testSingleRead(new Iterable<?>[] { values }, columnNames, columnTypes, parquetMetadataSource, tempFile.getFile());
assertEquals(parquetFileMetadataCache.stats().missCount(), 2);
assertEquals(parquetFileMetadataCache.stats().hitCount(), 3);
}
}
use of com.facebook.presto.parquet.cache.CachingParquetMetadataSource in project presto by prestodb.
the class HiveClientModule method createParquetMetadataSource.
@Singleton
@Provides
public ParquetMetadataSource createParquetMetadataSource(ParquetCacheConfig parquetCacheConfig, MBeanExporter exporter) {
ParquetMetadataSource parquetMetadataSource = new MetadataReader();
if (parquetCacheConfig.isMetadataCacheEnabled()) {
Cache<ParquetDataSourceId, ParquetFileMetadata> cache = CacheBuilder.newBuilder().maximumWeight(parquetCacheConfig.getMetadataCacheSize().toBytes()).weigher((id, metadata) -> ((ParquetFileMetadata) metadata).getMetadataSize()).expireAfterAccess(parquetCacheConfig.getMetadataCacheTtlSinceLastAccess().toMillis(), MILLISECONDS).recordStats().build();
CacheStatsMBean cacheStatsMBean = new CacheStatsMBean(cache);
parquetMetadataSource = new CachingParquetMetadataSource(cache, parquetMetadataSource);
exporter.export(generatedNameOf(CacheStatsMBean.class, connectorId + "_ParquetMetadata"), cacheStatsMBean);
}
return parquetMetadataSource;
}
Aggregations