Examples with CacheConfig com.facebook.presto.cache.CacheConfig used on opensource projects

Example 11 with CacheConfig

use of com.facebook.presto.cache.CacheConfig in project presto by prestodb.

the class TestHiveFileFormats method testDwrfOptimizedWriter.

@Test(dataProvider = "rowCount")
public void testDwrfOptimizedWriter(int rowCount) throws Exception {
    TestingConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveClientConfig().setOrcOptimizedWriterEnabled(true).setOrcWriterValidationPercentage(100.0), new OrcFileWriterConfig(), new ParquetFileWriterConfig(), new CacheConfig()).getSessionProperties());
    // DWRF does not support modern Hive types
    // A Presto page can not contain a map with null keys, so a page based writer can not write null keys
    List<TestColumn> testColumns = TEST_COLUMNS.stream().filter(testColumn -> !hasType(testColumn.getObjectInspector(), PrimitiveCategory.DATE, PrimitiveCategory.VARCHAR, PrimitiveCategory.CHAR, PrimitiveCategory.DECIMAL)).filter(testColumn -> !testColumn.getName().equals("t_map_null_key") && !testColumn.getName().equals("t_map_null_key_complex_value") && !testColumn.getName().equals("t_map_null_key_complex_key_value")).collect(toList());
    assertThatFileFormat(DWRF).withColumns(testColumns).withRowsCount(rowCount).withSession(session).withFileWriterFactory(new OrcFileWriterFactory(HDFS_ENVIRONMENT, new OutputStreamDataSinkFactory(), FUNCTION_AND_TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE, STATS, new OrcFileWriterConfig(), NO_ENCRYPTION)).isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)).isReadableByPageSource(new DwrfBatchPageSourceFactory(FUNCTION_AND_TYPE_MANAGER, FUNCTION_RESOLUTION, HIVE_CLIENT_CONFIG, HDFS_ENVIRONMENT, STATS, new StorageOrcFileTailSource(), StripeMetadataSourceFactory.of(new StorageStripeMetadataSource()), NO_ENCRYPTION));
}

Example 12 with CacheConfig

use of com.facebook.presto.cache.CacheConfig in project presto by prestodb.

the class TestHiveFileFormats method testOrcOptimizedWriter.

@Test(dataProvider = "rowCount")
public void testOrcOptimizedWriter(int rowCount) throws Exception {
    TestingConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveClientConfig().setOrcOptimizedWriterEnabled(true).setOrcWriterValidationPercentage(100.0), new OrcFileWriterConfig(), new ParquetFileWriterConfig(), new CacheConfig()).getSessionProperties());
    // A Presto page can not contain a map with null keys, so a page based writer can not write null keys
    List<TestColumn> testColumns = TEST_COLUMNS.stream().filter(testColumn -> !testColumn.getName().equals("t_map_null_key") && !testColumn.getName().equals("t_map_null_key_complex_value") && !testColumn.getName().equals("t_map_null_key_complex_key_value")).collect(toList());
    assertThatFileFormat(ORC).withColumns(testColumns).withRowsCount(rowCount).withSession(session).withFileWriterFactory(new OrcFileWriterFactory(HDFS_ENVIRONMENT, new OutputStreamDataSinkFactory(), FUNCTION_AND_TYPE_MANAGER, new NodeVersion("test"), HIVE_STORAGE_TIME_ZONE, STATS, new OrcFileWriterConfig(), NO_ENCRYPTION)).isReadableByRecordCursor(new GenericHiveRecordCursorProvider(HDFS_ENVIRONMENT)).isReadableByPageSource(new OrcBatchPageSourceFactory(FUNCTION_AND_TYPE_MANAGER, FUNCTION_RESOLUTION, false, HDFS_ENVIRONMENT, STATS, 100, new StorageOrcFileTailSource(), StripeMetadataSourceFactory.of(new StorageStripeMetadataSource())));
}

Example 13 with CacheConfig

use of com.facebook.presto.cache.CacheConfig in project presto by prestodb.

the class TestHiveFileFormats method testOrcUseColumnNames.

@Test(dataProvider = "rowCount")
public void testOrcUseColumnNames(int rowCount) throws Exception {
    TestingConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(new HiveClientConfig(), new OrcFileWriterConfig(), new ParquetFileWriterConfig(), new CacheConfig()).getSessionProperties());
    assertThatFileFormat(ORC).withWriteColumns(TEST_COLUMNS).withRowsCount(rowCount).withReadColumns(Lists.reverse(TEST_COLUMNS)).withSession(session).isReadableByPageSource(new OrcBatchPageSourceFactory(FUNCTION_AND_TYPE_MANAGER, FUNCTION_RESOLUTION, true, HDFS_ENVIRONMENT, STATS, 100, new StorageOrcFileTailSource(), StripeMetadataSourceFactory.of(new StorageStripeMetadataSource())));
}

Example 14 with CacheConfig

use of com.facebook.presto.cache.CacheConfig in project presto by prestodb.

the class TestAlluxioCachingFileSystem method testBasic.

private void testBasic(boolean validationEnabled) throws Exception {
    CacheConfig cacheConfig = new CacheConfig().setCacheType(ALLUXIO).setCachingEnabled(true).setBaseDirectory(cacheDirectory).setValidationEnabled(validationEnabled);
    AlluxioCacheConfig alluxioCacheConfig = new AlluxioCacheConfig();
    Configuration configuration = getHdfsConfiguration(cacheConfig, alluxioCacheConfig);
    AlluxioCachingFileSystem fileSystem = cachingFileSystem(configuration);
    Path p = new Path("/tmp");
    assertEquals(fileSystem.getDefaultBlockSize(p), 1024L);
    assertEquals(fileSystem.getDefaultReplication(p), 10);
    byte[] buffer = new byte[PAGE_SIZE * 2];
    int pageOffset = PAGE_SIZE;
    // new read
    resetBaseline();
    assertEquals(readFully(fileSystem, pageOffset + 10, buffer, 0, 100), 100);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE, 0);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL, 100);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL, PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_EVICTED, 0);
    validateBuffer(data, pageOffset + 10, buffer, 0, 100);
    // read within the cached page
    resetBaseline();
    assertEquals(readFully(fileSystem, pageOffset + 20, buffer, 0, 90), 90);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE, 90);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL, 0);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL, 0);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_EVICTED, 0);
    validateBuffer(data, pageOffset + 20, buffer, 0, 90);
    // read partially after the range of the cache
    resetBaseline();
    assertEquals(readFully(fileSystem, pageOffset + PAGE_SIZE - 10, buffer, 0, 100), 100);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE, 10);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL, 90);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL, PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_EVICTED, 0);
    validateBuffer(data, pageOffset + PAGE_SIZE - 10, buffer, 0, 100);
    // read partially before the range of the cache
    resetBaseline();
    assertEquals(readFully(fileSystem, pageOffset - 10, buffer, 10, 50), 50);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE, 40);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL, 10);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL, PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_EVICTED, 0);
    validateBuffer(data, pageOffset - 10, buffer, 10, 50);
    // skip one page
    resetBaseline();
    assertEquals(readFully(fileSystem, pageOffset + PAGE_SIZE * 3, buffer, 40, 50), 50);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE, 0);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL, 50);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL, PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_EVICTED, 0);
    validateBuffer(data, pageOffset + PAGE_SIZE * 3, buffer, 40, 50);
    // read between cached pages
    resetBaseline();
    assertEquals(readFully(fileSystem, pageOffset + PAGE_SIZE * 2 - 10, buffer, 400, PAGE_SIZE + 20), PAGE_SIZE + 20);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE, 20);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL, PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL, PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_EVICTED, 0);
    validateBuffer(data, pageOffset + PAGE_SIZE * 2 - 10, buffer, 400, PAGE_SIZE + 20);
}

Example 15 with CacheConfig

use of com.facebook.presto.cache.CacheConfig in project presto by prestodb.

the class TestAlluxioCachingFileSystem method testSyncRestoreFailure.

@Test(timeOut = 30_000, expectedExceptions = { IOException.class })
public void testSyncRestoreFailure() throws Exception {
    URI badCacheDirectory = createTempDirectory("alluxio_cache_bad").toUri();
    File cacheDirectory = new File(badCacheDirectory.getPath());
    cacheDirectory.setWritable(false);
    CacheConfig cacheConfig = new CacheConfig().setCacheType(ALLUXIO).setCachingEnabled(true).setBaseDirectory(badCacheDirectory);
    AlluxioCacheConfig alluxioCacheConfig = new AlluxioCacheConfig();
    Configuration configuration = getHdfsConfiguration(cacheConfig, alluxioCacheConfig);
    try {
        cachingFileSystem(configuration);
    } finally {
        cacheDirectory.setWritable(true);
    }
}