Examples with CacheQuota com.facebook.presto.hive.CacheQuota used on opensource projects

Example 1 with CacheQuota

use of com.facebook.presto.hive.CacheQuota in project presto by prestodb.

the class TestFileMergeCacheManager method testQuota.

@Test(timeOut = 30_000)
public void testQuota() throws InterruptedException, ExecutionException, IOException {
    TestingCacheStats stats = new TestingCacheStats();
    CacheManager cacheManager = fileMergeCacheManager(stats);
    byte[] buffer = new byte[10240];
    CacheQuota cacheQuota = new CacheQuota("test.table", Optional.of(DataSize.succinctDataSize(1, KILOBYTE)));
    // read within the cache quota
    assertFalse(readFully(cacheManager, cacheQuota, 42, buffer, 0, 100));
    assertEquals(stats.getCacheMiss(), 1);
    assertEquals(stats.getCacheHit(), 0);
    assertEquals(stats.getQuotaExceed(), 0);
    stats.trigger();
    assertEquals(stats.getInMemoryRetainedBytes(), 0);
    validateBuffer(data, 42, buffer, 0, 100);
    // read beyond cache quota
    assertFalse(readFully(cacheManager, cacheQuota, 47, buffer, 0, 9000));
    assertEquals(stats.getCacheMiss(), 1);
    assertEquals(stats.getCacheHit(), 0);
    assertEquals(stats.getQuotaExceed(), 1);
    assertEquals(stats.getInMemoryRetainedBytes(), 0);
    validateBuffer(data, 47, buffer, 0, 90);
    // previous data won't be evicted if last read exceed quota
    assertTrue(readFully(cacheManager, cacheQuota, 47, buffer, 0, 90));
    assertEquals(stats.getCacheMiss(), 1);
    assertEquals(stats.getCacheHit(), 1);
    assertEquals(stats.getQuotaExceed(), 1);
    assertEquals(stats.getInMemoryRetainedBytes(), 0);
    validateBuffer(data, 47, buffer, 0, 90);
}

Example 2 with CacheQuota

use of com.facebook.presto.hive.CacheQuota in project presto by prestodb.

the class TestAlluxioCachingFileSystem method testStressWithQuota.

@Test(invocationCount = 10)
public void testStressWithQuota() throws ExecutionException, InterruptedException, URISyntaxException, IOException {
    CacheQuota cacheQuota = new CacheQuota("test.table", Optional.of(DataSize.succinctDataSize(5, KILOBYTE)));
    CacheConfig cacheConfig = new CacheConfig().setCacheType(ALLUXIO).setCachingEnabled(true).setValidationEnabled(false).setBaseDirectory(cacheDirectory).setCacheQuotaScope(TABLE);
    AlluxioCacheConfig alluxioCacheConfig = new AlluxioCacheConfig().setMaxCacheSize(new DataSize(10, KILOBYTE)).setCacheQuotaEnabled(true);
    Configuration configuration = getHdfsConfiguration(cacheConfig, alluxioCacheConfig);
    AlluxioCachingFileSystem cachingFileSystem = cachingFileSystem(configuration);
    stressTest(data, (position, buffer, offset, length) -> {
        try {
            readFully(cachingFileSystem, cacheQuota, position, buffer, offset, length);
        } catch (Exception e) {
            e.printStackTrace();
        }
    });
}

Example 3 with CacheQuota

use of com.facebook.presto.hive.CacheQuota in project presto by prestodb.

the class TestAlluxioCachingFileSystem method testQuotaBasics.

@Test(timeOut = 30_000)
public void testQuotaBasics() throws Exception {
    DataSize quotaSize = DataSize.succinctDataSize(1, KILOBYTE);
    CacheQuota cacheQuota = new CacheQuota("test.table", Optional.of(quotaSize));
    CacheConfig cacheConfig = new CacheConfig().setCacheType(ALLUXIO).setCachingEnabled(true).setBaseDirectory(cacheDirectory).setValidationEnabled(false).setCacheQuotaScope(TABLE);
    AlluxioCacheConfig alluxioCacheConfig = new AlluxioCacheConfig().setCacheQuotaEnabled(true);
    Configuration configuration = getHdfsConfiguration(cacheConfig, alluxioCacheConfig);
    AlluxioCachingFileSystem fileSystem = cachingFileSystem(configuration);
    byte[] buffer = new byte[10240];
    // read within the cache quota
    resetBaseline();
    assertEquals(readFully(fileSystem, cacheQuota, 42, buffer, 0, 100), 100);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE, 0);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL, 100);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL, PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_EVICTED, 0);
    validateBuffer(data, 42, buffer, 0, 100);
    // read beyond cache quota
    resetBaseline();
    assertEquals(readFully(fileSystem, cacheQuota, 47, buffer, 0, 9000), 9000);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE, PAGE_SIZE - 47);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL, 9000 - PAGE_SIZE + 47);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL, (9000 / PAGE_SIZE) * PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_EVICTED, (9000 / PAGE_SIZE) * PAGE_SIZE);
    validateBuffer(data, 47, buffer, 0, 9000);
}

Example 4 with CacheQuota

use of com.facebook.presto.hive.CacheQuota in project presto by prestodb.

the class TestAlluxioCachingFileSystem method testQuotaUpdated.

@Test(timeOut = 30_000)
public void testQuotaUpdated() throws Exception {
    CacheQuota smallCacheQuota = new CacheQuota("test.table", Optional.of(DataSize.succinctDataSize(1, KILOBYTE)));
    CacheConfig cacheConfig = new CacheConfig().setCacheType(ALLUXIO).setCachingEnabled(true).setBaseDirectory(cacheDirectory).setValidationEnabled(false).setCacheQuotaScope(TABLE);
    AlluxioCacheConfig alluxioCacheConfig = new AlluxioCacheConfig().setCacheQuotaEnabled(true);
    Configuration configuration = getHdfsConfiguration(cacheConfig, alluxioCacheConfig);
    AlluxioCachingFileSystem fileSystem = cachingFileSystem(configuration);
    byte[] buffer = new byte[10240];
    // read beyond the small cache quota
    resetBaseline();
    assertEquals(readFully(fileSystem, smallCacheQuota, 0, buffer, 0, 9000), 9000);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE, 0);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL, 9000);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL, (9000 / PAGE_SIZE + 1) * PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_EVICTED, (9000 / PAGE_SIZE) * PAGE_SIZE);
    validateBuffer(data, 0, buffer, 0, 9000);
    // read again within an updated larger cache quota
    CacheQuota largeCacheQuota = new CacheQuota("test.table", Optional.of(DataSize.succinctDataSize(10, KILOBYTE)));
    resetBaseline();
    assertEquals(readFully(fileSystem, largeCacheQuota, 0, buffer, 0, 9000), 9000);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE, 9000 - (9000 / PAGE_SIZE) * PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL, (9000 / PAGE_SIZE) * PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_READ_EXTERNAL, (9000 / PAGE_SIZE) * PAGE_SIZE);
    checkMetrics(MetricKey.CLIENT_CACHE_BYTES_EVICTED, 0);
    validateBuffer(data, 0, buffer, 0, 9000);
}