Examples with CacheContext alluxio.client.file.CacheContext used on opensource projects

Example 6 with CacheContext

use of alluxio.client.file.CacheContext in project presto by prestodb.

the class AlluxioCachingFileSystem method initialize.

@Override
public synchronized void initialize(URI uri, Configuration configuration) throws IOException {
    this.localCacheFileSystem = new LocalCacheFileSystem(dataTier, uriStatus -> {
        // CacheContext is the mechanism to pass the hiveFileContext to the source filesystem
        // hiveFileContext is critical to use to open file.
        CacheContext cacheContext = uriStatus.getCacheContext();
        checkState(cacheContext instanceof PrestoCacheContext);
        HiveFileContext hiveFileContext = ((PrestoCacheContext) cacheContext).getHiveFileContext();
        try {
            return dataTier.openFile(new Path(uriStatus.getPath()), hiveFileContext);
        } catch (Exception e) {
            throw new IOException("Failed to open file", e);
        }
    });
    this.cacheQuotaEnabled = configuration.getBoolean(USER_CLIENT_CACHE_QUOTA_ENABLED.getName(), false);
    localCacheFileSystem.initialize(uri, configuration);
}

Example 7 with CacheContext

use of alluxio.client.file.CacheContext in project presto by prestodb.

the class AlluxioCachingFileSystem method openFile.

@Override
public FSDataInputStream openFile(Path path, HiveFileContext hiveFileContext) throws Exception {
    // Using Alluxio caching requires knowing file size for now
    if (hiveFileContext.isCacheable() && hiveFileContext.getFileSize().isPresent()) {
        // FilePath is a unique identifier for a file, however it can be a long string
        // hence using md5 hash of the file path as the identifier in the cache.
        // We don't set fileId because fileId is Alluxio specific
        FileInfo info = new FileInfo().setLastModificationTimeMs(hiveFileContext.getModificationTime()).setPath(path.toString()).setFolder(false).setLength(hiveFileContext.getFileSize().get());
        String cacheIdentifier = md5().hashString(path.toString(), UTF_8).toString();
        // CacheContext is the mechanism to pass the cache related context to the source filesystem
        CacheContext cacheContext = PrestoCacheContext.build(cacheIdentifier, hiveFileContext, cacheQuotaEnabled);
        URIStatus uriStatus = new URIStatus(info, cacheContext);
        FSDataInputStream cachingInputStream = localCacheFileSystem.open(uriStatus, BUFFER_SIZE);
        if (cacheValidationEnabled) {
            return new CacheValidatingInputStream(cachingInputStream, dataTier.openFile(path, hiveFileContext));
        }
        return cachingInputStream;
    }
    return dataTier.openFile(path, hiveFileContext);
}

Example 8 with CacheContext

use of alluxio.client.file.CacheContext in project alluxio by Alluxio.

the class LocalCacheManagerTest method putWithInsufficientQuota.

@Test
public void putWithInsufficientQuota() throws Exception {
    mConf.set(PropertyKey.USER_CLIENT_CACHE_QUOTA_ENABLED, true);
    mMetaStore = new QuotaMetaStore(mConf);
    mCacheManager = createLocalCacheManager(mConf, mMetaStore, mPageStore);
    CacheScope scope = CacheScope.create("schema.table.partition");
    CacheContext context = CacheContext.defaults().setCacheScope(scope);
    // insufficient partition quota
    assertFalse(mCacheManager.put(PAGE_ID1, PAGE1, context.setCacheQuota(new CacheQuota(ImmutableMap.of(CacheScope.Level.PARTITION, (long) PAGE1.length - 1)))));
    // insufficient table quota
    assertFalse(mCacheManager.put(PAGE_ID1, PAGE1, context.setCacheQuota(new CacheQuota(ImmutableMap.of(CacheScope.Level.TABLE, (long) PAGE1.length - 1)))));
    // insufficient schema quota
    assertFalse(mCacheManager.put(PAGE_ID1, PAGE1, context.setCacheQuota(new CacheQuota(ImmutableMap.of(CacheScope.Level.SCHEMA, (long) PAGE1.length - 1)))));
    // insufficient global quota
    assertFalse(mCacheManager.put(PAGE_ID1, PAGE1, context.setCacheQuota(new CacheQuota(ImmutableMap.of(CacheScope.Level.GLOBAL, (long) PAGE1.length - 1)))));
    // without quota
    assertTrue(mCacheManager.put(PAGE_ID1, PAGE1));
}

Example 9 with CacheContext

use of alluxio.client.file.CacheContext in project alluxio by Alluxio.

the class LocalCacheManagerTest method putWithQuotaMoreThanCacheCapacity.

@Test
public void putWithQuotaMoreThanCacheCapacity() throws Exception {
    mConf.set(PropertyKey.USER_CLIENT_CACHE_QUOTA_ENABLED, true);
    CacheScope partitionCacheScope = CacheScope.create("schema.table.partition");
    CacheScope tableCacheScope = CacheScope.create("schema.table");
    CacheScope schemaCacheScope = CacheScope.create("schema");
    CacheScope[] quotaCacheScopes = { partitionCacheScope, tableCacheScope, schemaCacheScope, CacheScope.GLOBAL };
    for (CacheScope cacheScope : quotaCacheScopes) {
        mMetaStore = new QuotaMetaStore(mConf);
        mPageStore = PageStore.create(PageStoreOptions.create(mConf));
        mCacheManager = createLocalCacheManager(mConf, mMetaStore, mPageStore);
        CacheQuota quota = new CacheQuota(ImmutableMap.of(cacheScope.level(), (long) CACHE_SIZE_BYTES + 1));
        int cacheSize = CACHE_SIZE_BYTES / PAGE_SIZE_BYTES;
        for (int i = 0; i < 2 * cacheSize; i++) {
            PageId pageId = new PageId("3", i);
            CacheContext context = CacheContext.defaults().setCacheScope(partitionCacheScope).setCacheQuota(quota);
            assertTrue(mCacheManager.put(pageId, page(i, PAGE_SIZE_BYTES), context));
            if (i >= cacheSize) {
                assertEquals(0, mCacheManager.get(new PageId("3", i - cacheSize), PAGE_SIZE_BYTES, mBuf, 0));
                // check the subsequent page is still in cache
                assertEquals(true, mMetaStore.hasPage(new PageId("3", i - cacheSize + 1)));
            }
        }
    }
}

Example 10 with CacheContext

use of alluxio.client.file.CacheContext in project alluxio by Alluxio.

the class LocalCacheManagerTest method putWithInsufficientParentQuota.

@Test
public void putWithInsufficientParentQuota() throws Exception {
    mConf.set(PropertyKey.USER_CLIENT_CACHE_QUOTA_ENABLED, true);
    CacheScope partitionCacheScope1 = CacheScope.create("schema.table.partition1");
    CacheScope partitionCacheScope2 = CacheScope.create("schema.table.partition2");
    CacheScope tableCacheScope = CacheScope.create("schema.table");
    CacheScope schemaCacheScope = CacheScope.create("schema");
    CacheScope[] quotaCacheScopes = { tableCacheScope, schemaCacheScope, CacheScope.GLOBAL };
    for (CacheScope cacheScope : quotaCacheScopes) {
        mMetaStore = new QuotaMetaStore(mConf);
        mPageStore = PageStore.create(PageStoreOptions.create(mConf));
        mCacheManager = createLocalCacheManager(mConf, mMetaStore, mPageStore);
        CacheQuota quota = new CacheQuota(ImmutableMap.of(partitionCacheScope1.level(), (long) PAGE1.length + PAGE2.length, cacheScope.level(), (long) PAGE1.length + PAGE2.length - 1));
        CacheContext context1 = CacheContext.defaults().setCacheScope(partitionCacheScope1).setCacheQuota(quota);
        assertTrue(mCacheManager.put(PAGE_ID1, PAGE1, context1));
        assertEquals(PAGE1.length, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0));
        CacheContext context2 = CacheContext.defaults().setCacheScope(partitionCacheScope2).setCacheQuota(quota);
        assertTrue(mCacheManager.put(PAGE_ID2, PAGE2, context2));
        assertEquals(0, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0));
        assertEquals(PAGE2.length, mCacheManager.get(PAGE_ID2, PAGE2.length, mBuf, 0));
    }
}