Examples with CacheContext alluxio.client.file.CacheContext used on opensource projects

Example 1 with CacheContext

use of alluxio.client.file.CacheContext in project alluxio by Alluxio.

the class LocalCacheFileSystem method open.

@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
    if (mCacheManager == null) {
        return mExternalFileSystem.open(path, bufferSize);
    }
    FileStatus externalFileStatus = mExternalFileSystem.getFileStatus(path);
    // Note that, we don't set have fileId here because fileId is Alluxio specific
    FileInfo info = new FileInfo().setLength(externalFileStatus.getLen()).setPath(externalFileStatus.getPath().toString()).setFolder(externalFileStatus.isDirectory()).setBlockSizeBytes(externalFileStatus.getBlockSize()).setLastModificationTimeMs(externalFileStatus.getModificationTime()).setLastAccessTimeMs(externalFileStatus.getAccessTime()).setOwner(externalFileStatus.getOwner()).setGroup(externalFileStatus.getGroup());
    // FilePath is a unique identifier for a file, however it can be a long string
    // hence using md5 hash of the file path as the identifier in the cache.
    CacheContext context = CacheContext.defaults().setCacheIdentifier(md5().hashString(externalFileStatus.getPath().toString(), UTF_8).toString());
    URIStatus status = new URIStatus(info, context);
    return open(status, bufferSize);
}

Example 2 with CacheContext

use of alluxio.client.file.CacheContext in project alluxio by Alluxio.

the class LocalCacheFileInStream method readInternal.

// TODO(binfan): take ByteBuffer once CacheManager takes ByteBuffer to avoid extra mem copy
private int readInternal(byte[] b, int off, int len, ReadType readType, long pos, boolean isPositionedRead) throws IOException {
    Preconditions.checkArgument(len >= 0, "length should be non-negative");
    Preconditions.checkArgument(off >= 0, "offset should be non-negative");
    Preconditions.checkArgument(pos >= 0, "position should be non-negative");
    if (len == 0) {
        return 0;
    }
    if (pos >= mStatus.getLength()) {
        // at end of file
        return -1;
    }
    int totalBytesRead = 0;
    long currentPosition = pos;
    long lengthToRead = Math.min(len, mStatus.getLength() - pos);
    // used in positionedRead, so make stopwatch a local variable rather than class member
    Stopwatch stopwatch = createUnstartedStopwatch();
    // for each page, check if it is available in the cache
    while (totalBytesRead < lengthToRead) {
        long currentPage = currentPosition / mPageSize;
        int currentPageOffset = (int) (currentPosition % mPageSize);
        int bytesLeftInPage = (int) Math.min(mPageSize - currentPageOffset, lengthToRead - totalBytesRead);
        PageId pageId;
        CacheContext cacheContext = mStatus.getCacheContext();
        if (cacheContext != null && cacheContext.getCacheIdentifier() != null) {
            pageId = new PageId(cacheContext.getCacheIdentifier(), currentPage);
        } else {
            pageId = new PageId(Long.toString(mStatus.getFileId()), currentPage);
        }
        stopwatch.reset().start();
        int bytesRead = mCacheManager.get(pageId, currentPageOffset, bytesLeftInPage, b, off + totalBytesRead, mCacheContext);
        stopwatch.stop();
        if (bytesRead > 0) {
            totalBytesRead += bytesRead;
            currentPosition += bytesRead;
            MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getName()).mark(bytesRead);
            if (cacheContext != null) {
                cacheContext.incrementCounter(MetricKey.CLIENT_CACHE_BYTES_READ_CACHE.getMetricName(), bytesRead);
                cacheContext.incrementCounter(MetricKey.CLIENT_CACHE_PAGE_READ_CACHE_TIME_NS.getMetricName(), stopwatch.elapsed(TimeUnit.NANOSECONDS));
            }
        } else {
            // on local cache miss, read a complete page from external storage. This will always make
            // progress or throw an exception
            stopwatch.reset().start();
            byte[] page = readExternalPage(currentPosition, readType);
            stopwatch.stop();
            if (page.length > 0) {
                System.arraycopy(page, currentPageOffset, b, off + totalBytesRead, bytesLeftInPage);
                totalBytesRead += bytesLeftInPage;
                currentPosition += bytesLeftInPage;
                // cache misses
                MetricsSystem.meter(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getName()).mark(bytesLeftInPage);
                if (cacheContext != null) {
                    cacheContext.incrementCounter(MetricKey.CLIENT_CACHE_BYTES_REQUESTED_EXTERNAL.getMetricName(), bytesLeftInPage);
                    cacheContext.incrementCounter(MetricKey.CLIENT_CACHE_PAGE_READ_EXTERNAL_TIME_NS.getMetricName(), stopwatch.elapsed(TimeUnit.NANOSECONDS));
                }
                mCacheManager.put(pageId, page, mCacheContext);
            }
        }
        if (!isPositionedRead) {
            mPosition = currentPosition;
        }
    }
    if (totalBytesRead > len || (totalBytesRead < len && currentPosition < mStatus.getLength())) {
        throw new IOException(String.format("Invalid number of bytes read - " + "bytes to read = %d, actual bytes read = %d, bytes remains in file %d", len, totalBytesRead, remaining()));
    }
    return totalBytesRead;
}

Example 3 with CacheContext

use of alluxio.client.file.CacheContext in project alluxio by Alluxio.

the class LocalCacheManagerTest method putWithQuotaEviction.

@Test
public void putWithQuotaEviction() throws Exception {
    mConf.set(PropertyKey.USER_CLIENT_CACHE_QUOTA_ENABLED, true);
    CacheScope partitionCacheScope = CacheScope.create("schema.table.partition");
    CacheScope tableCacheScope = CacheScope.create("schema.table");
    CacheScope schemaCacheScope = CacheScope.create("schema");
    CacheScope[] quotaCacheScopes = { partitionCacheScope, tableCacheScope, schemaCacheScope, CacheScope.GLOBAL };
    for (CacheScope cacheScope : quotaCacheScopes) {
        mMetaStore = new QuotaMetaStore(mConf);
        mPageStore = PageStore.create(PageStoreOptions.create(mConf));
        mCacheManager = createLocalCacheManager(mConf, mMetaStore, mPageStore);
        CacheQuota quota = new CacheQuota(ImmutableMap.of(cacheScope.level(), (long) PAGE1.length + PAGE2.length - 1));
        CacheContext context = CacheContext.defaults().setCacheScope(partitionCacheScope).setCacheQuota(quota);
        assertTrue(mCacheManager.put(PAGE_ID1, PAGE1, context));
        assertEquals(PAGE1.length, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0));
        assertTrue(mCacheManager.put(PAGE_ID2, PAGE2, context));
        assertEquals(0, mCacheManager.get(PAGE_ID1, PAGE1.length, mBuf, 0));
        assertEquals(PAGE2.length, mCacheManager.get(PAGE_ID2, PAGE2.length, mBuf, 0));
    }
}

Example 4 with CacheContext

use of alluxio.client.file.CacheContext in project alluxio by Alluxio.

the class LocalCacheManagerWithMemPageStoreTest method putWithInsufficientQuota.

@Test
public void putWithInsufficientQuota() throws Exception {
    mConf.set(PropertyKey.USER_CLIENT_CACHE_QUOTA_ENABLED, true);
    mMetaStore = new QuotaMetaStore(mConf);
    mCacheManager = createLocalCacheManager(mConf, mMetaStore, mPageStore);
    CacheScope scope = CacheScope.create("schema.table.partition");
    CacheContext context = CacheContext.defaults().setCacheScope(scope);
    // insufficient partition quota
    assertFalse(mCacheManager.put(PAGE_ID1, PAGE1, context.setCacheQuota(new CacheQuota(ImmutableMap.of(CacheScope.Level.PARTITION, (long) PAGE1.length - 1)))));
    // insufficient table quota
    assertFalse(mCacheManager.put(PAGE_ID1, PAGE1, context.setCacheQuota(new CacheQuota(ImmutableMap.of(CacheScope.Level.TABLE, (long) PAGE1.length - 1)))));
    // insufficient schema quota
    assertFalse(mCacheManager.put(PAGE_ID1, PAGE1, context.setCacheQuota(new CacheQuota(ImmutableMap.of(CacheScope.Level.SCHEMA, (long) PAGE1.length - 1)))));
    // insufficient global quota
    assertFalse(mCacheManager.put(PAGE_ID1, PAGE1, context.setCacheQuota(new CacheQuota(ImmutableMap.of(CacheScope.Level.GLOBAL, (long) PAGE1.length - 1)))));
    // without quota
    assertTrue(mCacheManager.put(PAGE_ID1, PAGE1));
}

Example 5 with CacheContext

use of alluxio.client.file.CacheContext in project presto by prestodb.

the class AlluxioCachingFileSystem method initialize.

@Override
public synchronized void initialize(URI uri, Configuration configuration) throws IOException {
    this.localCacheFileSystem = new LocalCacheFileSystem(dataTier, uriStatus -> {
        // CacheContext is the mechanism to pass the hiveFileContext to the source filesystem
        // hiveFileContext is critical to use to open file.
        CacheContext cacheContext = uriStatus.getCacheContext();
        checkState(cacheContext instanceof PrestoCacheContext);
        HiveFileContext hiveFileContext = ((PrestoCacheContext) cacheContext).getHiveFileContext();
        try {
            return dataTier.openFile(new Path(uriStatus.getPath()), hiveFileContext);
        } catch (Exception e) {
            throw new IOException("Failed to open file", e);
        }
    });
    this.cacheQuotaEnabled = configuration.getBoolean(USER_CLIENT_CACHE_QUOTA_ENABLED.getName(), false);
    localCacheFileSystem.initialize(uri, configuration);
}