Examples with OpenFilePOptions alluxio.grpc.OpenFilePOptions used on opensource projects

Example 1 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamUfsMockLocaltion.

@Test
public void getInStreamUfsMockLocaltion() throws Exception {
    try (Closeable c = new ConfigurationRule(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY, MockBlockLocationPolicy.class.getTypeName(), sConf).toResource()) {
        WorkerNetAddress worker1 = new WorkerNetAddress().setHost("worker1");
        WorkerNetAddress worker2 = new WorkerNetAddress().setHost("worker2");
        BlockInfo info = new BlockInfo().setBlockId(0);
        URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
        OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().build();
        InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf);
        ((MockBlockLocationPolicy) options.getUfsReadLocationPolicy()).setHosts(Arrays.asList(worker1, worker2));
        when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
        when(mContext.getCachedWorkers()).thenReturn(Lists.newArrayList(new BlockWorkerInfo(worker1, -1, -1), new BlockWorkerInfo(worker2, -1, -1)));
        // Location policy chooses worker1 first.
        assertEquals(worker1, mBlockStore.getInStream(BLOCK_ID, options).getAddress());
        // Location policy chooses worker2 second.
        assertEquals(worker2, mBlockStore.getInStream(BLOCK_ID, options).getAddress());
    }
}

Example 2 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class BaseFileSystem method openFile.

@Override
public FileInStream openFile(URIStatus status, OpenFilePOptions options) throws FileDoesNotExistException, OpenDirectoryException, FileIncompleteException, IOException, AlluxioException {
    AlluxioURI path = new AlluxioURI(status.getPath());
    if (status.isFolder()) {
        throw new OpenDirectoryException(path);
    }
    if (!status.isCompleted()) {
        throw new FileIncompleteException(path);
    }
    AlluxioConfiguration conf = mFsContext.getPathConf(path);
    OpenFilePOptions mergedOptions = FileSystemOptions.openFileDefaults(conf).toBuilder().mergeFrom(options).build();
    InStreamOptions inStreamOptions = new InStreamOptions(status, mergedOptions, conf);
    return new AlluxioFileInStream(status, inStreamOptions, mFsContext);
}

Example 3 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method testSeekWithNoLocalWorker.

/**
 * Tests reading and seeking with no local worker. Nothing should be cached.
 */
@Test
public void testSeekWithNoLocalWorker() throws IOException {
    // Overrides the get local worker call
    when(mContext.getNodeLocalWorker()).thenReturn(null);
    OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
    int readAmount = (int) (BLOCK_LENGTH / 2);
    byte[] buffer = new byte[readAmount];
    // read and seek several times
    mTestStream.read(buffer);
    assertEquals(readAmount, mInStreams.get(0).getBytesRead());
    mTestStream.seek(BLOCK_LENGTH + BLOCK_LENGTH / 2);
    mTestStream.seek(0);
    // only reads the read amount, regardless of block source
    assertEquals(readAmount, mInStreams.get(0).getBytesRead());
    assertEquals(0, mInStreams.get(1).getBytesRead());
}

Example 4 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method triggerAsyncOnClose.

// See https://github.com/Alluxio/alluxio/issues/13828
@Test
public void triggerAsyncOnClose() throws Exception {
    assumeTrue(mBlockSource == BlockInStreamSource.UFS);
    mInfo.setReplicationMax(1);
    mStatus = new URIStatus(mInfo);
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, readOptions, mConf), mContext);
    mTestStream.read(new byte[(int) mFileSize], 0, (int) mFileSize);
    assertEquals(mFileSize, mTestStream.getPos());
    assertTrue(mTestStream.triggerAsyncCaching(mInStreams.get(mInStreams.size() - 1)));
}

Example 5 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method shortSeekForwardCachingPartiallyReadBlocks.

/**
 * Tests seeking with incomplete block caching enabled. It seeks forward within a block.
 */
@Test
public void shortSeekForwardCachingPartiallyReadBlocks() throws IOException {
    OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
    int seekAmount = (int) (BLOCK_LENGTH / 4);
    int readAmount = (int) (BLOCK_LENGTH * 2 - BLOCK_LENGTH / 2);
    byte[] buffer = new byte[readAmount];
    mTestStream.read(buffer);
    // Seek backward.
    mTestStream.seek(readAmount + seekAmount);
    // Block 1 (till seek pos) is being cached.
    validatePartialCaching(1, (int) BLOCK_LENGTH / 2);
    // Seek forward many times. The prefix is always cached.
    for (int i = 0; i < seekAmount; i++) {
        mTestStream.seek(readAmount + seekAmount + i);
        validatePartialCaching(1, (int) BLOCK_LENGTH / 2);
    }
}