Examples with OpenFilePOptions alluxio.grpc.OpenFilePOptions used on opensource projects

Example 21 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method seekBackwardToFileBeginning.

/**
 * Tests seeking with incomplete block caching enabled. It seeks forward for more than a block
 * and then seek to the file beginning.
 */
@Test
public void seekBackwardToFileBeginning() throws IOException {
    OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
    int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH);
    // Seek forward.
    mTestStream.seek(seekAmount);
    // Block 1 is partially cached though it is not fully read.
    validatePartialCaching(1, 0);
    // Seek backward.
    mTestStream.seek(0);
    // Block 1 is fully cached though it is not fully read.
    validatePartialCaching(1, 0);
    mTestStream.close();
    // block 0 is cached
    validatePartialCaching(0, 0);
}

Example 22 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamUfsLocalFirst.

@Test
public void getInStreamUfsLocalFirst() throws Exception {
    WorkerNetAddress remote = new WorkerNetAddress().setHost("remote");
    WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    BlockInfo info = new BlockInfo().setBlockId(0);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().build();
    InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
    when(mContext.getCachedWorkers()).thenReturn(Lists.newArrayList(new BlockWorkerInfo(remote, 100, 0), new BlockWorkerInfo(local, 100, 0)));
    BlockInStream stream = mBlockStore.getInStream(BLOCK_ID, options);
    assertEquals(local, stream.getAddress());
    assertEquals(GrpcDataReader.Factory.class.getName(), stream.getDataReaderFactory().getClass().getName());
}

Example 23 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamUfsProcessLocal.

@Test
public void getInStreamUfsProcessLocal() throws Exception {
    WorkerNetAddress remote = new WorkerNetAddress().setHost("remote");
    WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    BlockInfo info = new BlockInfo().setBlockId(0);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().build();
    InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
    when(mContext.getCachedWorkers()).thenReturn(Lists.newArrayList(new BlockWorkerInfo(remote, 100, 0), new BlockWorkerInfo(local, 100, 0)));
    when(mContext.getNodeLocalWorker()).thenReturn(local);
    when(mContext.hasProcessLocalWorker()).thenReturn(true);
    BlockWorker blockWorker = Mockito.mock(BlockWorker.class);
    when(mContext.getProcessLocalWorker()).thenReturn(blockWorker);
    BlockInStream stream = mBlockStore.getInStream(BLOCK_ID, options);
    assertEquals(local, stream.getAddress());
    assertEquals(BlockWorkerDataReader.Factory.class.getName(), stream.getDataReaderFactory().getClass().getName());
}

Example 24 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class LoadCommand method runLoadTask.

private void runLoadTask(AlluxioURI filePath, URIStatus status, boolean local) throws IOException {
    AlluxioConfiguration conf = mFsContext.getPathConf(filePath);
    OpenFilePOptions options = FileSystemOptions.openFileDefaults(conf);
    BlockLocationPolicy policy = Preconditions.checkNotNull(BlockLocationPolicy.Factory.create(conf.getString(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY), conf), "UFS read location policy Required when loading files");
    WorkerNetAddress dataSource;
    List<Long> blockIds = status.getBlockIds();
    for (long blockId : blockIds) {
        if (local) {
            dataSource = mFsContext.getNodeLocalWorker();
        } else {
            // send request to data source
            AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext);
            Pair<WorkerNetAddress, BlockInStream.BlockInStreamSource> dataSourceAndType = blockStore.getDataSourceAndType(status.getBlockInfo(blockId), status, policy, ImmutableMap.of());
            dataSource = dataSourceAndType.getFirst();
        }
        Protocol.OpenUfsBlockOptions openUfsBlockOptions = new InStreamOptions(status, options, conf).getOpenUfsBlockOptions(blockId);
        cacheBlock(blockId, dataSource, status, openUfsBlockOptions);
    }
}

Example 25 with OpenFilePOptions

use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.

the class CacheRequestManagerTest method before.

/**
 * Sets up all dependencies before a test runs.
 */
@Before
public void before() throws IOException {
    BlockMasterClient blockMasterClient = mock(BlockMasterClient.class);
    BlockMasterClientPool blockMasterClientPool = spy(new BlockMasterClientPool());
    when(blockMasterClientPool.createNewResource()).thenReturn(blockMasterClient);
    TieredBlockStore blockStore = new TieredBlockStore();
    FileSystemMasterClient fileSystemMasterClient = mock(FileSystemMasterClient.class);
    Sessions sessions = mock(Sessions.class);
    // Connect to the real UFS for testing
    UfsManager ufsManager = mock(UfsManager.class);
    mRootUfs = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    UfsManager.UfsClient ufsClient = new UfsManager.UfsClient(() -> UnderFileSystem.Factory.create(mRootUfs, UnderFileSystemConfiguration.defaults(ServerConfiguration.global())), new AlluxioURI(mRootUfs));
    when(ufsManager.get(anyLong())).thenReturn(ufsClient);
    mBlockWorker = spy(new DefaultBlockWorker(blockMasterClientPool, fileSystemMasterClient, sessions, blockStore, ufsManager));
    FileSystemContext context = mock(FileSystemContext.class);
    mCacheRequestManager = spy(new CacheRequestManager(GrpcExecutors.CACHE_MANAGER_EXECUTOR, mBlockWorker, context));
    // Write an actual file to UFS
    String testFilePath = File.createTempFile("temp", null, new File(mRootUfs)).getAbsolutePath();
    byte[] buffer = BufferUtils.getIncreasingByteArray(CHUNK_SIZE);
    BufferUtils.writeBufferToFile(testFilePath, buffer);
    // create options
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLength(CHUNK_SIZE);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setUfsPath(testFilePath).setBlockIds(Collections.singletonList(BLOCK_ID)).setLength(CHUNK_SIZE).setBlockSizeBytes(CHUNK_SIZE).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    OpenFilePOptions readOptions = FileSystemOptions.openFileDefaults(mConf);
    InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, mConf);
    mOpenUfsBlockOptions = options.getOpenUfsBlockOptions(BLOCK_ID);
}