Examples with InStreamOptions alluxio.client.file.options.InStreamOptions used on opensource projects

Example 31 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamNoWorkers.

@Test
public void getInStreamNoWorkers() throws Exception {
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)));
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(sConf), sConf);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
    when(mContext.getCachedWorkers()).thenReturn(Collections.emptyList());
    Exception e = assertThrows(UnavailableException.class, () -> mBlockStore.getInStream(BLOCK_ID, options).getAddress());
    assertTrue(e.getMessage().contains(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage()));
}

Example 32 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamUfsProcessLocal.

@Test
public void getInStreamUfsProcessLocal() throws Exception {
    WorkerNetAddress remote = new WorkerNetAddress().setHost("remote");
    WorkerNetAddress local = new WorkerNetAddress().setHost(WORKER_HOSTNAME_LOCAL);
    BlockInfo info = new BlockInfo().setBlockId(0);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setBlockIds(Collections.singletonList(0L)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().build();
    InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, sConf);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(new BlockInfo());
    when(mContext.getCachedWorkers()).thenReturn(Lists.newArrayList(new BlockWorkerInfo(remote, 100, 0), new BlockWorkerInfo(local, 100, 0)));
    when(mContext.getNodeLocalWorker()).thenReturn(local);
    when(mContext.hasProcessLocalWorker()).thenReturn(true);
    BlockWorker blockWorker = Mockito.mock(BlockWorker.class);
    when(mContext.getProcessLocalWorker()).thenReturn(blockWorker);
    BlockInStream stream = mBlockStore.getInStream(BLOCK_ID, options);
    assertEquals(local, stream.getAddress());
    assertEquals(BlockWorkerDataReader.Factory.class.getName(), stream.getDataReaderFactory().getClass().getName());
}

Example 33 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getInStreamRemote.

@Test
public void getInStreamRemote() throws Exception {
    WorkerNetAddress remote1 = new WorkerNetAddress().setHost("remote1");
    WorkerNetAddress remote2 = new WorkerNetAddress().setHost("remote2");
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.asList(new BlockLocation().setWorkerAddress(remote1), new BlockLocation().setWorkerAddress(remote2)));
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    // We should sometimes get remote1 and sometimes get remote2.
    Set<WorkerNetAddress> results = new HashSet<>();
    for (int i = 0; i < 40; i++) {
        results.add(mBlockStore.getInStream(BLOCK_ID, new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Lists.newArrayList(BLOCK_ID))), sConf)).getAddress());
    }
    assertEquals(Sets.newHashSet(remote1, remote2), results);
}

Example 34 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class LoadCommand method runLoadTask.

private void runLoadTask(AlluxioURI filePath, URIStatus status, boolean local) throws IOException {
    AlluxioConfiguration conf = mFsContext.getPathConf(filePath);
    OpenFilePOptions options = FileSystemOptions.openFileDefaults(conf);
    BlockLocationPolicy policy = Preconditions.checkNotNull(BlockLocationPolicy.Factory.create(conf.getString(PropertyKey.USER_UFS_BLOCK_READ_LOCATION_POLICY), conf), "UFS read location policy Required when loading files");
    WorkerNetAddress dataSource;
    List<Long> blockIds = status.getBlockIds();
    for (long blockId : blockIds) {
        if (local) {
            dataSource = mFsContext.getNodeLocalWorker();
        } else {
            // send request to data source
            AlluxioBlockStore blockStore = AlluxioBlockStore.create(mFsContext);
            Pair<WorkerNetAddress, BlockInStream.BlockInStreamSource> dataSourceAndType = blockStore.getDataSourceAndType(status.getBlockInfo(blockId), status, policy, ImmutableMap.of());
            dataSource = dataSourceAndType.getFirst();
        }
        Protocol.OpenUfsBlockOptions openUfsBlockOptions = new InStreamOptions(status, options, conf).getOpenUfsBlockOptions(blockId);
        cacheBlock(blockId, dataSource, status, openUfsBlockOptions);
    }
}

Example 35 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class CacheRequestManagerTest method before.

/**
 * Sets up all dependencies before a test runs.
 */
@Before
public void before() throws IOException {
    BlockMasterClient blockMasterClient = mock(BlockMasterClient.class);
    BlockMasterClientPool blockMasterClientPool = spy(new BlockMasterClientPool());
    when(blockMasterClientPool.createNewResource()).thenReturn(blockMasterClient);
    TieredBlockStore blockStore = new TieredBlockStore();
    FileSystemMasterClient fileSystemMasterClient = mock(FileSystemMasterClient.class);
    Sessions sessions = mock(Sessions.class);
    // Connect to the real UFS for testing
    UfsManager ufsManager = mock(UfsManager.class);
    mRootUfs = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    UfsManager.UfsClient ufsClient = new UfsManager.UfsClient(() -> UnderFileSystem.Factory.create(mRootUfs, UnderFileSystemConfiguration.defaults(ServerConfiguration.global())), new AlluxioURI(mRootUfs));
    when(ufsManager.get(anyLong())).thenReturn(ufsClient);
    mBlockWorker = spy(new DefaultBlockWorker(blockMasterClientPool, fileSystemMasterClient, sessions, blockStore, ufsManager));
    FileSystemContext context = mock(FileSystemContext.class);
    mCacheRequestManager = spy(new CacheRequestManager(GrpcExecutors.CACHE_MANAGER_EXECUTOR, mBlockWorker, context));
    // Write an actual file to UFS
    String testFilePath = File.createTempFile("temp", null, new File(mRootUfs)).getAbsolutePath();
    byte[] buffer = BufferUtils.getIncreasingByteArray(CHUNK_SIZE);
    BufferUtils.writeBufferToFile(testFilePath, buffer);
    // create options
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLength(CHUNK_SIZE);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setUfsPath(testFilePath).setBlockIds(Collections.singletonList(BLOCK_ID)).setLength(CHUNK_SIZE).setBlockSizeBytes(CHUNK_SIZE).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    OpenFilePOptions readOptions = FileSystemOptions.openFileDefaults(mConf);
    InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, mConf);
    mOpenUfsBlockOptions = options.getOpenUfsBlockOptions(BLOCK_ID);
}