Examples with InStreamOptions alluxio.client.file.options.InStreamOptions used on opensource projects

Example 6 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method testGetInStreamFallback.

private void testGetInStreamFallback(int workerCount, boolean isPersisted, int[] blockLocations, Map<Integer, Long> failedWorkers, int expectedWorker) throws Exception {
    WorkerNetAddress[] workers = new WorkerNetAddress[workerCount];
    Arrays.setAll(workers, i -> new WorkerNetAddress().setHost(String.format("worker-%d", i)));
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLocations(Arrays.stream(blockLocations).mapToObj(x -> new BlockLocation().setWorkerAddress(workers[x])).collect(Collectors.toList()));
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(isPersisted).setBlockIds(Collections.singletonList(BLOCK_ID)).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    BlockLocationPolicy mockPolicy = mock(BlockLocationPolicy.class);
    when(mockPolicy.getWorker(any())).thenAnswer(arg -> arg.getArgument(0, GetWorkerOptions.class).getBlockWorkerInfos().iterator().next().getNetAddress());
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(sConf), sConf);
    options.setUfsReadLocationPolicy(mockPolicy);
    when(mMasterClient.getBlockInfo(BLOCK_ID)).thenReturn(info);
    when(mContext.getCachedWorkers()).thenReturn(Arrays.stream(workers).map(x -> new BlockWorkerInfo(x, -1, -1)).collect((Collectors.toList())));
    Map<WorkerNetAddress, Long> failedWorkerAddresses = failedWorkers.entrySet().stream().map(x -> new AbstractMap.SimpleImmutableEntry<>(workers[x.getKey()], x.getValue())).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
    BlockInStream inStream = mBlockStore.getInStream(BLOCK_ID, options, failedWorkerAddresses);
    assertEquals(workers[expectedWorker], inStream.getAddress());
}

Example 7 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class BlockInStreamTest method before.

@Before
public void before() throws Exception {
    BlockWorkerClient workerClient = Mockito.mock(BlockWorkerClient.class);
    ClientCallStreamObserver requestObserver = Mockito.mock(ClientCallStreamObserver.class);
    when(requestObserver.isReady()).thenReturn(true);
    when(workerClient.openLocalBlock(any(StreamObserver.class))).thenAnswer(new Answer() {

        public Object answer(InvocationOnMock invocation) {
            mResponseObserver = invocation.getArgument(0, StreamObserver.class);
            return requestObserver;
        }
    });
    doAnswer(invocation -> {
        mResponseObserver.onNext(OpenLocalBlockResponse.newBuilder().setPath("/tmp").build());
        mResponseObserver.onCompleted();
        return null;
    }).when(requestObserver).onNext(any(OpenLocalBlockRequest.class));
    mMockContext = Mockito.mock(FileSystemContext.class);
    when(mMockContext.acquireBlockWorkerClient(ArgumentMatchers.any(WorkerNetAddress.class))).thenReturn(new NoopClosableResource<>(workerClient));
    when(mMockContext.getClientContext()).thenReturn(ClientContext.create(mConf));
    when(mMockContext.getClusterConf()).thenReturn(mConf);
    mInfo = new BlockInfo().setBlockId(1);
    mOptions = new InStreamOptions(new URIStatus(new FileInfo().setBlockIds(Collections.singletonList(1L))), mConf);
}

Example 8 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class BaseFileSystem method openFile.

@Override
public FileInStream openFile(URIStatus status, OpenFilePOptions options) throws FileDoesNotExistException, OpenDirectoryException, FileIncompleteException, IOException, AlluxioException {
    AlluxioURI path = new AlluxioURI(status.getPath());
    if (status.isFolder()) {
        throw new OpenDirectoryException(path);
    }
    if (!status.isCompleted()) {
        throw new FileIncompleteException(path);
    }
    AlluxioConfiguration conf = mFsContext.getPathConf(path);
    OpenFilePOptions mergedOptions = FileSystemOptions.openFileDefaults(conf).toBuilder().mergeFrom(options).build();
    InStreamOptions inStreamOptions = new InStreamOptions(status, mergedOptions, conf);
    return new AlluxioFileInStream(status, inStreamOptions, mFsContext);
}

Example 9 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method testSeekWithNoLocalWorker.

/**
 * Tests reading and seeking with no local worker. Nothing should be cached.
 */
@Test
public void testSeekWithNoLocalWorker() throws IOException {
    // Overrides the get local worker call
    when(mContext.getNodeLocalWorker()).thenReturn(null);
    OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
    int readAmount = (int) (BLOCK_LENGTH / 2);
    byte[] buffer = new byte[readAmount];
    // read and seek several times
    mTestStream.read(buffer);
    assertEquals(readAmount, mInStreams.get(0).getBytesRead());
    mTestStream.seek(BLOCK_LENGTH + BLOCK_LENGTH / 2);
    mTestStream.seek(0);
    // only reads the read amount, regardless of block source
    assertEquals(readAmount, mInStreams.get(0).getBytesRead());
    assertEquals(0, mInStreams.get(1).getBytesRead());
}

Example 10 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method triggerAsyncOnClose.

// See https://github.com/Alluxio/alluxio/issues/13828
@Test
public void triggerAsyncOnClose() throws Exception {
    assumeTrue(mBlockSource == BlockInStreamSource.UFS);
    mInfo.setReplicationMax(1);
    mStatus = new URIStatus(mInfo);
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, readOptions, mConf), mContext);
    mTestStream.read(new byte[(int) mFileSize], 0, (int) mFileSize);
    assertEquals(mFileSize, mTestStream.getPos());
    assertTrue(mTestStream.triggerAsyncCaching(mInStreams.get(mInStreams.size() - 1)));
}