Examples with InStreamOptions alluxio.client.file.options.InStreamOptions used on opensource projects

Example 21 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class RemoteReadIntegrationTest method readTest6.

/**
 * Tests the batch read API with offset and length from a remote location when the data is in an
 * Alluxio worker.
 */
@Test
public void readTest6() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteAlluxio, k);
        URIStatus status = mFileSystem.getStatus(uri);
        InStreamOptions options = new InStreamOptions(status, ServerConfiguration.global());
        long blockId = status.getBlockIds().get(0);
        BlockInfo info = AlluxioBlockStore.create(FileSystemContext.create(ServerConfiguration.global())).getInfo(blockId);
        WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress();
        BlockInStream is = BlockInStream.create(mFsContext, options.getBlockInfo(blockId), workerAddr, BlockInStreamSource.REMOTE, options);
        byte[] ret = new byte[k / 2];
        int read = 0;
        while (read < k / 2) {
            read += is.read(ret, read, k / 2 - read);
        }
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(read, ret));
        is.close();
        FileSystemUtils.waitForAlluxioPercentage(mFileSystem, uri, 100);
    }
}

Example 22 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class RemoteReadIntegrationTest method remoteReadLock.

/**
 * Tests remote reads lock blocks correctly.
 */
@Test
public void remoteReadLock() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteAlluxio, k);
        URIStatus status = mFileSystem.getStatus(uri);
        InStreamOptions options = new InStreamOptions(status, ServerConfiguration.global());
        long blockId = status.getBlockIds().get(0);
        BlockInfo info = AlluxioBlockStore.create(FileSystemContext.create(ServerConfiguration.global())).getInfo(blockId);
        WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress();
        BlockInStream is = BlockInStream.create(mFsContext, options.getBlockInfo(blockId), workerAddr, BlockInStreamSource.REMOTE, options);
        Assert.assertEquals(0, is.read());
        mFileSystem.delete(uri);
        // The file has been deleted.
        Assert.assertFalse(mFileSystem.exists(uri));
        // Look! We can still read the deleted file since we have a lock!
        byte[] ret = new byte[k / 2];
        Assert.assertTrue(is.read(ret, 0, k / 2) > 0);
        is.close();
        Assert.assertFalse(mFileSystem.exists(uri));
        // Try to create an in stream again, and it should fail.
        BlockInStream is2 = null;
        try {
            is2 = BlockInStream.create(mFsContext, options.getBlockInfo(blockId), workerAddr, BlockInStreamSource.REMOTE, options);
        } catch (NotFoundException e) {
        // Expected since the file has been deleted.
        } finally {
            if (is2 != null) {
                is2.close();
            }
        }
    }
}

Example 23 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method shortSeekBackwardCachingPartiallyReadBlocks.

/**
 * Tests seeking with incomplete block caching enabled. It seeks backward within 1 block.
 */
@Test
public void shortSeekBackwardCachingPartiallyReadBlocks() throws IOException {
    OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
    int seekAmount = (int) (BLOCK_LENGTH / 4);
    int readAmount = (int) (BLOCK_LENGTH * 2 - BLOCK_LENGTH / 2);
    byte[] buffer = new byte[readAmount];
    mTestStream.read(buffer);
    // Seek backward.
    mTestStream.seek(readAmount - seekAmount);
    // Block 1 is cached though it is not fully read.
    validatePartialCaching(1, (int) BLOCK_LENGTH / 2);
    // Seek many times. It will cache block 1 only once.
    for (int i = 0; i <= seekAmount; i++) {
        mTestStream.seek(readAmount - seekAmount - i);
    }
    validatePartialCaching(1, (int) BLOCK_LENGTH / 2);
}

Example 24 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method longSeekForwardCachingPartiallyReadBlocks.

/**
 * Tests seeking with incomplete block caching enabled. It seeks forward for more than a block.
 */
@Test
public void longSeekForwardCachingPartiallyReadBlocks() throws IOException {
    OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
    int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH);
    int readAmount = (int) (BLOCK_LENGTH / 2);
    byte[] buffer = new byte[readAmount];
    mTestStream.read(buffer);
    // Seek backward.
    mTestStream.seek(readAmount + seekAmount);
    // Block 0 is cached though it is not fully read.
    validatePartialCaching(0, readAmount);
    // Block 1 is being cached though its prefix it not read.
    validatePartialCaching(1, 0);
    mTestStream.close();
    validatePartialCaching(1, 0);
}

Example 25 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method before.

/**
 * Sets up the context and streams before a test runs.
 */
@Before
public void before() throws Exception {
    mInfo = new FileInfo().setBlockSizeBytes(BLOCK_LENGTH).setLength(mFileSize);
    ClientTestUtils.setSmallBufferSizes(mConf);
    mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MIN, "1ms");
    mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MAX, "5ms");
    mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_MAX_DURATION, "1s");
    BlockWorkerClient client = mock(BlockWorkerClient.class);
    doNothing().when(client).cache(any());
    mContext = mock(FileSystemContext.class);
    when(mContext.getClientContext()).thenReturn(ClientContext.create(mConf));
    when(mContext.getClusterConf()).thenReturn(mConf);
    when(mContext.getPathConf(any(AlluxioURI.class))).thenReturn(mConf);
    when(mContext.getNodeLocalWorker()).thenReturn(new WorkerNetAddress());
    when(mContext.getCachedWorkers()).thenReturn(new ArrayList<>());
    when(mContext.acquireBlockWorkerClient(any())).thenReturn(new CloseableResource<BlockWorkerClient>(client) {

        @Override
        public void closeResource() {
        }
    });
    mBlockStore = mock(AlluxioBlockStore.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mContext)).thenReturn(mBlockStore);
    // Set up BufferedBlockInStreams and caching streams
    mInStreams = new ArrayList<>();
    List<Long> blockIds = new ArrayList<>();
    List<FileBlockInfo> fileBlockInfos = new ArrayList<>();
    for (int i = 0; i < mNumBlocks; i++) {
        blockIds.add((long) i);
        FileBlockInfo fbInfo = new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(i));
        fileBlockInfos.add(fbInfo);
        final byte[] input = BufferUtils.getIncreasingByteArray((int) (i * BLOCK_LENGTH), (int) getBlockLength(i));
        mInStreams.add(new TestBlockInStream(input, i, input.length, false, mBlockSource));
        when(mContext.getCachedWorkers()).thenReturn(Arrays.asList(new BlockWorkerInfo(new WorkerNetAddress(), 0, 0)));
        when(mBlockStore.getInStream(eq((long) i), any(InStreamOptions.class), any())).thenAnswer(invocation -> {
            long blockId = (Long) invocation.getArguments()[0];
            return mInStreams.get((int) blockId).isClosed() ? new TestBlockInStream(input, blockId, input.length, false, mBlockSource) : mInStreams.get((int) blockId);
        });
        when(mBlockStore.getInStream(eq(new BlockInfo().setBlockId(i)), any(InStreamOptions.class), any())).thenAnswer(invocation -> {
            long blockId = ((BlockInfo) invocation.getArguments()[0]).getBlockId();
            return mInStreams.get((int) blockId).isClosed() ? new TestBlockInStream(input, blockId, input.length, false, mBlockSource) : mInStreams.get((int) blockId);
        });
    }
    mInfo.setBlockIds(blockIds);
    mInfo.setFileBlockInfos(fileBlockInfos).setReplicationMax(1);
    mStatus = new URIStatus(mInfo);
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, readOptions, mConf), mContext);
}