Examples with InStreamOptions alluxio.client.file.options.InStreamOptions used on opensource projects

Example 11 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method shortSeekForwardCachingPartiallyReadBlocks.

/**
 * Tests seeking with incomplete block caching enabled. It seeks forward within a block.
 */
@Test
public void shortSeekForwardCachingPartiallyReadBlocks() throws IOException {
    OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
    int seekAmount = (int) (BLOCK_LENGTH / 4);
    int readAmount = (int) (BLOCK_LENGTH * 2 - BLOCK_LENGTH / 2);
    byte[] buffer = new byte[readAmount];
    mTestStream.read(buffer);
    // Seek backward.
    mTestStream.seek(readAmount + seekAmount);
    // Block 1 (till seek pos) is being cached.
    validatePartialCaching(1, (int) BLOCK_LENGTH / 2);
    // Seek forward many times. The prefix is always cached.
    for (int i = 0; i < seekAmount; i++) {
        mTestStream.seek(readAmount + seekAmount + i);
        validatePartialCaching(1, (int) BLOCK_LENGTH / 2);
    }
}

Example 12 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class RemoteReadIntegrationTest method readTest5.

/**
 * Tests the batch read API from a remote location when the data is only in an Alluxio worker.
 */
@Test
public void readTest5() throws Exception {
    String uniqPath = PathUtils.uniqPath();
    for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
        AlluxioURI uri = new AlluxioURI(uniqPath + "/file_" + k);
        FileSystemTestUtils.createByteFile(mFileSystem, uri, mWriteAlluxio, k);
        URIStatus status = mFileSystem.getStatus(uri);
        InStreamOptions options = new InStreamOptions(status, ServerConfiguration.global());
        long blockId = status.getBlockIds().get(0);
        BlockInfo info = AlluxioBlockStore.create(FileSystemContext.create(ServerConfiguration.global())).getInfo(blockId);
        WorkerNetAddress workerAddr = info.getLocations().get(0).getWorkerAddress();
        BlockInStream is = BlockInStream.create(mFsContext, options.getBlockInfo(blockId), workerAddr, BlockInStreamSource.REMOTE, options);
        byte[] ret = new byte[k];
        int read = is.read(ret);
        Assert.assertTrue(BufferUtils.equalIncreasingByteArray(read, Arrays.copyOfRange(ret, 0, read)));
        is.close();
        FileSystemUtils.waitForAlluxioPercentage(mFileSystem, uri, 100);
    }
}

Example 13 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class MultiWorkerIntegrationTest method replicateFileBlocks.

private void replicateFileBlocks(AlluxioURI filePath) throws Exception {
    FileSystemContext fsContext = FileSystemContext.create(ServerConfiguration.global());
    AlluxioBlockStore store = AlluxioBlockStore.create(fsContext);
    URIStatus status = mResource.get().getClient().getStatus(filePath);
    List<FileBlockInfo> blocks = status.getFileBlockInfos();
    List<BlockWorkerInfo> workers = fsContext.getCachedWorkers();
    for (FileBlockInfo block : blocks) {
        BlockInfo blockInfo = block.getBlockInfo();
        WorkerNetAddress src = blockInfo.getLocations().get(0).getWorkerAddress();
        WorkerNetAddress dest = workers.stream().filter(candidate -> !candidate.getNetAddress().equals(src)).findFirst().get().getNetAddress();
        try (OutputStream outStream = store.getOutStream(blockInfo.getBlockId(), blockInfo.getLength(), dest, OutStreamOptions.defaults(fsContext.getClientContext()).setBlockSizeBytes(8 * Constants.MB).setWriteType(WriteType.MUST_CACHE))) {
            try (InputStream inStream = store.getInStream(blockInfo.getBlockId(), new InStreamOptions(status, ServerConfiguration.global()))) {
                ByteStreams.copy(inStream, outStream);
            }
        }
    }
}

Example 14 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class BlockWorkerDataReaderTest method before.

@Before
public void before() throws Exception {
    BlockMasterClient blockMasterClient = mock(BlockMasterClient.class);
    BlockMasterClientPool blockMasterClientPool = spy(new BlockMasterClientPool());
    when(blockMasterClientPool.createNewResource()).thenReturn(blockMasterClient);
    TieredBlockStore blockStore = new TieredBlockStore();
    FileSystemMasterClient fileSystemMasterClient = mock(FileSystemMasterClient.class);
    Sessions sessions = mock(Sessions.class);
    // Connect to the real UFS for UFS read testing
    UfsManager ufsManager = mock(UfsManager.class);
    mRootUfs = ServerConfiguration.getString(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS);
    UfsManager.UfsClient ufsClient = new UfsManager.UfsClient(() -> UnderFileSystem.Factory.create(mRootUfs, UnderFileSystemConfiguration.defaults(ServerConfiguration.global())), new AlluxioURI(mRootUfs));
    when(ufsManager.get(anyLong())).thenReturn(ufsClient);
    mBlockWorker = new DefaultBlockWorker(blockMasterClientPool, fileSystemMasterClient, sessions, blockStore, ufsManager);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setBlockIds(Collections.singletonList(BLOCK_ID)));
    InStreamOptions options = new InStreamOptions(dummyStatus, FileSystemOptions.openFileDefaults(mConf), mConf);
    mDataReaderFactory = new BlockWorkerDataReader.Factory(mBlockWorker, BLOCK_ID, CHUNK_SIZE, options);
}

Example 15 with InStreamOptions

use of alluxio.client.file.options.InStreamOptions in project alluxio by Alluxio.

the class BlockWorkerDataReaderTest method readChunkUfs.

@Test
public void readChunkUfs() throws Exception {
    // Write an actual file to UFS
    String testFilePath = File.createTempFile("temp", null, new File(mRootUfs)).getAbsolutePath();
    byte[] buffer = BufferUtils.getIncreasingByteArray(CHUNK_SIZE * 5);
    BufferUtils.writeBufferToFile(testFilePath, buffer);
    BlockInfo info = new BlockInfo().setBlockId(BLOCK_ID).setLength(CHUNK_SIZE * 5);
    URIStatus dummyStatus = new URIStatus(new FileInfo().setPersisted(true).setUfsPath(testFilePath).setBlockIds(Collections.singletonList(BLOCK_ID)).setLength(CHUNK_SIZE * 5).setBlockSizeBytes(CHUNK_SIZE).setFileBlockInfos(Collections.singletonList(new FileBlockInfo().setBlockInfo(info))));
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.NO_CACHE).build();
    InStreamOptions options = new InStreamOptions(dummyStatus, readOptions, mConf);
    BlockWorkerDataReader.Factory factory = new BlockWorkerDataReader.Factory(mBlockWorker, BLOCK_ID, CHUNK_SIZE, options);
    int len = CHUNK_SIZE * 3 / 2;
    try (DataReader dataReader = factory.create(0, len)) {
        validateBuffer(dataReader.readChunk(), 0, CHUNK_SIZE);
        assertEquals(CHUNK_SIZE, dataReader.pos());
        validateBuffer(dataReader.readChunk(), CHUNK_SIZE, len - CHUNK_SIZE);
        assertEquals(len, dataReader.pos());
    }
}