use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.
the class AlluxioFileInStreamTest method longSeekForwardCachingPartiallyReadBlocks.
/**
* Tests seeking with incomplete block caching enabled. It seeks forward for more than a block.
*/
@Test
public void longSeekForwardCachingPartiallyReadBlocks() throws IOException {
OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH);
int readAmount = (int) (BLOCK_LENGTH / 2);
byte[] buffer = new byte[readAmount];
mTestStream.read(buffer);
// Seek backward.
mTestStream.seek(readAmount + seekAmount);
// Block 0 is cached though it is not fully read.
validatePartialCaching(0, readAmount);
// Block 1 is being cached though its prefix it not read.
validatePartialCaching(1, 0);
mTestStream.close();
validatePartialCaching(1, 0);
}
use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.
the class AlluxioFileInStreamTest method before.
/**
* Sets up the context and streams before a test runs.
*/
@Before
public void before() throws Exception {
mInfo = new FileInfo().setBlockSizeBytes(BLOCK_LENGTH).setLength(mFileSize);
ClientTestUtils.setSmallBufferSizes(mConf);
mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MIN, "1ms");
mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MAX, "5ms");
mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_MAX_DURATION, "1s");
BlockWorkerClient client = mock(BlockWorkerClient.class);
doNothing().when(client).cache(any());
mContext = mock(FileSystemContext.class);
when(mContext.getClientContext()).thenReturn(ClientContext.create(mConf));
when(mContext.getClusterConf()).thenReturn(mConf);
when(mContext.getPathConf(any(AlluxioURI.class))).thenReturn(mConf);
when(mContext.getNodeLocalWorker()).thenReturn(new WorkerNetAddress());
when(mContext.getCachedWorkers()).thenReturn(new ArrayList<>());
when(mContext.acquireBlockWorkerClient(any())).thenReturn(new CloseableResource<BlockWorkerClient>(client) {
@Override
public void closeResource() {
}
});
mBlockStore = mock(AlluxioBlockStore.class);
PowerMockito.mockStatic(AlluxioBlockStore.class);
PowerMockito.when(AlluxioBlockStore.create(mContext)).thenReturn(mBlockStore);
// Set up BufferedBlockInStreams and caching streams
mInStreams = new ArrayList<>();
List<Long> blockIds = new ArrayList<>();
List<FileBlockInfo> fileBlockInfos = new ArrayList<>();
for (int i = 0; i < mNumBlocks; i++) {
blockIds.add((long) i);
FileBlockInfo fbInfo = new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(i));
fileBlockInfos.add(fbInfo);
final byte[] input = BufferUtils.getIncreasingByteArray((int) (i * BLOCK_LENGTH), (int) getBlockLength(i));
mInStreams.add(new TestBlockInStream(input, i, input.length, false, mBlockSource));
when(mContext.getCachedWorkers()).thenReturn(Arrays.asList(new BlockWorkerInfo(new WorkerNetAddress(), 0, 0)));
when(mBlockStore.getInStream(eq((long) i), any(InStreamOptions.class), any())).thenAnswer(invocation -> {
long blockId = (Long) invocation.getArguments()[0];
return mInStreams.get((int) blockId).isClosed() ? new TestBlockInStream(input, blockId, input.length, false, mBlockSource) : mInStreams.get((int) blockId);
});
when(mBlockStore.getInStream(eq(new BlockInfo().setBlockId(i)), any(InStreamOptions.class), any())).thenAnswer(invocation -> {
long blockId = ((BlockInfo) invocation.getArguments()[0]).getBlockId();
return mInStreams.get((int) blockId).isClosed() ? new TestBlockInStream(input, blockId, input.length, false, mBlockSource) : mInStreams.get((int) blockId);
});
}
mInfo.setBlockIds(blockIds);
mInfo.setFileBlockInfos(fileBlockInfos).setReplicationMax(1);
mStatus = new URIStatus(mInfo);
OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, readOptions, mConf), mContext);
}
use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.
the class AlluxioFileInStreamTest method longSeekBackwardCachingPartiallyReadBlocks.
/**
* Tests seeking with incomplete block caching enabled. It seeks backward for more than a block.
*/
@Test
public void longSeekBackwardCachingPartiallyReadBlocks() throws IOException {
OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
int seekAmount = (int) (BLOCK_LENGTH / 4 + BLOCK_LENGTH);
int readAmount = (int) (BLOCK_LENGTH * 3 - BLOCK_LENGTH / 2);
byte[] buffer = new byte[readAmount];
mTestStream.read(buffer);
// Seek backward.
mTestStream.seek(readAmount - seekAmount);
// Block 2 is cached though it is not fully read.
validatePartialCaching(2, (int) BLOCK_LENGTH / 2);
}
use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.
the class AlluxioFileInStreamTest method seekBackwardSmallSeekBuffer.
/**
* Tests skipping backwards when the seek buffer size is smaller than block size.
*/
@Test
public void seekBackwardSmallSeekBuffer() throws IOException {
OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
int readAmount = (int) (BLOCK_LENGTH / 2);
byte[] buffer = new byte[readAmount];
mTestStream.read(buffer);
mTestStream.seek(readAmount - 1);
validatePartialCaching(0, readAmount);
}
use of alluxio.grpc.OpenFilePOptions in project alluxio by Alluxio.
the class AlluxioFileInStreamTest method seekAndClose.
@Test
public void seekAndClose() throws IOException {
OpenFilePOptions options = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, options, mConf), mContext);
int seekAmount = (int) (BLOCK_LENGTH / 2);
mTestStream.seek(seekAmount);
mTestStream.close();
// Block 0 is cached though it is not fully read.
validatePartialCaching(0, 0);
}
Aggregations