Examples with TestBlockInStream alluxio.client.block.stream.TestBlockInStream used on opensource projects

Example 1 with TestBlockInStream

use of alluxio.client.block.stream.TestBlockInStream in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method readBufferRetry.

@Test
public void readBufferRetry() throws Exception {
    TestBlockInStream workingStream = mInStreams.get(0);
    TestBlockInStream brokenStream = mock(TestBlockInStream.class);
    when(mBlockStore.getInStream(any(BlockInfo.class), any(InStreamOptions.class), any())).thenReturn(brokenStream).thenReturn(workingStream);
    when(brokenStream.read(any(ByteBuffer.class), anyInt(), anyInt())).thenThrow(new UnavailableException("test exception"));
    when(brokenStream.getPos()).thenReturn(BLOCK_LENGTH / 2);
    mTestStream.seek(BLOCK_LENGTH / 2);
    byte[] b = new byte[(int) BLOCK_LENGTH * 2];
    mTestStream.read(b, 0, b.length);
    doReturn(0).when(brokenStream).read(any(ByteBuffer.class), anyInt(), anyInt());
    verify(brokenStream, times(1)).read(any(ByteBuffer.class), anyInt(), anyInt());
    assertArrayEquals(BufferUtils.getIncreasingByteArray((int) BLOCK_LENGTH / 2, (int) BLOCK_LENGTH * 2), b);
}

Example 2 with TestBlockInStream

use of alluxio.client.block.stream.TestBlockInStream in project alluxio by Alluxio.

the class ReplicateDefinitionTest method runTaskNoBlockWorker.

@Test
public void runTaskNoBlockWorker() throws Exception {
    byte[] input = BufferUtils.getIncreasingByteArray(0, (int) TEST_BLOCK_SIZE);
    TestBlockInStream mockInStream = new TestBlockInStream(input, TEST_BLOCK_ID, input.length, false, BlockInStreamSource.NODE_LOCAL);
    TestBlockOutStream mockOutStream = new TestBlockOutStream(ByteBuffer.allocate(MAX_BYTES), TEST_BLOCK_SIZE);
    mThrown.expect(NotFoundException.class);
    mThrown.expectMessage(ExceptionMessage.NO_LOCAL_BLOCK_WORKER_LOAD_TASK.getMessage(TEST_BLOCK_ID));
    runTaskReplicateTestHelper(Lists.<BlockWorkerInfo>newArrayList(), mockInStream, mockOutStream);
}

Example 3 with TestBlockInStream

use of alluxio.client.block.stream.TestBlockInStream in project alluxio by Alluxio.

the class ReplicateDefinitionTest method runTaskLocalBlockWorkerDifferentFileStatus.

@Test
public void runTaskLocalBlockWorkerDifferentFileStatus() throws Exception {
    for (boolean persisted : new boolean[] { true, false }) {
        for (boolean pinned : new boolean[] { true, false }) {
            mTestStatus.getFileInfo().setPersisted(persisted).setMediumTypes(pinned ? Sets.newHashSet(Constants.MEDIUM_MEM) : Collections.emptySet());
            byte[] input = BufferUtils.getIncreasingByteArray(0, (int) TEST_BLOCK_SIZE);
            TestBlockInStream mockInStream = new TestBlockInStream(input, TEST_BLOCK_ID, input.length, false, BlockInStreamSource.NODE_LOCAL);
            TestBlockOutStream mockOutStream = new TestBlockOutStream(ByteBuffer.allocate(MAX_BYTES), TEST_BLOCK_SIZE);
            BlockWorkerInfo localBlockWorker = new BlockWorkerInfo(LOCAL_ADDRESS, TEST_BLOCK_SIZE, 0);
            runTaskReplicateTestHelper(Lists.newArrayList(localBlockWorker), mockInStream, mockOutStream);
            assertEquals(TEST_BLOCK_SIZE, mockInStream.getBytesRead());
            if (!persisted || pinned) {
                assertArrayEquals(String.format("input-output mismatched: pinned=%s, persisted=%s", pinned, persisted), input, mockOutStream.getWrittenData());
            }
        }
    }
}

Example 4 with TestBlockInStream

use of alluxio.client.block.stream.TestBlockInStream in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method blockInStreamOutOfSync.

/**
 * Tests that when the underlying blocks are inconsistent with the metadata in terms of block
 * length, an exception is thrown rather than client hanging indefinitely. This case may happen if
 * the file in Alluxio and UFS is out of sync.
 */
@Test
public void blockInStreamOutOfSync() throws Exception {
    when(mBlockStore.getInStream(any(BlockInfo.class), any(InStreamOptions.class), any())).thenAnswer(new Answer<BlockInStream>() {

        @Override
        public BlockInStream answer(InvocationOnMock invocation) throws Throwable {
            return new TestBlockInStream(new byte[1], 0, BLOCK_LENGTH, false, mBlockSource);
        }
    });
    byte[] buffer = new byte[(int) BLOCK_LENGTH];
    try {
        mTestStream.read(buffer, 0, (int) BLOCK_LENGTH);
        fail("BlockInStream is inconsistent, an Exception is expected");
    } catch (IllegalStateException e) {
    // expect an exception to throw
    }
}

Example 5 with TestBlockInStream

use of alluxio.client.block.stream.TestBlockInStream in project alluxio by Alluxio.

the class AlluxioFileInStreamTest method before.

/**
 * Sets up the context and streams before a test runs.
 */
@Before
public void before() throws Exception {
    mInfo = new FileInfo().setBlockSizeBytes(BLOCK_LENGTH).setLength(mFileSize);
    ClientTestUtils.setSmallBufferSizes(mConf);
    mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MIN, "1ms");
    mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_SLEEP_MAX, "5ms");
    mConf.set(PropertyKey.USER_BLOCK_READ_RETRY_MAX_DURATION, "1s");
    BlockWorkerClient client = mock(BlockWorkerClient.class);
    doNothing().when(client).cache(any());
    mContext = mock(FileSystemContext.class);
    when(mContext.getClientContext()).thenReturn(ClientContext.create(mConf));
    when(mContext.getClusterConf()).thenReturn(mConf);
    when(mContext.getPathConf(any(AlluxioURI.class))).thenReturn(mConf);
    when(mContext.getNodeLocalWorker()).thenReturn(new WorkerNetAddress());
    when(mContext.getCachedWorkers()).thenReturn(new ArrayList<>());
    when(mContext.acquireBlockWorkerClient(any())).thenReturn(new CloseableResource<BlockWorkerClient>(client) {

        @Override
        public void closeResource() {
        }
    });
    mBlockStore = mock(AlluxioBlockStore.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mContext)).thenReturn(mBlockStore);
    // Set up BufferedBlockInStreams and caching streams
    mInStreams = new ArrayList<>();
    List<Long> blockIds = new ArrayList<>();
    List<FileBlockInfo> fileBlockInfos = new ArrayList<>();
    for (int i = 0; i < mNumBlocks; i++) {
        blockIds.add((long) i);
        FileBlockInfo fbInfo = new FileBlockInfo().setBlockInfo(new BlockInfo().setBlockId(i));
        fileBlockInfos.add(fbInfo);
        final byte[] input = BufferUtils.getIncreasingByteArray((int) (i * BLOCK_LENGTH), (int) getBlockLength(i));
        mInStreams.add(new TestBlockInStream(input, i, input.length, false, mBlockSource));
        when(mContext.getCachedWorkers()).thenReturn(Arrays.asList(new BlockWorkerInfo(new WorkerNetAddress(), 0, 0)));
        when(mBlockStore.getInStream(eq((long) i), any(InStreamOptions.class), any())).thenAnswer(invocation -> {
            long blockId = (Long) invocation.getArguments()[0];
            return mInStreams.get((int) blockId).isClosed() ? new TestBlockInStream(input, blockId, input.length, false, mBlockSource) : mInStreams.get((int) blockId);
        });
        when(mBlockStore.getInStream(eq(new BlockInfo().setBlockId(i)), any(InStreamOptions.class), any())).thenAnswer(invocation -> {
            long blockId = ((BlockInfo) invocation.getArguments()[0]).getBlockId();
            return mInStreams.get((int) blockId).isClosed() ? new TestBlockInStream(input, blockId, input.length, false, mBlockSource) : mInStreams.get((int) blockId);
        });
    }
    mInfo.setBlockIds(blockIds);
    mInfo.setFileBlockInfos(fileBlockInfos).setReplicationMax(1);
    mStatus = new URIStatus(mInfo);
    OpenFilePOptions readOptions = OpenFilePOptions.newBuilder().setReadType(ReadPType.CACHE_PROMOTE).build();
    mTestStream = new AlluxioFileInStream(mStatus, new InStreamOptions(mStatus, readOptions, mConf), mContext);
}