Examples with BufferedBlockOutStream alluxio.client.block.BufferedBlockOutStream used on opensource projects

Example 1 with BufferedBlockOutStream

use of alluxio.client.block.BufferedBlockOutStream in project alluxio by Alluxio.

the class FileOutStreamTest method before.

/**
   * Sets up the different contexts and clients before a test runs.
   */
@Before
public void before() throws Exception {
    GroupMappingServiceTestUtils.resetCache();
    ClientTestUtils.setSmallBufferSizes();
    // PowerMock enums and final classes
    mFileSystemContext = PowerMockito.mock(FileSystemContext.class);
    mBlockStore = PowerMockito.mock(AlluxioBlockStore.class);
    mFileSystemMasterClient = PowerMockito.mock(FileSystemMasterClient.class);
    mFactory = PowerMockito.mock(UnderFileSystemFileOutStream.Factory.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mFileSystemContext)).thenReturn(mBlockStore);
    when(mFileSystemContext.acquireMasterClientResource()).thenReturn(new DummyCloseableResource<>(mFileSystemMasterClient));
    when(mFileSystemMasterClient.getStatus(any(AlluxioURI.class))).thenReturn(new URIStatus(new FileInfo()));
    // Worker file client mocking
    mWorkerClient = PowerMockito.mock(FileSystemWorkerClient.class);
    when(mFileSystemContext.createFileSystemWorkerClient()).thenReturn(mWorkerClient);
    when(mWorkerClient.createUfsFile(any(AlluxioURI.class), any(CreateUfsFileOptions.class))).thenReturn(UFS_FILE_ID);
    // Return sequentially increasing numbers for new block ids
    when(mFileSystemMasterClient.getNewBlockIdForFile(FILE_NAME)).thenAnswer(new Answer<Long>() {

        private long mCount = 0;

        @Override
        public Long answer(InvocationOnMock invocation) throws Throwable {
            return mCount++;
        }
    });
    // Set up out streams. When they are created, add them to outStreamMap
    final Map<Long, TestBufferedBlockOutStream> outStreamMap = new HashMap<>();
    when(mBlockStore.getOutStream(anyLong(), eq(BLOCK_LENGTH), any(OutStreamOptions.class))).thenAnswer(new Answer<BufferedBlockOutStream>() {

        @Override
        public BufferedBlockOutStream answer(InvocationOnMock invocation) throws Throwable {
            Long blockId = invocation.getArgumentAt(0, Long.class);
            if (!outStreamMap.containsKey(blockId)) {
                TestBufferedBlockOutStream newStream = new TestBufferedBlockOutStream(blockId, BLOCK_LENGTH, mFileSystemContext);
                outStreamMap.put(blockId, newStream);
            }
            return outStreamMap.get(blockId);
        }
    });
    BlockWorkerInfo workerInfo = new BlockWorkerInfo(new WorkerNetAddress().setHost("localhost").setRpcPort(1).setDataPort(2).setWebPort(3), Constants.GB, 0);
    when(mBlockStore.getWorkerInfoList()).thenReturn(Lists.newArrayList(workerInfo));
    mAlluxioOutStreamMap = outStreamMap;
    // Create an under storage stream so that we can check whether it has been flushed
    final AtomicBoolean underStorageFlushed = new AtomicBoolean(false);
    mUnderStorageOutputStream = new ByteArrayOutputStream() {

        @Override
        public void flush() {
            underStorageFlushed.set(true);
        }
    };
    mUnderStorageFlushed = underStorageFlushed;
    when(mFactory.create(any(FileSystemContext.class), any(InetSocketAddress.class), anyLong())).thenReturn(mUnderStorageOutputStream);
    // Set up underFileStorage so that we can test UnderStorageType.SYNC_PERSIST
    mUnderFileSystem = ClientMockUtils.mockUnderFileSystem();
    when(mUnderFileSystem.create(anyString())).thenReturn(mUnderStorageOutputStream);
    when(mUnderFileSystem.create(anyString(), any(CreateOptions.class))).thenReturn(mUnderStorageOutputStream);
    when(mUnderFileSystem.isDirectory(anyString())).thenReturn(true);
    OutStreamOptions options = OutStreamOptions.defaults().setBlockSizeBytes(BLOCK_LENGTH).setWriteType(WriteType.CACHE_THROUGH).setUfsPath(FILE_NAME.getPath());
    mTestStream = createTestStream(FILE_NAME, options);
}

Example 2 with BufferedBlockOutStream

use of alluxio.client.block.BufferedBlockOutStream in project alluxio by Alluxio.

the class FileOutStreamTest method cacheWriteExceptionNonSyncPersist.

/**
   * Tests that if an exception is thrown by the underlying out stream, and the user is using
   * {@link UnderStorageType#NO_PERSIST} for their under storage type, the correct exception
   * message will be thrown.
   */
@Test
public void cacheWriteExceptionNonSyncPersist() throws IOException {
    OutStreamOptions options = OutStreamOptions.defaults().setBlockSizeBytes(BLOCK_LENGTH).setWriteType(WriteType.MUST_CACHE);
    BufferedBlockOutStream stream = mock(BufferedBlockOutStream.class);
    when(mBlockStore.getOutStream(anyInt(), anyLong(), any(OutStreamOptions.class))).thenReturn(stream);
    mTestStream = createTestStream(FILE_NAME, options);
    when(stream.remaining()).thenReturn(BLOCK_LENGTH);
    doThrow(new IOException("test error")).when(stream).write((byte) 7);
    try {
        mTestStream.write(7);
        Assert.fail("the test should fail");
    } catch (IOException e) {
        Assert.assertEquals(ExceptionMessage.FAILED_CACHE.getMessage("test error"), e.getMessage());
    }
}

Example 3 with BufferedBlockOutStream

use of alluxio.client.block.BufferedBlockOutStream in project alluxio by Alluxio.

the class FileInStreamTest method before.

/**
   * Sets up the context and streams before a test runs.
   *
   * @throws AlluxioException when the worker ufs operations fail
   * @throws IOException when the read and write streams fail
   */
@Before
public void before() throws Exception {
    mInfo = new FileInfo().setBlockSizeBytes(BLOCK_LENGTH).setLength(FILE_LENGTH);
    mDelegateUfsOps = Configuration.getBoolean(PropertyKey.USER_UFS_DELEGATION_ENABLED);
    ClientTestUtils.setSmallBufferSizes();
    mContext = PowerMockito.mock(FileSystemContext.class);
    mBlockStore = Mockito.mock(AlluxioBlockStore.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mContext)).thenReturn(mBlockStore);
    PowerMockito.when(mBlockStore.getWorkerInfoList()).thenReturn(new ArrayList<BlockWorkerInfo>());
    Mockito.mock(StreamFactory.class);
    PowerMockito.mockStatic(StreamFactory.class);
    // Set up BufferedBlockInStreams and caching streams
    mCacheStreams = new ArrayList<>();
    List<Long> blockIds = new ArrayList<>();
    for (int i = 0; i < NUM_STREAMS; i++) {
        blockIds.add((long) i);
        mCacheStreams.add(new TestBufferedBlockOutStream(i, getBlockLength(i), mContext));
        Mockito.when(mBlockStore.getInStream(Mockito.eq((long) i), Mockito.any(InStreamOptions.class))).thenAnswer(new Answer<BufferedBlockInStream>() {

            @Override
            public BufferedBlockInStream answer(InvocationOnMock invocation) throws Throwable {
                long i = (Long) invocation.getArguments()[0];
                byte[] input = BufferUtils.getIncreasingByteArray((int) (i * BLOCK_LENGTH), (int) getBlockLength((int) i));
                return new TestBufferedBlockInStream(i, input);
            }
        });
        Mockito.when(mBlockStore.getOutStream(Mockito.eq((long) i), Mockito.anyLong(), Mockito.any(WorkerNetAddress.class), Mockito.any(OutStreamOptions.class))).thenAnswer(new Answer<BufferedBlockOutStream>() {

            @Override
            public BufferedBlockOutStream answer(InvocationOnMock invocation) throws Throwable {
                long i = (Long) invocation.getArguments()[0];
                return mCacheStreams.get((int) i).isClosed() ? null : mCacheStreams.get((int) i);
            }
        });
    }
    mInfo.setBlockIds(blockIds);
    mStatus = new URIStatus(mInfo);
    mTestStream = new FileInStream(mStatus, InStreamOptions.defaults().setReadType(ReadType.CACHE_PROMOTE).setCachePartiallyReadBlock(false), mContext);
}

Example 4 with BufferedBlockOutStream

use of alluxio.client.block.BufferedBlockOutStream in project alluxio by Alluxio.

the class FileOutStreamTest method cacheWriteExceptionSyncPersist.

/**
   * Tests that if an exception is thrown by the underlying out stream, and the user is using
   * {@link UnderStorageType#SYNC_PERSIST} for their under storage type, the error is recovered
   * from by writing the data to the under storage out stream.
   */
@Test
public void cacheWriteExceptionSyncPersist() throws IOException {
    BufferedBlockOutStream stream = mock(BufferedBlockOutStream.class);
    when(mBlockStore.getOutStream(anyLong(), anyLong(), any(OutStreamOptions.class))).thenReturn(stream);
    when(stream.remaining()).thenReturn(BLOCK_LENGTH);
    doThrow(new IOException("test error")).when(stream).write((byte) 7);
    mTestStream.write(7);
    mTestStream.write(8);
    Assert.assertArrayEquals(new byte[] { 7, 8 }, mUnderStorageOutputStream.toByteArray());
    // The cache stream is written to only once - the FileInStream gives up on it after it throws
    // the first exception.
    verify(stream, times(1)).write(anyByte());
}