Examples with OutStreamOptions alluxio.client.file.options.OutStreamOptions used on opensource projects

Example 1 with OutStreamOptions

use of alluxio.client.file.options.OutStreamOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getOutStreamLocal.

@Test
public void getOutStreamLocal() throws Exception {
    File tmp = mTestFolder.newFile();
    Mockito.when(mBlockWorkerClient.requestBlockLocation(Matchers.eq(BLOCK_ID), Matchers.anyLong(), Matchers.anyInt())).thenReturn(tmp.getAbsolutePath());
    OutStreamOptions options = OutStreamOptions.defaults().setBlockSizeBytes(BLOCK_LENGTH).setLocationPolicy(new MockFileWriteLocationPolicy(Lists.newArrayList(WORKER_NET_ADDRESS_LOCAL))).setWriteType(WriteType.MUST_CACHE);
    OutputStream stream = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options);
    if (Configuration.getBoolean(PropertyKey.USER_PACKET_STREAMING_ENABLED)) {
        Assert.assertEquals(alluxio.client.block.stream.BlockOutStream.class, stream.getClass());
    } else {
        Assert.assertEquals(LocalBlockOutStream.class, stream.getClass());
    }
}

Example 2 with OutStreamOptions

use of alluxio.client.file.options.OutStreamOptions in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getOutStreamUsingLocationPolicy.

@Test
public void getOutStreamUsingLocationPolicy() throws Exception {
    OutStreamOptions options = OutStreamOptions.defaults().setWriteType(WriteType.MUST_CACHE).setLocationPolicy(new FileWriteLocationPolicy() {

        @Override
        public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) {
            throw new RuntimeException("policy threw exception");
        }
    });
    try {
        mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options);
        Assert.fail("An exception should have been thrown");
    } catch (Exception e) {
        Assert.assertEquals("policy threw exception", e.getMessage());
    }
}

Example 3 with OutStreamOptions

use of alluxio.client.file.options.OutStreamOptions in project alluxio by Alluxio.

the class FileOutStreamTest method before.

/**
   * Sets up the different contexts and clients before a test runs.
   */
@Before
public void before() throws Exception {
    GroupMappingServiceTestUtils.resetCache();
    ClientTestUtils.setSmallBufferSizes();
    // PowerMock enums and final classes
    mFileSystemContext = PowerMockito.mock(FileSystemContext.class);
    mBlockStore = PowerMockito.mock(AlluxioBlockStore.class);
    mFileSystemMasterClient = PowerMockito.mock(FileSystemMasterClient.class);
    mFactory = PowerMockito.mock(UnderFileSystemFileOutStream.Factory.class);
    PowerMockito.mockStatic(AlluxioBlockStore.class);
    PowerMockito.when(AlluxioBlockStore.create(mFileSystemContext)).thenReturn(mBlockStore);
    when(mFileSystemContext.acquireMasterClientResource()).thenReturn(new DummyCloseableResource<>(mFileSystemMasterClient));
    when(mFileSystemMasterClient.getStatus(any(AlluxioURI.class))).thenReturn(new URIStatus(new FileInfo()));
    // Worker file client mocking
    mWorkerClient = PowerMockito.mock(FileSystemWorkerClient.class);
    when(mFileSystemContext.createFileSystemWorkerClient()).thenReturn(mWorkerClient);
    when(mWorkerClient.createUfsFile(any(AlluxioURI.class), any(CreateUfsFileOptions.class))).thenReturn(UFS_FILE_ID);
    // Return sequentially increasing numbers for new block ids
    when(mFileSystemMasterClient.getNewBlockIdForFile(FILE_NAME)).thenAnswer(new Answer<Long>() {

        private long mCount = 0;

        @Override
        public Long answer(InvocationOnMock invocation) throws Throwable {
            return mCount++;
        }
    });
    // Set up out streams. When they are created, add them to outStreamMap
    final Map<Long, TestBufferedBlockOutStream> outStreamMap = new HashMap<>();
    when(mBlockStore.getOutStream(anyLong(), eq(BLOCK_LENGTH), any(OutStreamOptions.class))).thenAnswer(new Answer<BufferedBlockOutStream>() {

        @Override
        public BufferedBlockOutStream answer(InvocationOnMock invocation) throws Throwable {
            Long blockId = invocation.getArgumentAt(0, Long.class);
            if (!outStreamMap.containsKey(blockId)) {
                TestBufferedBlockOutStream newStream = new TestBufferedBlockOutStream(blockId, BLOCK_LENGTH, mFileSystemContext);
                outStreamMap.put(blockId, newStream);
            }
            return outStreamMap.get(blockId);
        }
    });
    BlockWorkerInfo workerInfo = new BlockWorkerInfo(new WorkerNetAddress().setHost("localhost").setRpcPort(1).setDataPort(2).setWebPort(3), Constants.GB, 0);
    when(mBlockStore.getWorkerInfoList()).thenReturn(Lists.newArrayList(workerInfo));
    mAlluxioOutStreamMap = outStreamMap;
    // Create an under storage stream so that we can check whether it has been flushed
    final AtomicBoolean underStorageFlushed = new AtomicBoolean(false);
    mUnderStorageOutputStream = new ByteArrayOutputStream() {

        @Override
        public void flush() {
            underStorageFlushed.set(true);
        }
    };
    mUnderStorageFlushed = underStorageFlushed;
    when(mFactory.create(any(FileSystemContext.class), any(InetSocketAddress.class), anyLong())).thenReturn(mUnderStorageOutputStream);
    // Set up underFileStorage so that we can test UnderStorageType.SYNC_PERSIST
    mUnderFileSystem = ClientMockUtils.mockUnderFileSystem();
    when(mUnderFileSystem.create(anyString())).thenReturn(mUnderStorageOutputStream);
    when(mUnderFileSystem.create(anyString(), any(CreateOptions.class))).thenReturn(mUnderStorageOutputStream);
    when(mUnderFileSystem.isDirectory(anyString())).thenReturn(true);
    OutStreamOptions options = OutStreamOptions.defaults().setBlockSizeBytes(BLOCK_LENGTH).setWriteType(WriteType.CACHE_THROUGH).setUfsPath(FILE_NAME.getPath());
    mTestStream = createTestStream(FILE_NAME, options);
}

Example 4 with OutStreamOptions

use of alluxio.client.file.options.OutStreamOptions in project alluxio by Alluxio.

the class FileOutStreamTest method cacheWriteExceptionNonSyncPersist.

/**
   * Tests that if an exception is thrown by the underlying out stream, and the user is using
   * {@link UnderStorageType#NO_PERSIST} for their under storage type, the correct exception
   * message will be thrown.
   */
@Test
public void cacheWriteExceptionNonSyncPersist() throws IOException {
    OutStreamOptions options = OutStreamOptions.defaults().setBlockSizeBytes(BLOCK_LENGTH).setWriteType(WriteType.MUST_CACHE);
    BufferedBlockOutStream stream = mock(BufferedBlockOutStream.class);
    when(mBlockStore.getOutStream(anyInt(), anyLong(), any(OutStreamOptions.class))).thenReturn(stream);
    mTestStream = createTestStream(FILE_NAME, options);
    when(stream.remaining()).thenReturn(BLOCK_LENGTH);
    doThrow(new IOException("test error")).when(stream).write((byte) 7);
    try {
        mTestStream.write(7);
        Assert.fail("the test should fail");
    } catch (IOException e) {
        Assert.assertEquals(ExceptionMessage.FAILED_CACHE.getMessage("test error"), e.getMessage());
    }
}

Example 5 with OutStreamOptions

use of alluxio.client.file.options.OutStreamOptions in project alluxio by Alluxio.

the class FileOutStreamTest method asyncWrite.

/**
   * Tests that the async write invokes the expected client APIs.
   */
@Test
public void asyncWrite() throws Exception {
    OutStreamOptions options = OutStreamOptions.defaults().setBlockSizeBytes(BLOCK_LENGTH).setWriteType(WriteType.ASYNC_THROUGH);
    mTestStream = createTestStream(FILE_NAME, options);
    when(mUnderFileSystem.renameFile(anyString(), anyString())).thenReturn(true);
    mTestStream.write(BufferUtils.getIncreasingByteArray((int) (BLOCK_LENGTH * 1.5)));
    mTestStream.close();
    verify(mFileSystemMasterClient).completeFile(eq(FILE_NAME), any(CompleteFileOptions.class));
    verify(mFileSystemMasterClient).scheduleAsyncPersist(eq(FILE_NAME));
}