Examples with BlockOutStream alluxio.client.block.stream.BlockOutStream used on opensource projects

Example 1 with BlockOutStream

use of alluxio.client.block.stream.BlockOutStream in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getOutStreamLocal.

@Test
public void getOutStreamLocal() throws Exception {
    File file = File.createTempFile("test", ".tmp");
    CreateLocalBlockResponse response = CreateLocalBlockResponse.newBuilder().setPath(file.getAbsolutePath()).build();
    when(mWorkerClient.createLocalBlock(any(StreamObserver.class))).thenAnswer(new Answer() {

        public Object answer(InvocationOnMock invocation) {
            StreamObserver<CreateLocalBlockResponse> observer = invocation.getArgument(0, StreamObserver.class);
            observer.onNext(response);
            return mStreamObserver;
        }
    });
    OutStreamOptions options = OutStreamOptions.defaults(mClientContext).setBlockSizeBytes(BLOCK_LENGTH).setLocationPolicy(new MockBlockLocationPolicy(Lists.newArrayList(WORKER_NET_ADDRESS_LOCAL))).setWriteType(WriteType.MUST_CACHE);
    BlockOutStream stream = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options);
    assertEquals(WORKER_NET_ADDRESS_LOCAL, stream.getAddress());
}

Example 2 with BlockOutStream

use of alluxio.client.block.stream.BlockOutStream in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getOutStreamWithReplicated.

@Test
public void getOutStreamWithReplicated() throws Exception {
    File file = File.createTempFile("test", ".tmp");
    CreateLocalBlockResponse response = CreateLocalBlockResponse.newBuilder().setPath(file.getAbsolutePath()).build();
    when(mWorkerClient.createLocalBlock(any(StreamObserver.class))).thenAnswer(new Answer() {

        public Object answer(InvocationOnMock invocation) {
            StreamObserver<CreateLocalBlockResponse> observer = invocation.getArgument(0, StreamObserver.class);
            observer.onNext(response);
            return mStreamObserver;
        }
    });
    when(mContext.getCachedWorkers()).thenReturn(Lists.newArrayList(new BlockWorkerInfo(WORKER_NET_ADDRESS_LOCAL, -1, -1), new BlockWorkerInfo(WORKER_NET_ADDRESS_REMOTE, -1, -1)));
    OutStreamOptions options = OutStreamOptions.defaults(mClientContext).setBlockSizeBytes(BLOCK_LENGTH).setLocationPolicy(new MockBlockLocationPolicy(Lists.newArrayList(WORKER_NET_ADDRESS_LOCAL, WORKER_NET_ADDRESS_REMOTE))).setWriteType(WriteType.MUST_CACHE).setReplicationMin(2);
    BlockOutStream stream = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options);
    assertEquals(alluxio.client.block.stream.BlockOutStream.class, stream.getClass());
}

Example 3 with BlockOutStream

use of alluxio.client.block.stream.BlockOutStream in project alluxio by Alluxio.

the class FileOutStreamTest method cacheWriteExceptionNonSyncPersist.

/**
 * Tests that if an exception is thrown by the underlying out stream, and the user is using
 * {@link UnderStorageType#NO_PERSIST} for their under storage type, the correct exception
 * message will be thrown.
 */
@Test
public void cacheWriteExceptionNonSyncPersist() throws IOException {
    OutStreamOptions options = OutStreamOptions.defaults(mClientContext).setBlockSizeBytes(BLOCK_LENGTH).setWriteType(WriteType.MUST_CACHE);
    BlockOutStream stream = mock(BlockOutStream.class);
    when(mBlockStore.getOutStream(anyLong(), anyLong(), any(OutStreamOptions.class))).thenReturn(stream);
    mTestStream = createTestStream(FILE_NAME, options);
    when(stream.remaining()).thenReturn(BLOCK_LENGTH);
    doThrow(new IOException("test error")).when(stream).write(7);
    try {
        mTestStream.write(7);
        fail("the test should fail");
    } catch (IOException e) {
        assertEquals(ExceptionMessage.FAILED_CACHE.getMessage("test error"), e.getMessage());
    }
}

Example 4 with BlockOutStream

use of alluxio.client.block.stream.BlockOutStream in project alluxio by Alluxio.

the class ReplicateDefinitionTest method runTaskInputIOException.

@Test
public void runTaskInputIOException() throws Exception {
    // file is pinned on a medium
    mTestStatus.getFileInfo().setMediumTypes(Sets.newHashSet(Constants.MEDIUM_MEM));
    BlockInStream mockInStream = mock(BlockInStream.class);
    BlockOutStream mockOutStream = mock(BlockOutStream.class);
    BlockWorkerInfo localBlockWorker = new BlockWorkerInfo(LOCAL_ADDRESS, TEST_BLOCK_SIZE, 0);
    doThrow(new IOException("test")).when(mockInStream).read(any(byte[].class), anyInt(), anyInt());
    doThrow(new IOException("test")).when(mockInStream).read(any(byte[].class));
    try {
        runTaskReplicateTestHelper(Lists.newArrayList(localBlockWorker), mockInStream, mockOutStream);
        fail("Expected the task to throw and IOException");
    } catch (IOException e) {
        assertEquals("test", e.getMessage());
    }
    verify(mockOutStream).cancel();
}

Example 5 with BlockOutStream

use of alluxio.client.block.stream.BlockOutStream in project alluxio by Alluxio.

the class AlluxioFileOutStream method close.

@Override
public void close() throws IOException {
    if (mClosed) {
        return;
    }
    try {
        if (mCurrentBlockOutStream != null) {
            mPreviousBlockOutStreams.add(mCurrentBlockOutStream);
        }
        CompleteFilePOptions.Builder optionsBuilder = CompleteFilePOptions.newBuilder();
        optionsBuilder.setCommonOptions(FileSystemMasterCommonPOptions.newBuilder().setOperationId(new OperationId(UUID.randomUUID()).toFsProto()).buildPartial());
        if (mUnderStorageType.isSyncPersist()) {
            if (mCanceled) {
                mUnderStorageOutputStream.cancel();
            } else {
                mUnderStorageOutputStream.close();
                optionsBuilder.setUfsLength(mBytesWritten);
            }
        }
        if (mAlluxioStorageType.isStore()) {
            if (mCanceled) {
                for (BlockOutStream bos : mPreviousBlockOutStreams) {
                    bos.cancel();
                }
            } else {
                // write(blockN), and blockN evicts the committed blockN-1 and causing file lost.
                if (mCurrentBlockOutStream != null) {
                    mCurrentBlockOutStream.close();
                }
                for (BlockOutStream bos : mPreviousBlockOutStreams) {
                    bos.close();
                }
            }
        }
        // Whether to complete file with async persist request.
        if (!mCanceled && mUnderStorageType.isAsyncPersist() && mOptions.getPersistenceWaitTime() != Constants.NO_AUTO_PERSIST) {
            optionsBuilder.setAsyncPersistOptions(FileSystemOptions.scheduleAsyncPersistDefaults(mContext.getPathConf(mUri)).toBuilder().setCommonOptions(mOptions.getCommonOptions()).setPersistenceWaitTime(mOptions.getPersistenceWaitTime()));
        }
        // Complete the file if it's ready to be completed.
        if (!mCanceled && (mUnderStorageType.isSyncPersist() || mAlluxioStorageType.isStore())) {
            try (CloseableResource<FileSystemMasterClient> masterClient = mContext.acquireMasterClientResource()) {
                masterClient.get().completeFile(mUri, optionsBuilder.build());
            }
        }
    } catch (Throwable e) {
        // IOException will be thrown as-is.
        throw mCloser.rethrow(e);
    } finally {
        mClosed = true;
        mCloser.close();
    }
}