Examples with BlockWriter alluxio.worker.block.io.BlockWriter used on opensource projects

Example 11 with BlockWriter

use of alluxio.worker.block.io.BlockWriter in project alluxio by Alluxio.

the class DefaultBlockWorkerTest method getTempBlockWriter.

@Test
public void getTempBlockWriter() throws Exception {
    long blockId = mRandom.nextLong();
    long sessionId = mRandom.nextLong();
    mBlockWorker.createBlock(sessionId, blockId, 0, Constants.MEDIUM_MEM, 1);
    try (BlockWriter blockWriter = mBlockWorker.createBlockWriter(sessionId, blockId)) {
        blockWriter.append(BufferUtils.getIncreasingByteBuffer(10));
        TempBlockMeta meta = mBlockWorker.getTempBlockMeta(sessionId, blockId);
        assertEquals(Constants.MEDIUM_MEM, meta.getBlockLocation().mediumType());
    }
    mBlockWorker.abortBlock(sessionId, blockId);
}

Example 12 with BlockWriter

use of alluxio.worker.block.io.BlockWriter in project alluxio by Alluxio.

the class BlockWriteHandlerTest method before.

@Before
public void before() throws Exception {
    mFile = mTestFolder.newFile();
    mBlockWorker = new NoopBlockWorker() {

        @Override
        public BlockWriter createBlockWriter(long sessionId, long blockId) {
            return mBlockWriter;
        }
    };
    mBlockWriter = new LocalFileBlockWriter(mFile.getPath());
    mResponseObserver = Mockito.mock(StreamObserver.class);
    mWriteHandler = new BlockWriteHandler(mBlockWorker, mResponseObserver, mUserInfo, false);
    setupResponseTrigger();
}

Example 13 with BlockWriter

use of alluxio.worker.block.io.BlockWriter in project alluxio by Alluxio.

the class UfsFallbackBlockWriteHandlerTest method before.

@Before
public void before() throws Exception {
    mFile = mTestFolder.newFile();
    mOutputStream = new FileOutputStream(mFile);
    mBlockStore = new TieredBlockStore();
    mBlockWorker = new MockBlockWorker();
    UnderFileSystem mockUfs = Mockito.mock(UnderFileSystem.class);
    UfsManager ufsManager = Mockito.mock(UfsManager.class);
    UfsManager.UfsClient ufsClient = new UfsManager.UfsClient(() -> mockUfs, AlluxioURI.EMPTY_URI);
    Mockito.when(ufsManager.get(Mockito.anyLong())).thenReturn(ufsClient);
    Mockito.when(mockUfs.createNonexistingFile(Mockito.anyString(), Mockito.any(CreateOptions.class))).thenReturn(mOutputStream).thenReturn(new FileOutputStream(mFile, true));
    mResponseObserver = Mockito.mock(StreamObserver.class);
    mWriteHandler = new UfsFallbackBlockWriteHandler(mBlockWorker, ufsManager, mResponseObserver, mUserInfo, false);
    setupResponseTrigger();
    // create a partial block in block store first
    mBlockStore.createBlock(TEST_SESSION_ID, TEST_BLOCK_ID, AllocateOptions.forCreate(CHUNK_SIZE, BlockStoreLocation.anyDirInTier(Constants.MEDIUM_MEM)));
    BlockWriter writer = mBlockStore.getBlockWriter(TEST_SESSION_ID, TEST_BLOCK_ID);
    DataBuffer buffer = newDataBuffer(PARTIAL_WRITTEN);
    mPartialChecksum = getChecksum(buffer);
    writer.append((ByteBuf) buffer.getNettyOutput());
    writer.close();
}

Example 14 with BlockWriter

use of alluxio.worker.block.io.BlockWriter in project alluxio by Alluxio.

the class BlockWorkerDataWriter method create.

/**
 * Creates an instance of {@link BlockWorkerDataWriter}.
 *
 * @param context the file system context
 * @param blockId the block ID
 * @param blockSize the block size in bytes
 * @param options the output stream options
 * @return the {@link BlockWorkerDataWriter} created
 */
public static BlockWorkerDataWriter create(final FileSystemContext context, long blockId, long blockSize, OutStreamOptions options) throws IOException {
    AlluxioConfiguration conf = context.getClusterConf();
    int chunkSize = (int) conf.getBytes(PropertyKey.USER_LOCAL_WRITER_CHUNK_SIZE_BYTES);
    long reservedBytes = Math.min(blockSize, conf.getBytes(PropertyKey.USER_FILE_RESERVED_BYTES));
    BlockWorker blockWorker = context.getProcessLocalWorker();
    Preconditions.checkNotNull(blockWorker, "blockWorker");
    long sessionId = SessionIdUtils.createSessionId();
    try {
        blockWorker.createBlock(sessionId, blockId, options.getWriteTier(), options.getMediumType(), reservedBytes);
        BlockWriter blockWriter = blockWorker.createBlockWriter(sessionId, blockId);
        return new BlockWorkerDataWriter(sessionId, blockId, options, blockWriter, blockWorker, chunkSize, reservedBytes, conf);
    } catch (BlockAlreadyExistsException | WorkerOutOfSpaceException | BlockDoesNotExistException | InvalidWorkerStateException e) {
        throw new IOException(e);
    }
}

Example 15 with BlockWriter

use of alluxio.worker.block.io.BlockWriter in project alluxio by Alluxio.

the class CacheRequestManager method cacheBlockFromRemoteWorker.

/**
 * Caches the block at best effort from a remote worker (possibly from UFS indirectly).
 *
 * @param blockId block ID
 * @param blockSize block size
 * @param sourceAddress the source to read the block previously by client
 * @param openUfsBlockOptions options to open the UFS file
 * @return if the block is cached
 */
private boolean cacheBlockFromRemoteWorker(long blockId, long blockSize, InetSocketAddress sourceAddress, Protocol.OpenUfsBlockOptions openUfsBlockOptions) throws IOException, AlluxioException {
    try {
        mBlockWorker.createBlock(Sessions.CACHE_WORKER_SESSION_ID, blockId, 0, "", blockSize);
    } catch (BlockAlreadyExistsException e) {
        // It is already cached
        LOG.debug("block already cached: {}", blockId);
        return true;
    }
    try (BlockReader reader = getRemoteBlockReader(blockId, blockSize, sourceAddress, openUfsBlockOptions);
        BlockWriter writer = mBlockWorker.createBlockWriter(Sessions.CACHE_WORKER_SESSION_ID, blockId)) {
        BufferUtils.transfer(reader.getChannel(), writer.getChannel());
        mBlockWorker.commitBlock(Sessions.CACHE_WORKER_SESSION_ID, blockId, false);
        return true;
    } catch (AlluxioException | IOException e) {
        LOG.warn("Failed to async cache block {} from remote worker ({}) on copying the block: {}", blockId, sourceAddress, e.toString());
        try {
            mBlockWorker.abortBlock(Sessions.CACHE_WORKER_SESSION_ID, blockId);
        } catch (AlluxioException | IOException ee) {
            LOG.warn("Failed to abort block {}: {}", blockId, ee.toString());
        }
        throw e;
    }
}