Examples with LocalFileBlockWriter alluxio.worker.block.io.LocalFileBlockWriter used on opensource projects

Example 6 with LocalFileBlockWriter

use of alluxio.worker.block.io.LocalFileBlockWriter in project alluxio by Alluxio.

the class TieredBlockStoreTestUtils method cache.

/**
 * Caches bytes into {@link BlockStore} at specific location.
 *
 * @param sessionId session who caches the data
 * @param blockId id of the cached block
 * @param options allocation options
 * @param blockStore block store that the block is written into
 * @param pinOnCreate whether to pin block on create
 */
public static void cache(long sessionId, long blockId, AllocateOptions options, BlockStore blockStore, boolean pinOnCreate) throws Exception {
    TempBlockMeta tempBlockMeta = blockStore.createBlock(sessionId, blockId, options);
    // write data
    BlockWriter writer = new LocalFileBlockWriter(tempBlockMeta.getPath());
    writer.append(BufferUtils.getIncreasingByteBuffer(Ints.checkedCast(options.getSize())));
    writer.close();
    // commit block
    blockStore.commitBlock(sessionId, blockId, pinOnCreate);
}

Example 7 with LocalFileBlockWriter

use of alluxio.worker.block.io.LocalFileBlockWriter in project alluxio by Alluxio.

the class TieredBlockStoreTestUtils method cache.

/**
 * Caches bytes into {@link BlockStore} at specific location.
 *
 * @param sessionId session who caches the data
 * @param blockId id of the cached block
 * @param bytes size of the block in bytes
 * @param blockStore block store that the block is written into
 * @param location the location where the block resides
 * @param pinOnCreate whether to pin block on create
 */
public static void cache(long sessionId, long blockId, long bytes, BlockStore blockStore, BlockStoreLocation location, boolean pinOnCreate) throws Exception {
    TempBlockMeta tempBlockMeta = blockStore.createBlock(sessionId, blockId, AllocateOptions.forCreate(bytes, location));
    // write data
    BlockWriter writer = new LocalFileBlockWriter(tempBlockMeta.getPath());
    writer.append(BufferUtils.getIncreasingByteBuffer(Ints.checkedCast(bytes)));
    writer.close();
    // commit block
    blockStore.commitBlock(sessionId, blockId, pinOnCreate);
}

Example 8 with LocalFileBlockWriter

use of alluxio.worker.block.io.LocalFileBlockWriter in project alluxio by Alluxio.

the class BlockWriteHandlerTest method before.

@Before
public void before() throws Exception {
    mFile = mTestFolder.newFile();
    mBlockWorker = new NoopBlockWorker() {

        @Override
        public BlockWriter createBlockWriter(long sessionId, long blockId) {
            return mBlockWriter;
        }
    };
    mBlockWriter = new LocalFileBlockWriter(mFile.getPath());
    mResponseObserver = Mockito.mock(StreamObserver.class);
    mWriteHandler = new BlockWriteHandler(mBlockWorker, mResponseObserver, mUserInfo, false);
    setupResponseTrigger();
}

Example 9 with LocalFileBlockWriter

use of alluxio.worker.block.io.LocalFileBlockWriter in project alluxio by Alluxio.

the class LocalFileDataWriterTest method streamClosed.

@Test
public void streamClosed() throws Exception {
    LocalFileDataWriter writer = LocalFileDataWriter.create(mContext, mAddress, BLOCK_ID, 128, /* unused */
    OutStreamOptions.defaults(mClientContext));
    // Close stream before closing the writer
    PowerMockito.when(mStream.isCanceled()).thenReturn(true);
    PowerMockito.when(mStream.isClosed()).thenReturn(true);
    PowerMockito.when(mStream.isOpen()).thenReturn(true);
    writer.close();
    // Verify there are no open files
    LocalFileBlockWriter blockWriter = Whitebox.getInternalState(writer, "mWriter");
    Assert.assertTrue(Whitebox.getInternalState(blockWriter, "mClosed"));
}

Example 10 with LocalFileBlockWriter

use of alluxio.worker.block.io.LocalFileBlockWriter in project alluxio by Alluxio.

the class LocalFileDataWriter method create.

/**
 * Creates an instance of {@link LocalFileDataWriter}. This requires the block to be locked
 * beforehand.
 *
 * @param context the file system context
 * @param address the worker network address
 * @param blockId the block ID
 * @param blockSize the block size in bytes
 * @param options the output stream options
 * @return the {@link LocalFileDataWriter} created
 */
public static LocalFileDataWriter create(final FileSystemContext context, final WorkerNetAddress address, long blockId, long blockSize, OutStreamOptions options) throws IOException {
    AlluxioConfiguration conf = context.getClusterConf();
    long chunkSize = conf.getBytes(PropertyKey.USER_LOCAL_WRITER_CHUNK_SIZE_BYTES);
    Closer closer = Closer.create();
    try {
        CloseableResource<BlockWorkerClient> blockWorker = context.acquireBlockWorkerClient(address);
        closer.register(blockWorker);
        int writerBufferSizeMessages = conf.getInt(PropertyKey.USER_STREAMING_WRITER_BUFFER_SIZE_MESSAGES);
        long fileBufferBytes = conf.getBytes(PropertyKey.USER_FILE_BUFFER_BYTES);
        long dataTimeout = conf.getMs(PropertyKey.USER_STREAMING_DATA_WRITE_TIMEOUT);
        // in cases we know precise block size, make more accurate reservation.
        long reservedBytes = Math.min(blockSize, conf.getBytes(PropertyKey.USER_FILE_RESERVED_BYTES));
        CreateLocalBlockRequest.Builder builder = CreateLocalBlockRequest.newBuilder().setBlockId(blockId).setTier(options.getWriteTier()).setSpaceToReserve(reservedBytes).setMediumType(options.getMediumType()).setPinOnCreate(options.getWriteType() == WriteType.ASYNC_THROUGH);
        if (options.getWriteType() == WriteType.ASYNC_THROUGH && conf.getBoolean(PropertyKey.USER_FILE_UFS_TIER_ENABLED)) {
            builder.setCleanupOnFailure(false);
        }
        CreateLocalBlockRequest createRequest = builder.build();
        GrpcBlockingStream<CreateLocalBlockRequest, CreateLocalBlockResponse> stream = new GrpcBlockingStream<>(blockWorker.get()::createLocalBlock, writerBufferSizeMessages, MoreObjects.toStringHelper(LocalFileDataWriter.class).add("request", createRequest).add("address", address).toString());
        stream.send(createRequest, dataTimeout);
        CreateLocalBlockResponse response = stream.receive(dataTimeout);
        Preconditions.checkState(response != null && response.hasPath());
        LocalFileBlockWriter writer = closer.register(new LocalFileBlockWriter(response.getPath()));
        return new LocalFileDataWriter(chunkSize, writer, createRequest, stream, closer, fileBufferBytes, dataTimeout);
    } catch (Exception e) {
        throw CommonUtils.closeAndRethrow(closer, e);
    }
}