Examples with CreateLocalBlockRequest alluxio.grpc.CreateLocalBlockRequest used on opensource projects

Example 1 with CreateLocalBlockRequest

use of alluxio.grpc.CreateLocalBlockRequest in project alluxio by Alluxio.

the class LocalFileDataWriter method create.

/**
 * Creates an instance of {@link LocalFileDataWriter}. This requires the block to be locked
 * beforehand.
 *
 * @param context the file system context
 * @param address the worker network address
 * @param blockId the block ID
 * @param blockSize the block size in bytes
 * @param options the output stream options
 * @return the {@link LocalFileDataWriter} created
 */
public static LocalFileDataWriter create(final FileSystemContext context, final WorkerNetAddress address, long blockId, long blockSize, OutStreamOptions options) throws IOException {
    AlluxioConfiguration conf = context.getClusterConf();
    long chunkSize = conf.getBytes(PropertyKey.USER_LOCAL_WRITER_CHUNK_SIZE_BYTES);
    Closer closer = Closer.create();
    try {
        CloseableResource<BlockWorkerClient> blockWorker = context.acquireBlockWorkerClient(address);
        closer.register(blockWorker);
        int writerBufferSizeMessages = conf.getInt(PropertyKey.USER_STREAMING_WRITER_BUFFER_SIZE_MESSAGES);
        long fileBufferBytes = conf.getBytes(PropertyKey.USER_FILE_BUFFER_BYTES);
        long dataTimeout = conf.getMs(PropertyKey.USER_STREAMING_DATA_WRITE_TIMEOUT);
        // in cases we know precise block size, make more accurate reservation.
        long reservedBytes = Math.min(blockSize, conf.getBytes(PropertyKey.USER_FILE_RESERVED_BYTES));
        CreateLocalBlockRequest.Builder builder = CreateLocalBlockRequest.newBuilder().setBlockId(blockId).setTier(options.getWriteTier()).setSpaceToReserve(reservedBytes).setMediumType(options.getMediumType()).setPinOnCreate(options.getWriteType() == WriteType.ASYNC_THROUGH);
        if (options.getWriteType() == WriteType.ASYNC_THROUGH && conf.getBoolean(PropertyKey.USER_FILE_UFS_TIER_ENABLED)) {
            builder.setCleanupOnFailure(false);
        }
        CreateLocalBlockRequest createRequest = builder.build();
        GrpcBlockingStream<CreateLocalBlockRequest, CreateLocalBlockResponse> stream = new GrpcBlockingStream<>(blockWorker.get()::createLocalBlock, writerBufferSizeMessages, MoreObjects.toStringHelper(LocalFileDataWriter.class).add("request", createRequest).add("address", address).toString());
        stream.send(createRequest, dataTimeout);
        CreateLocalBlockResponse response = stream.receive(dataTimeout);
        Preconditions.checkState(response != null && response.hasPath());
        LocalFileBlockWriter writer = closer.register(new LocalFileBlockWriter(response.getPath()));
        return new LocalFileDataWriter(chunkSize, writer, createRequest, stream, closer, fileBufferBytes, dataTimeout);
    } catch (Exception e) {
        throw CommonUtils.closeAndRethrow(closer, e);
    }
}

Example 2 with CreateLocalBlockRequest

use of alluxio.grpc.CreateLocalBlockRequest in project alluxio by Alluxio.

the class LocalFileDataWriter method ensureReserved.

/**
 * Reserves enough space in the block worker.
 *
 * @param pos the pos of the file/block to reserve to
 */
private void ensureReserved(long pos) throws IOException {
    if (pos <= mPosReserved) {
        return;
    }
    long toReserve = Math.max(pos - mPosReserved, mFileBufferBytes);
    CreateLocalBlockRequest request = mCreateRequest.toBuilder().setSpaceToReserve(toReserve).setOnlyReserveSpace(true).build();
    mStream.send(request, mDataTimeoutMs);
    CreateLocalBlockResponse response = mStream.receive(mDataTimeoutMs);
    Preconditions.checkState(response != null, String.format("Stream closed while waiting for reserve request %s", request.toString()));
    Preconditions.checkState(!response.hasPath(), String.format("Invalid response for reserve request %s", request.toString()));
    mPosReserved += toReserve;
}