Examples with CreateLocalBlockResponse alluxio.grpc.CreateLocalBlockResponse used on opensource projects

Example 1 with CreateLocalBlockResponse

use of alluxio.grpc.CreateLocalBlockResponse in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getOutStreamLocal.

@Test
public void getOutStreamLocal() throws Exception {
    File file = File.createTempFile("test", ".tmp");
    CreateLocalBlockResponse response = CreateLocalBlockResponse.newBuilder().setPath(file.getAbsolutePath()).build();
    when(mWorkerClient.createLocalBlock(any(StreamObserver.class))).thenAnswer(new Answer() {

        public Object answer(InvocationOnMock invocation) {
            StreamObserver<CreateLocalBlockResponse> observer = invocation.getArgument(0, StreamObserver.class);
            observer.onNext(response);
            return mStreamObserver;
        }
    });
    OutStreamOptions options = OutStreamOptions.defaults(mClientContext).setBlockSizeBytes(BLOCK_LENGTH).setLocationPolicy(new MockBlockLocationPolicy(Lists.newArrayList(WORKER_NET_ADDRESS_LOCAL))).setWriteType(WriteType.MUST_CACHE);
    BlockOutStream stream = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options);
    assertEquals(WORKER_NET_ADDRESS_LOCAL, stream.getAddress());
}

Example 2 with CreateLocalBlockResponse

use of alluxio.grpc.CreateLocalBlockResponse in project alluxio by Alluxio.

the class AlluxioBlockStoreTest method getOutStreamWithReplicated.

@Test
public void getOutStreamWithReplicated() throws Exception {
    File file = File.createTempFile("test", ".tmp");
    CreateLocalBlockResponse response = CreateLocalBlockResponse.newBuilder().setPath(file.getAbsolutePath()).build();
    when(mWorkerClient.createLocalBlock(any(StreamObserver.class))).thenAnswer(new Answer() {

        public Object answer(InvocationOnMock invocation) {
            StreamObserver<CreateLocalBlockResponse> observer = invocation.getArgument(0, StreamObserver.class);
            observer.onNext(response);
            return mStreamObserver;
        }
    });
    when(mContext.getCachedWorkers()).thenReturn(Lists.newArrayList(new BlockWorkerInfo(WORKER_NET_ADDRESS_LOCAL, -1, -1), new BlockWorkerInfo(WORKER_NET_ADDRESS_REMOTE, -1, -1)));
    OutStreamOptions options = OutStreamOptions.defaults(mClientContext).setBlockSizeBytes(BLOCK_LENGTH).setLocationPolicy(new MockBlockLocationPolicy(Lists.newArrayList(WORKER_NET_ADDRESS_LOCAL, WORKER_NET_ADDRESS_REMOTE))).setWriteType(WriteType.MUST_CACHE).setReplicationMin(2);
    BlockOutStream stream = mBlockStore.getOutStream(BLOCK_ID, BLOCK_LENGTH, options);
    assertEquals(alluxio.client.block.stream.BlockOutStream.class, stream.getClass());
}

Example 3 with CreateLocalBlockResponse

use of alluxio.grpc.CreateLocalBlockResponse in project alluxio by Alluxio.

the class ShortCircuitBlockWriteHandler method onNext.

/**
 * Handles request to create local block. No exceptions should be thrown.
 *
 * @param request a create request
 */
@Override
public void onNext(CreateLocalBlockRequest request) {
    final String methodName = request.getOnlyReserveSpace() ? "ReserveSpace" : "CreateBlock";
    RpcUtils.streamingRPCAndLog(LOG, new RpcUtils.StreamingRpcCallable<CreateLocalBlockResponse>() {

        @Override
        public CreateLocalBlockResponse call() throws Exception {
            if (request.getOnlyReserveSpace()) {
                mBlockWorker.requestSpace(mSessionId, request.getBlockId(), request.getSpaceToReserve());
                return CreateLocalBlockResponse.newBuilder().build();
            } else {
                Preconditions.checkState(mRequest == null);
                mRequest = request;
                if (mSessionId == INVALID_SESSION_ID) {
                    mSessionId = IdUtils.createSessionId();
                    String path = mBlockWorker.createBlock(mSessionId, request.getBlockId(), request.getTier(), request.getMediumType(), request.getSpaceToReserve());
                    CreateLocalBlockResponse response = CreateLocalBlockResponse.newBuilder().setPath(path).build();
                    return response;
                } else {
                    LOG.warn("Create block {} without closing the previous session {}.", request.getBlockId(), mSessionId);
                    throw new InvalidWorkerStateException(ExceptionMessage.SESSION_NOT_CLOSED.getMessage(mSessionId));
                }
            }
        }

        @Override
        public void exceptionCaught(Throwable throwable) {
            if (mSessionId != INVALID_SESSION_ID) {
                // In case the client is a UfsFallbackDataWriter, DO NOT clean the temp blocks.
                if (throwable instanceof alluxio.exception.WorkerOutOfSpaceException && request.hasCleanupOnFailure() && !request.getCleanupOnFailure()) {
                    mResponseObserver.onError(GrpcExceptionUtils.fromThrowable(throwable));
                    return;
                }
                mBlockWorker.cleanupSession(mSessionId);
                mSessionId = INVALID_SESSION_ID;
            }
            mResponseObserver.onError(GrpcExceptionUtils.fromThrowable(throwable));
        }
    }, methodName, true, false, mResponseObserver, "Session=%d, Request=%s", mSessionId, request);
}

Example 4 with CreateLocalBlockResponse

use of alluxio.grpc.CreateLocalBlockResponse in project alluxio by Alluxio.

the class LocalFileDataWriter method create.

/**
 * Creates an instance of {@link LocalFileDataWriter}. This requires the block to be locked
 * beforehand.
 *
 * @param context the file system context
 * @param address the worker network address
 * @param blockId the block ID
 * @param blockSize the block size in bytes
 * @param options the output stream options
 * @return the {@link LocalFileDataWriter} created
 */
public static LocalFileDataWriter create(final FileSystemContext context, final WorkerNetAddress address, long blockId, long blockSize, OutStreamOptions options) throws IOException {
    AlluxioConfiguration conf = context.getClusterConf();
    long chunkSize = conf.getBytes(PropertyKey.USER_LOCAL_WRITER_CHUNK_SIZE_BYTES);
    Closer closer = Closer.create();
    try {
        CloseableResource<BlockWorkerClient> blockWorker = context.acquireBlockWorkerClient(address);
        closer.register(blockWorker);
        int writerBufferSizeMessages = conf.getInt(PropertyKey.USER_STREAMING_WRITER_BUFFER_SIZE_MESSAGES);
        long fileBufferBytes = conf.getBytes(PropertyKey.USER_FILE_BUFFER_BYTES);
        long dataTimeout = conf.getMs(PropertyKey.USER_STREAMING_DATA_WRITE_TIMEOUT);
        // in cases we know precise block size, make more accurate reservation.
        long reservedBytes = Math.min(blockSize, conf.getBytes(PropertyKey.USER_FILE_RESERVED_BYTES));
        CreateLocalBlockRequest.Builder builder = CreateLocalBlockRequest.newBuilder().setBlockId(blockId).setTier(options.getWriteTier()).setSpaceToReserve(reservedBytes).setMediumType(options.getMediumType()).setPinOnCreate(options.getWriteType() == WriteType.ASYNC_THROUGH);
        if (options.getWriteType() == WriteType.ASYNC_THROUGH && conf.getBoolean(PropertyKey.USER_FILE_UFS_TIER_ENABLED)) {
            builder.setCleanupOnFailure(false);
        }
        CreateLocalBlockRequest createRequest = builder.build();
        GrpcBlockingStream<CreateLocalBlockRequest, CreateLocalBlockResponse> stream = new GrpcBlockingStream<>(blockWorker.get()::createLocalBlock, writerBufferSizeMessages, MoreObjects.toStringHelper(LocalFileDataWriter.class).add("request", createRequest).add("address", address).toString());
        stream.send(createRequest, dataTimeout);
        CreateLocalBlockResponse response = stream.receive(dataTimeout);
        Preconditions.checkState(response != null && response.hasPath());
        LocalFileBlockWriter writer = closer.register(new LocalFileBlockWriter(response.getPath()));
        return new LocalFileDataWriter(chunkSize, writer, createRequest, stream, closer, fileBufferBytes, dataTimeout);
    } catch (Exception e) {
        throw CommonUtils.closeAndRethrow(closer, e);
    }
}

Example 5 with CreateLocalBlockResponse

use of alluxio.grpc.CreateLocalBlockResponse in project alluxio by Alluxio.

the class LocalFileDataWriter method ensureReserved.

/**
 * Reserves enough space in the block worker.
 *
 * @param pos the pos of the file/block to reserve to
 */
private void ensureReserved(long pos) throws IOException {
    if (pos <= mPosReserved) {
        return;
    }
    long toReserve = Math.max(pos - mPosReserved, mFileBufferBytes);
    CreateLocalBlockRequest request = mCreateRequest.toBuilder().setSpaceToReserve(toReserve).setOnlyReserveSpace(true).build();
    mStream.send(request, mDataTimeoutMs);
    CreateLocalBlockResponse response = mStream.receive(mDataTimeoutMs);
    Preconditions.checkState(response != null, String.format("Stream closed while waiting for reserve request %s", request.toString()));
    Preconditions.checkState(!response.hasPath(), String.format("Invalid response for reserve request %s", request.toString()));
    mPosReserved += toReserve;
}