Examples with ReadObjectResponse com.google.storage.v2.ReadObjectResponse used on opensource projects

Example 6 with ReadObjectResponse

use of com.google.storage.v2.ReadObjectResponse in project hadoop-connectors by GoogleCloudDataproc.

the class WatchdogTest method watchPassThroughServerStreamingRPC.

@Test
public void watchPassThroughServerStreamingRPC() {
    CancellableContext requestContext = Context.current().withCancellation();
    ReadObjectResponse defaultInstance = ReadObjectResponse.getDefaultInstance();
    Response<ReadObjectResponse> validResponse = new Response<>(defaultInstance);
    Response<ReadObjectResponse> errorResponse = new Response<>(new RuntimeException("Read timeout out"));
    List<Response<ReadObjectResponse>> responseList = Lists.newArrayList(validResponse, errorResponse);
    ResponseIteratorStub<ReadObjectResponse> responseIterator = new ResponseIteratorStub<>(responseList);
    Iterator<ReadObjectResponse> watch = watchdog.watch(requestContext, responseIterator, waitTime);
    ReadObjectResponse next = watch.next();
    assertThat(next).isEqualTo(validResponse.object);
    assertThat(watchdog).isNotNull();
    assertThat(watchdog.getOpenStreams()).hasSize(1);
    assertThrows(RuntimeException.class, watch::hasNext);
    assertThat(watchdog.getOpenStreams().isEmpty()).isTrue();
}

Example 7 with ReadObjectResponse

use of com.google.storage.v2.ReadObjectResponse in project grpc-gcp-java by GoogleCloudPlatform.

the class GrpcClient method makeReadObjectRequest.

private void makeReadObjectRequest(ManagedChannel channel, ResultTable results, int threadId) {
    StorageGrpc.StorageBlockingStub blockingStub = StorageGrpc.newBlockingStub(channel);
    if (creds != null) {
        blockingStub = blockingStub.withCallCredentials(MoreCallCredentials.from(creds));
    }
    byte[] scratch = new byte[4 * 1024 * 1024];
    for (int i = 0; i < args.calls; i++) {
        String object = objectResolver.Resolve(threadId, i);
        ReadObjectRequest readRequest = ReadObjectRequest.newBuilder().setBucket(toV2BucketName(args.bkt)).setObject(object).build();
        long start = System.currentTimeMillis();
        long totalBytes = 0;
        Iterator<ReadObjectResponse> resIterator;
        if (useZeroCopy) {
            resIterator = io.grpc.stub.ClientCalls.blockingServerStreamingCall(blockingStub.getChannel(), readObjectMethod, blockingStub.getCallOptions(), readRequest);
        } else {
            resIterator = blockingStub.readObject(readRequest);
        }
        try {
            while (true) {
                ReadObjectResponse res = resIterator.next();
                // When zero-copy mashaller is used, the stream that backs ReadObjectResponse
                // should be closed when the mssage is no longed needed so that all buffers in the
                // stream can be reclaimed. If zero-copy is not used, stream will be null.
                InputStream stream = ReadObjectResponseMarshaller.popStream(res);
                try {
                    // Just copy to scratch memory to ensure its data is consumed.
                    ByteString content = res.getChecksummedData().getContent();
                    totalBytes += content.size();
                    content.copyTo(scratch, 0);
                } finally {
                    if (stream != null) {
                        try {
                            stream.close();
                        } catch (IOException e) {
                            throw new RuntimeException(e);
                        }
                    }
                }
            }
        } catch (NoSuchElementException e) {
        }
        long dur = System.currentTimeMillis() - start;
        results.reportResult(args.bkt, object, totalBytes, dur);
    }
}

Example 8 with ReadObjectResponse

use of com.google.storage.v2.ReadObjectResponse in project hadoop-connectors by GoogleCloudDataproc.

the class WatchdogTest method watchMultipleStreams.

@Test
public void watchMultipleStreams() {
    NoopClientCallStub<WriteObjectRequest, WriteObjectResponse> clientCall = new NoopClientCallStub<>();
    StreamObserver<WriteObjectRequest> timeoutStreamObserver = new StreamObserver<>() {

        @Override
        public void onNext(WriteObjectRequest value) {
            logger.atInfo().log("Sleeping for 10 seconds");
            sleepUninterruptibly(Duration.ofSeconds(10));
        }

        @Override
        public void onError(Throwable t) {
        }

        @Override
        public void onCompleted() {
        }
    };
    CancellableContext requestContext = Context.current().withCancellation();
    Iterator<ReadObjectResponse> responseIterator = new Iterator<>() {

        @Override
        public boolean hasNext() {
            logger.atInfo().log("Sleeping for 10 seconds");
            sleepUninterruptibly(Duration.ofSeconds(10));
            return true;
        }

        @Override
        public ReadObjectResponse next() {
            return null;
        }
    };
    Iterator<ReadObjectResponse> clientStreamingRPCWatch = watchdog.watch(requestContext, responseIterator, waitTime);
    StreamObserver<WriteObjectRequest> serverStreamingRPCWatch = watchdog.watch(clientCall, timeoutStreamObserver, waitTime);
    assertThat(watchdog).isNotNull();
    assertThat(watchdog.getOpenStreams()).hasSize(2);
    boolean actual = clientStreamingRPCWatch.hasNext();
    WriteObjectRequest value = WriteObjectRequest.newBuilder().build();
    serverStreamingRPCWatch.onNext(value);
    assertThat(actual).isTrue();
    assertThat(requestContext.isCancelled()).isTrue();
    assertThat(clientCall.cancelled).isTrue();
    assertThat(clientCall.cause).isInstanceOf(TimeoutException.class);
    assertThat(watchdog.getOpenStreams().isEmpty()).isTrue();
}

Example 9 with ReadObjectResponse

use of com.google.storage.v2.ReadObjectResponse in project hadoop-connectors by GoogleCloudDataproc.

the class GoogleCloudStorageGrpcReadChannel method readObjectContentFromGCS.

private int readObjectContentFromGCS(ByteBuffer byteBuffer) throws IOException {
    int bytesRead = 0;
    ReadObjectResponse res = resIterator.next();
    // When zero-copy marshaller is used, the stream that backs GetObjectMediaResponse
    // should be closed when the message is no longer needed so that all buffers in the
    // stream can be reclaimed. If zero-copy is not used, stream will be null.
    InputStream stream = getObjectMediaResponseMarshaller.popStream(res);
    try {
        ByteString content = res.getChecksummedData().getContent();
        if (bytesToSkipBeforeReading >= 0 && bytesToSkipBeforeReading < content.size()) {
            content = content.substring((int) bytesToSkipBeforeReading);
            positionInGrpcStream += bytesToSkipBeforeReading;
            bytesToSkipBeforeReading = 0;
        } else if (bytesToSkipBeforeReading >= content.size()) {
            positionInGrpcStream += content.size();
            bytesToSkipBeforeReading -= content.size();
            return bytesRead;
        }
        if (readOptions.isGrpcChecksumsEnabled() && res.getChecksummedData().hasCrc32C()) {
            validateChecksum(res);
        }
        boolean responseSizeLargerThanRemainingBuffer = content.size() > byteBuffer.remaining();
        int bytesToWrite = responseSizeLargerThanRemainingBuffer ? byteBuffer.remaining() : content.size();
        put(content, 0, bytesToWrite, byteBuffer);
        bytesRead += bytesToWrite;
        positionInGrpcStream += bytesToWrite;
        if (responseSizeLargerThanRemainingBuffer) {
            invalidateBufferedContent();
            bufferedContent = content;
            bufferedContentReadOffset = bytesToWrite;
            // This is to keep the stream alive for the message backed by this.
            streamForBufferedContent = stream;
            stream = null;
        }
    } finally {
        if (stream != null) {
            stream.close();
        }
    }
    return bytesRead;
}