Examples with WriteObjectRequest com.google.storage.v2.WriteObjectRequest used on opensource projects

Example 1 with WriteObjectRequest

use of com.google.storage.v2.WriteObjectRequest in project gapic-generator-java by googleapis.

the class AsyncWriteObject method asyncWriteObject.

public static void asyncWriteObject() throws Exception {
    // It may require modifications to work in your environment.
    try (StorageClient storageClient = StorageClient.create()) {
        ApiStreamObserver<WriteObjectRequest> responseObserver = new ApiStreamObserver<WriteObjectRequest>() {

            @Override
            public void onNext(WriteObjectResponse response) {
            // Do something when a response is received.
            }

            @Override
            public void onError(Throwable t) {
            // Add error-handling
            }

            @Override
            public void onCompleted() {
            // Do something when complete.
            }
        };
        ApiStreamObserver<WriteObjectRequest> requestObserver = storageClient.writeObject().clientStreamingCall(responseObserver);
        WriteObjectRequest request = WriteObjectRequest.newBuilder().setWriteOffset(-1559543565).setObjectChecksums(ObjectChecksums.newBuilder().build()).setFinishWrite(true).setCommonObjectRequestParams(CommonObjectRequestParams.newBuilder().build()).setCommonRequestParams(CommonRequestParams.newBuilder().build()).build();
        requestObserver.onNext(request);
    }
}

Example 2 with WriteObjectRequest

use of com.google.storage.v2.WriteObjectRequest in project grpc-gcp-java by GoogleCloudPlatform.

the class GrpcClient method makeInsertRequest.

private void makeInsertRequest(ManagedChannel channel, ResultTable results, int threadId) throws InterruptedException {
    StorageGrpc.StorageStub asyncStub = StorageGrpc.newStub(channel);
    if (creds != null) {
        asyncStub = asyncStub.withCallCredentials(MoreCallCredentials.from(creds));
    }
    int totalBytes = args.size * 1024;
    byte[] data = new byte[totalBytes];
    for (int i = 0; i < args.calls; i++) {
        String obj = objectResolver.Resolve(threadId, i);
        int offset = 0;
        boolean isFirst = true;
        boolean isLast = false;
        final CountDownLatch finishLatch = new CountDownLatch(1);
        StreamObserver<WriteObjectResponse> responseObserver = new StreamObserver<WriteObjectResponse>() {

            long start = System.currentTimeMillis();

            @Override
            public void onNext(WriteObjectResponse value) {
            }

            @Override
            public void onError(Throwable t) {
                logger.warning("InsertObject failed with: " + Status.fromThrowable(t));
                finishLatch.countDown();
            }

            @Override
            public void onCompleted() {
                long dur = System.currentTimeMillis() - start;
                results.reportResult(args.bkt, obj, totalBytes, dur);
                finishLatch.countDown();
            }
        };
        StreamObserver<WriteObjectRequest> requestObserver = asyncStub.writeObject(responseObserver);
        while (offset < totalBytes) {
            int add;
            if (offset + Values.MAX_WRITE_CHUNK_BYTES_VALUE <= totalBytes) {
                add = Values.MAX_WRITE_CHUNK_BYTES_VALUE;
            } else {
                add = totalBytes - offset;
            }
            if (offset + add == totalBytes) {
                isLast = true;
            }
            WriteObjectRequest req = getWriteRequest(isFirst, isLast, offset, ByteString.copyFrom(data, offset, add), obj);
            requestObserver.onNext(req);
            if (finishLatch.getCount() == 0) {
                logger.warning("Stream completed before finishing sending requests");
                return;
            }
            offset += add;
        }
        requestObserver.onCompleted();
        if (!finishLatch.await(20, TimeUnit.MINUTES)) {
            logger.warning("insertObject cannot finish within 20 minutes");
        }
    }
}

Example 3 with WriteObjectRequest

use of com.google.storage.v2.WriteObjectRequest in project hadoop-connectors by GoogleCloudDataproc.

the class GoogleCloudStorageGrpcWriteChannelTest method writeOneChunkWithSingleErrorAndResume.

@Test
public void writeOneChunkWithSingleErrorAndResume() throws Exception {
    AsyncWriteChannelOptions options = AsyncWriteChannelOptions.builder().setUploadChunkSize(GCS_MINIMUM_CHUNK_SIZE).build();
    ObjectWriteConditions writeConditions = ObjectWriteConditions.NONE;
    GoogleCloudStorageGrpcWriteChannel writeChannel = newWriteChannel(options, writeConditions, /* requesterPaysProject= */
    null, () -> BackOff.ZERO_BACKOFF);
    fakeService.setInsertObjectExceptions(ImmutableList.of(new StatusException(Status.DEADLINE_EXCEEDED)));
    fakeService.setQueryWriteStatusResponses(ImmutableList.of(QueryWriteStatusResponse.newBuilder().setPersistedSize(1).build()).iterator());
    ByteString chunk = createTestData(GCS_MINIMUM_CHUNK_SIZE);
    ArgumentCaptor<WriteObjectRequest> requestCaptor = ArgumentCaptor.forClass(WriteObjectRequest.class);
    writeChannel.initialize();
    writeChannel.write(chunk.asReadOnlyByteBuffer());
    writeChannel.close();
    verify(fakeService, times(1)).startResumableWrite(eq(START_REQUEST), any());
    verify(fakeService, times(1)).queryWriteStatus(eq(WRITE_STATUS_REQUEST), any());
    verify(fakeService.insertRequestObserver, atLeast(1)).onNext(requestCaptor.capture());
    // TODO(hgong): Figure out a way to check the expected requests and actual reqeusts builder.
    // assertEquals(expectedRequests, requestCaptor.getAllValues());
    verify(fakeService.insertRequestObserver, atLeast(1)).onCompleted();
}

Example 4 with WriteObjectRequest

use of com.google.storage.v2.WriteObjectRequest in project hadoop-connectors by GoogleCloudDataproc.

the class GoogleCloudStorageGrpcWriteChannelTest method writeHandlesUncommittedData.

@Test
public void writeHandlesUncommittedData() throws Exception {
    GoogleCloudStorageGrpcWriteChannel writeChannel = newWriteChannel();
    fakeService.setQueryWriteStatusResponses(ImmutableList.of(QueryWriteStatusResponse.newBuilder().setPersistedSize(GCS_MINIMUM_CHUNK_SIZE * 3 / 4).build()).iterator());
    ByteString data = createTestData(GCS_MINIMUM_CHUNK_SIZE * 3 / 2);
    writeChannel.initialize();
    writeChannel.write(data.asReadOnlyByteBuffer());
    writeChannel.close();
    ArgumentCaptor<WriteObjectRequest> requestCaptor = ArgumentCaptor.forClass(WriteObjectRequest.class);
    verify(fakeService, times(1)).startResumableWrite(eq(START_REQUEST), any());
    // TODO(b/150892988): Use this mock when implement resuming after a transient error.
    // verify(fakeService, times(1)).queryWriteStatus(eq(WRITE_STATUS_REQUEST), any());
    verify(fakeService.insertRequestObserver, times(1)).onNext(requestCaptor.capture());
    verify(fakeService.insertRequestObserver, atLeast(1)).onCompleted();
}

Example 5 with WriteObjectRequest

use of com.google.storage.v2.WriteObjectRequest in project hadoop-connectors by GoogleCloudDataproc.

the class GoogleCloudStorageGrpcWriteChannelTest method writeSendsMultipleInsertObjectRequests.

@Test
public void writeSendsMultipleInsertObjectRequests() throws Exception {
    GoogleCloudStorageGrpcWriteChannel writeChannel = newWriteChannel();
    fakeService.setQueryWriteStatusResponses(ImmutableList.of(QueryWriteStatusResponse.newBuilder().setPersistedSize(GCS_MINIMUM_CHUNK_SIZE).build(), QueryWriteStatusResponse.newBuilder().setPersistedSize(2 * GCS_MINIMUM_CHUNK_SIZE).build()).iterator());
    ByteString data = createTestData(GCS_MINIMUM_CHUNK_SIZE * 5 / 2);
    writeChannel.initialize();
    writeChannel.write(data.asReadOnlyByteBuffer());
    writeChannel.close();
    ArgumentCaptor<WriteObjectRequest> requestCaptor = ArgumentCaptor.forClass(WriteObjectRequest.class);
    verify(fakeService, times(1)).startResumableWrite(eq(START_REQUEST), any());
    verify(fakeService.insertRequestObserver, times(1)).onNext(requestCaptor.capture());
    verify(fakeService.insertRequestObserver, atLeast(1)).onCompleted();
}