use of com.google.bytestream.ByteStreamProto.WriteResponse in project bazel-buildfarm by bazelbuild.
the class Executor method loadFilesIntoCAS.
private static void loadFilesIntoCAS(String instanceName, Channel channel, Path blobsDir) throws Exception {
ContentAddressableStorageBlockingStub casStub = ContentAddressableStorageGrpc.newBlockingStub(channel);
List<Digest> missingDigests = findMissingBlobs(instanceName, blobsDir, casStub);
UUID uploadId = UUID.randomUUID();
int[] bucketSizes = new int[128];
BatchUpdateBlobsRequest.Builder[] buckets = new BatchUpdateBlobsRequest.Builder[128];
for (int i = 0; i < 128; i++) {
bucketSizes[i] = 0;
buckets[i] = BatchUpdateBlobsRequest.newBuilder().setInstanceName(instanceName);
}
ByteStreamStub bsStub = ByteStreamGrpc.newStub(channel);
for (Digest missingDigest : missingDigests) {
Path path = blobsDir.resolve(missingDigest.getHash() + "_" + missingDigest.getSizeBytes());
if (missingDigest.getSizeBytes() < Size.mbToBytes(1)) {
Request request = Request.newBuilder().setDigest(missingDigest).setData(ByteString.copyFrom(Files.readAllBytes(path))).build();
int maxBucketSize = 0;
long minBucketSize = Size.mbToBytes(2) + 1;
int maxBucketIndex = 0;
int minBucketIndex = -1;
int size = (int) missingDigest.getSizeBytes() + 48;
for (int i = 0; i < 128; i++) {
int newBucketSize = bucketSizes[i] + size;
if (newBucketSize < Size.mbToBytes(2) && bucketSizes[i] < minBucketSize) {
minBucketSize = bucketSizes[i];
minBucketIndex = i;
}
if (bucketSizes[i] > maxBucketSize) {
maxBucketSize = bucketSizes[i];
maxBucketIndex = i;
}
}
if (minBucketIndex < 0) {
bucketSizes[maxBucketIndex] = size;
BatchUpdateBlobsRequest batchRequest = buckets[maxBucketIndex].build();
Stopwatch stopwatch = Stopwatch.createStarted();
BatchUpdateBlobsResponse batchResponse = casStub.batchUpdateBlobs(batchRequest);
long usecs = stopwatch.elapsed(MICROSECONDS);
checkState(batchResponse.getResponsesList().stream().allMatch(response -> Code.forNumber(response.getStatus().getCode()) == Code.OK));
System.out.println("Updated " + batchRequest.getRequestsCount() + " blobs in " + (usecs / 1000.0) + "ms");
buckets[maxBucketIndex] = BatchUpdateBlobsRequest.newBuilder().setInstanceName(instanceName).addRequests(request);
} else {
bucketSizes[minBucketIndex] += size;
buckets[minBucketIndex].addRequests(request);
}
} else {
Stopwatch stopwatch = Stopwatch.createStarted();
SettableFuture<WriteResponse> writtenFuture = SettableFuture.create();
StreamObserver<WriteRequest> requestObserver = bsStub.write(new StreamObserver<WriteResponse>() {
@Override
public void onNext(WriteResponse response) {
writtenFuture.set(response);
}
@Override
public void onCompleted() {
}
@Override
public void onError(Throwable t) {
writtenFuture.setException(t);
}
});
HashCode hash = HashCode.fromString(missingDigest.getHash());
String resourceName = uploadResourceName(instanceName, uploadId, hash, missingDigest.getSizeBytes());
try (InputStream in = Files.newInputStream(path)) {
boolean first = true;
long writtenBytes = 0;
byte[] buf = new byte[64 * 1024];
while (writtenBytes != missingDigest.getSizeBytes()) {
int len = in.read(buf);
WriteRequest.Builder request = WriteRequest.newBuilder();
if (first) {
request.setResourceName(resourceName);
}
request.setData(ByteString.copyFrom(buf, 0, len)).setWriteOffset(writtenBytes);
if (writtenBytes + len == missingDigest.getSizeBytes()) {
request.setFinishWrite(true);
}
requestObserver.onNext(request.build());
writtenBytes += len;
first = false;
}
writtenFuture.get();
System.out.println("Wrote long " + DigestUtil.toString(missingDigest) + " in " + (stopwatch.elapsed(MICROSECONDS) / 1000.0) + "ms");
}
}
}
for (int i = 0; i < 128; i++) {
if (bucketSizes[i] > 0) {
BatchUpdateBlobsRequest batchRequest = buckets[i].build();
Stopwatch stopwatch = Stopwatch.createStarted();
BatchUpdateBlobsResponse batchResponse = casStub.batchUpdateBlobs(batchRequest);
long usecs = stopwatch.elapsed(MICROSECONDS);
checkState(batchResponse.getResponsesList().stream().allMatch(response -> Code.forNumber(response.getStatus().getCode()) == Code.OK));
System.out.println("Updated " + batchRequest.getRequestsCount() + " blobs in " + (usecs / 1000.0) + "ms");
}
}
}
use of com.google.bytestream.ByteStreamProto.WriteResponse in project bazel-buildfarm by bazelbuild.
the class StubInstanceTest method completedWriteBeforeCloseThrowsOnNextInteraction.
@Test
public void completedWriteBeforeCloseThrowsOnNextInteraction() throws IOException, InterruptedException {
String resourceName = "early-completed-output-stream-test";
serviceRegistry.addService(new ByteStreamImplBase() {
boolean completed = false;
int writtenBytes = 0;
@Override
public StreamObserver<WriteRequest> write(StreamObserver<WriteResponse> responseObserver) {
return new StreamObserver<WriteRequest>() {
@Override
public void onNext(WriteRequest request) {
if (!completed) {
writtenBytes = request.getData().size();
responseObserver.onNext(WriteResponse.newBuilder().setCommittedSize(writtenBytes).build());
responseObserver.onCompleted();
completed = true;
}
}
@Override
public void onError(Throwable t) {
t.printStackTrace();
}
@Override
public void onCompleted() {
}
};
}
@Override
public void queryWriteStatus(QueryWriteStatusRequest request, StreamObserver<QueryWriteStatusResponse> responseObserver) {
if (request.getResourceName().equals(resourceName)) {
responseObserver.onNext(QueryWriteStatusResponse.newBuilder().setCommittedSize(writtenBytes).setComplete(completed).build());
responseObserver.onCompleted();
} else {
responseObserver.onError(Status.NOT_FOUND.asException());
}
}
});
Instance instance = newStubInstance("early-completed-outputStream-test");
ByteString content = ByteString.copyFromUtf8("test-content");
boolean writeThrewException = false;
Write operationStreamWrite = instance.getOperationStreamWrite(resourceName);
try (OutputStream out = operationStreamWrite.getOutput(1, SECONDS, () -> {
})) {
content.writeTo(out);
try {
content.writeTo(out);
} catch (Exception e) {
writeThrewException = true;
}
}
assertThat(writeThrewException).isTrue();
instance.stop();
}
use of com.google.bytestream.ByteStreamProto.WriteResponse in project bazel-buildfarm by bazelbuild.
the class ByteStreamServiceTest method writePutsIntoBlobStore.
@Test
public void writePutsIntoBlobStore() throws IOException, InterruptedException {
ByteString helloWorld = ByteString.copyFromUtf8("Hello, World!");
Digest digest = DIGEST_UTIL.compute(helloWorld);
String uuid = UUID.randomUUID().toString();
String resourceName = createBlobUploadResourceName(uuid, digest);
Channel channel = InProcessChannelBuilder.forName(fakeServerName).directExecutor().build();
ClientCall<WriteRequest, WriteResponse> call = channel.newCall(ByteStreamGrpc.getWriteMethod(), CallOptions.DEFAULT);
ClientCall.Listener<WriteResponse> callListener = new ClientCall.Listener<WriteResponse>() {
boolean complete = false;
boolean callHalfClosed = false;
@Override
public void onReady() {
while (call.isReady()) {
if (complete) {
if (!callHalfClosed) {
call.halfClose();
callHalfClosed = true;
}
return;
}
call.sendMessage(WriteRequest.newBuilder().setResourceName(resourceName).setData(helloWorld).setFinishWrite(true).build());
complete = true;
}
}
};
call.start(callListener, new Metadata());
call.request(1);
verify(simpleBlobStore, times(1)).put(eq(digest.getHash()), eq(digest.getSizeBytes()), any(InputStream.class));
}
use of com.google.bytestream.ByteStreamProto.WriteResponse in project bazel-buildfarm by bazelbuild.
the class WriteStreamObserver method commitActive.
void commitActive(long committedSize) {
WriteResponse response = WriteResponse.newBuilder().setCommittedSize(committedSize).build();
if (exception.compareAndSet(null, null)) {
try {
logger.log(Level.FINER, format("delivering committed_size for %s of %d", name, committedSize));
responseObserver.onNext(response);
responseObserver.onCompleted();
} catch (Exception e) {
logger.log(Level.SEVERE, format("error delivering committed_size to %s", name), e);
}
}
}
use of com.google.bytestream.ByteStreamProto.WriteResponse in project bazel-buildfarm by bazelbuild.
the class ByteStreamServiceTest method writeCanBeResumed.
@Test
public void writeCanBeResumed() throws IOException, InterruptedException {
ByteString helloWorld = ByteString.copyFromUtf8("Hello, World!");
Digest digest = DIGEST_UTIL.compute(helloWorld);
String uuid = UUID.randomUUID().toString();
String resourceName = createBlobUploadResourceName(uuid, digest);
Channel channel = InProcessChannelBuilder.forName(fakeServerName).directExecutor().build();
ClientCall<WriteRequest, WriteResponse> initialCall = channel.newCall(ByteStreamGrpc.getWriteMethod(), CallOptions.DEFAULT);
ByteString initialData = helloWorld.substring(0, 6);
ClientCall.Listener<WriteResponse> initialCallListener = new ClientCall.Listener<WriteResponse>() {
boolean complete = false;
boolean callHalfClosed = false;
@Override
public void onReady() {
while (initialCall.isReady()) {
if (complete) {
if (!callHalfClosed) {
initialCall.halfClose();
callHalfClosed = true;
}
return;
}
initialCall.sendMessage(WriteRequest.newBuilder().setResourceName(resourceName).setData(initialData).build());
complete = true;
}
}
};
initialCall.start(initialCallListener, new Metadata());
initialCall.request(1);
ByteStreamBlockingStub service = ByteStreamGrpc.newBlockingStub(channel);
QueryWriteStatusResponse response = service.queryWriteStatus(QueryWriteStatusRequest.newBuilder().setResourceName(resourceName).build());
assertThat(response.getCommittedSize()).isEqualTo(initialData.size());
assertThat(response.getComplete()).isFalse();
ClientCall<WriteRequest, WriteResponse> finishCall = channel.newCall(ByteStreamGrpc.getWriteMethod(), CallOptions.DEFAULT);
ClientCall.Listener<WriteResponse> finishCallListener = new ClientCall.Listener<WriteResponse>() {
boolean complete = false;
boolean callHalfClosed = false;
@Override
public void onReady() {
while (finishCall.isReady()) {
if (complete) {
if (!callHalfClosed) {
finishCall.halfClose();
callHalfClosed = true;
}
return;
}
finishCall.sendMessage(WriteRequest.newBuilder().setResourceName(resourceName).setWriteOffset(initialData.size()).setData(helloWorld.substring(initialData.size())).setFinishWrite(true).build());
complete = true;
}
}
};
finishCall.start(finishCallListener, new Metadata());
finishCall.request(1);
ArgumentCaptor<InputStream> inputStreamCaptor = ArgumentCaptor.forClass(InputStream.class);
verify(simpleBlobStore, times(1)).put(eq(digest.getHash()), eq(digest.getSizeBytes()), inputStreamCaptor.capture());
InputStream inputStream = inputStreamCaptor.getValue();
assertThat(inputStream.available()).isEqualTo(helloWorld.size());
byte[] data = new byte[helloWorld.size()];
assertThat(inputStream.read(data)).isEqualTo(helloWorld.size());
assertThat(data).isEqualTo(helloWorld.toByteArray());
}
Aggregations