Search in sources :

Example 11 with ReadResponse

use of com.google.bytestream.ByteStreamProto.ReadResponse in project bazel-buildfarm by bazelbuild.

the class ByteStreamHelperTest method newInputThrowsOnNotFound.

@SuppressWarnings("unchecked")
@Test
public void newInputThrowsOnNotFound() {
    String resourceName = "not/found/resource";
    ReadRequest readRequest = ReadRequest.newBuilder().setResourceName(resourceName).build();
    doAnswer(invocation -> {
        StreamObserver<ReadResponse> observer = invocation.getArgument(1);
        observer.onError(Status.NOT_FOUND.asException());
        return null;
    }).when(serviceImpl).read(eq(readRequest), any(StreamObserver.class));
    try (InputStream in = ByteStreamHelper.newInput(resourceName, /* offset=*/
    0, Suppliers.ofInstance(ByteStreamGrpc.newStub(channel)), NO_RETRIES::newBackoff, NO_RETRIES::isRetriable, /* retryService=*/
    null)) {
        fail("should not get here");
    } catch (IOException e) {
        assertThat(e).isInstanceOf(NoSuchFileException.class);
    }
    verify(serviceImpl, times(1)).read(eq(readRequest), any(StreamObserver.class));
}
Also used : StreamObserver(io.grpc.stub.StreamObserver) ReadResponse(com.google.bytestream.ByteStreamProto.ReadResponse) InputStream(java.io.InputStream) NoSuchFileException(java.nio.file.NoSuchFileException) NO_RETRIES(build.buildfarm.common.grpc.Retrier.NO_RETRIES) IOException(java.io.IOException) ReadRequest(com.google.bytestream.ByteStreamProto.ReadRequest) Test(org.junit.Test)

Example 12 with ReadResponse

use of com.google.bytestream.ByteStreamProto.ReadResponse in project bazel-buildfarm by bazelbuild.

the class ByteStreamService method readFrom.

void readFrom(InputStream in, long limit, CallStreamObserver<ReadResponse> target) {
    final class ReadFromOnReadyHandler implements Runnable {

        private final byte[] buf = new byte[CHUNK_SIZE];

        private final boolean unlimited = limit == 0;

        private long remaining = limit;

        private boolean complete = false;

        ReadResponse next() throws IOException {
            int readBytes = in.read(buf, 0, (int) Math.min(remaining, buf.length));
            if (readBytes <= 0) {
                if (readBytes == -1) {
                    if (!unlimited) {
                        throw new UnexpectedEndOfStreamException(remaining, limit);
                    }
                    complete = true;
                }
                return ReadResponse.getDefaultInstance();
            }
            if (readBytes > remaining) {
                logger.log(Level.WARNING, format("read %d bytes, expected %d", readBytes, remaining));
                readBytes = (int) remaining;
            }
            remaining -= readBytes;
            complete = remaining == 0;
            return ReadResponse.newBuilder().setData(ByteString.copyFrom(buf, 0, readBytes)).build();
        }

        @Override
        public void run() {
            if (!complete) {
                copy();
            }
        }

        void copy() {
            try {
                while (target.isReady() && !complete) {
                    ReadResponse response = next();
                    if (response.getData().size() != 0) {
                        target.onNext(response);
                    }
                }
                if (complete) {
                    in.close();
                    target.onCompleted();
                }
            } catch (Exception e) {
                complete = true;
                try {
                    in.close();
                } catch (IOException closeEx) {
                    e.addSuppressed(closeEx);
                }
                if (e instanceof UnexpectedEndOfStreamException) {
                    target.onError(Status.UNAVAILABLE.withCause(e).asException());
                } else {
                    target.onError(e);
                }
            }
        }
    }
    target.setOnReadyHandler(new ReadFromOnReadyHandler());
}
Also used : ReadResponse(com.google.bytestream.ByteStreamProto.ReadResponse) IOException(java.io.IOException) NoSuchFileException(java.nio.file.NoSuchFileException) InvalidResourceNameException(build.buildfarm.common.UrlPath.InvalidResourceNameException) EntryLimitException(build.buildfarm.common.EntryLimitException) IOException(java.io.IOException)

Example 13 with ReadResponse

use of com.google.bytestream.ByteStreamProto.ReadResponse in project bazel-buildfarm by bazelbuild.

the class ByteStreamService method onErrorLogReadObserver.

ServerCallStreamObserver<ReadResponse> onErrorLogReadObserver(String name, long offset, ServerCallStreamObserver<ReadResponse> delegate) {
    return new UniformDelegateServerCallStreamObserver<ReadResponse>(delegate) {

        long responseCount = 0;

        long responseBytes = 0;

        @Override
        public void onNext(ReadResponse response) {
            delegate.onNext(response);
            responseCount++;
            responseBytes += response.getData().size();
        }

        @Override
        public void onError(Throwable t) {
            Status status = Status.fromThrowable(t);
            if (status.getCode() != Code.NOT_FOUND) {
                java.util.logging.Level level = Level.SEVERE;
                if (responseCount > 0 && status.getCode() == Code.DEADLINE_EXCEEDED || status.getCode() == Code.CANCELLED) {
                    level = Level.WARNING;
                }
                String message = format("error reading %s at offset %d", name, offset);
                if (responseCount > 0) {
                    message += format(" after %d responses and %d bytes of content", responseCount, responseBytes);
                }
                logger.log(level, message, t);
            }
            delegate.onError(t);
        }

        @Override
        public void onCompleted() {
            delegate.onCompleted();
        }
    };
}
Also used : Status(io.grpc.Status) ReadResponse(com.google.bytestream.ByteStreamProto.ReadResponse) UniformDelegateServerCallStreamObserver(build.buildfarm.common.grpc.UniformDelegateServerCallStreamObserver) ByteString(com.google.protobuf.ByteString) Level(java.util.logging.Level)

Example 14 with ReadResponse

use of com.google.bytestream.ByteStreamProto.ReadResponse in project bazel-buildfarm by bazelbuild.

the class ByteStreamService method readOperationStream.

void readOperationStream(Instance instance, String resourceName, long offset, long limit, StreamObserver<ReadResponse> responseObserver) {
    try {
        InputStream in = instance.newOperationStreamInput(resourceName, offset, TracingMetadataUtils.fromCurrentContext());
        ServerCallStreamObserver<ReadResponse> target = (ServerCallStreamObserver<ReadResponse>) responseObserver;
        target.setOnCancelHandler(() -> {
            try {
                in.close();
            } catch (IOException e) {
                logger.log(Level.SEVERE, "error closing stream", e);
            }
        });
        readFrom(in, limit, onErrorLogReadObserver(resourceName, offset, target));
    } catch (NoSuchFileException e) {
        responseObserver.onError(NOT_FOUND.asException());
    } catch (IOException e) {
        responseObserver.onError(Status.fromThrowable(e).asException());
    }
}
Also used : UniformDelegateServerCallStreamObserver(build.buildfarm.common.grpc.UniformDelegateServerCallStreamObserver) ServerCallStreamObserver(io.grpc.stub.ServerCallStreamObserver) DelegateServerCallStreamObserver(build.buildfarm.common.grpc.DelegateServerCallStreamObserver) ReadResponse(com.google.bytestream.ByteStreamProto.ReadResponse) InputStream(java.io.InputStream) NoSuchFileException(java.nio.file.NoSuchFileException) IOException(java.io.IOException)

Example 15 with ReadResponse

use of com.google.bytestream.ByteStreamProto.ReadResponse in project tools_remote by bazelbuild.

the class GrpcRemoteCacheTest method testDownloadBlobMultipleChunks.

@Test
public void testDownloadBlobMultipleChunks() throws Exception {
    final GrpcRemoteCache client = newClient();
    final Digest digest = DIGEST_UTIL.computeAsUtf8("abcdefg");
    serviceRegistry.addService(new ByteStreamImplBase() {

        @Override
        public void read(ReadRequest request, StreamObserver<ReadResponse> responseObserver) {
            assertThat(request.getResourceName().contains(digest.getHash())).isTrue();
            responseObserver.onNext(ReadResponse.newBuilder().setData(ByteString.copyFromUtf8("abc")).build());
            responseObserver.onNext(ReadResponse.newBuilder().setData(ByteString.copyFromUtf8("def")).build());
            responseObserver.onNext(ReadResponse.newBuilder().setData(ByteString.copyFromUtf8("g")).build());
            responseObserver.onCompleted();
        }
    });
    assertThat(new String(client.downloadBlob(digest), UTF_8)).isEqualTo("abcdefg");
}
Also used : Digest(build.bazel.remote.execution.v2.Digest) ReadResponse(com.google.bytestream.ByteStreamProto.ReadResponse) ByteStreamImplBase(com.google.bytestream.ByteStreamGrpc.ByteStreamImplBase) ByteString(com.google.protobuf.ByteString) ReadRequest(com.google.bytestream.ByteStreamProto.ReadRequest) Test(org.junit.Test)

Aggregations

ReadResponse (com.google.bytestream.ByteStreamProto.ReadResponse)15 ReadRequest (com.google.bytestream.ByteStreamProto.ReadRequest)11 ByteString (com.google.protobuf.ByteString)10 Test (org.junit.Test)9 Digest (build.bazel.remote.execution.v2.Digest)8 ByteStreamImplBase (com.google.bytestream.ByteStreamGrpc.ByteStreamImplBase)7 IOException (java.io.IOException)7 InputStream (java.io.InputStream)6 Status (io.grpc.Status)5 NoSuchFileException (java.nio.file.NoSuchFileException)5 Instance (build.buildfarm.instance.Instance)3 ByteArrayOutputStream (java.io.ByteArrayOutputStream)3 OutputStream (java.io.OutputStream)3 UniformDelegateServerCallStreamObserver (build.buildfarm.common.grpc.UniformDelegateServerCallStreamObserver)2 ByteStreamUploader (build.buildfarm.instance.stub.ByteStreamUploader)2 StreamObserver (io.grpc.stub.StreamObserver)2 ExecutionException (java.util.concurrent.ExecutionException)2 AtomicReference (java.util.concurrent.atomic.AtomicReference)2 EntryLimitException (build.buildfarm.common.EntryLimitException)1 InvalidResourceNameException (build.buildfarm.common.UrlPath.InvalidResourceNameException)1