Examples with ReadRequest com.google.bytestream.ByteStreamProto.ReadRequest used on opensource projects

Example 6 with ReadRequest

use of com.google.bytestream.ByteStreamProto.ReadRequest in project bazel-buildfarm by bazelbuild.

the class ByteStreamServiceTest method readSlicesLargeChunksFromInstance.

@SuppressWarnings("unchecked")
@Test
public void readSlicesLargeChunksFromInstance() throws Exception {
    // pick a large chunk size
    long size = CHUNK_SIZE * 10 + CHUNK_SIZE - 47;
    ByteString content;
    try (ByteString.Output out = ByteString.newOutput(ByteStreamService.CHUNK_SIZE * 10 + ByteStreamService.CHUNK_SIZE - 47)) {
        for (long i = 0; i < size; i++) {
            out.write((int) (i & 0xff));
        }
        content = out.toByteString();
    }
    Digest digest = DIGEST_UTIL.compute(content);
    String resourceName = "blobs/" + DigestUtil.toString(digest);
    ReadRequest request = ReadRequest.newBuilder().setResourceName(resourceName).build();
    doAnswer(answerVoid((blobDigest, offset, limit, chunkObserver, metadata) -> {
    })).when(instance).getBlob(eq(digest), eq(request.getReadOffset()), eq((long) content.size()), any(ServerCallStreamObserver.class), eq(RequestMetadata.getDefaultInstance()));
    Channel channel = InProcessChannelBuilder.forName(fakeServerName).directExecutor().build();
    ByteStreamStub service = ByteStreamGrpc.newStub(channel);
    CountingReadObserver readObserver = new CountingReadObserver();
    service.read(request, readObserver);
    ArgumentCaptor<ServerCallStreamObserver<ByteString>> observerCaptor = ArgumentCaptor.forClass(ServerCallStreamObserver.class);
    verify(instance, times(1)).getBlob(eq(digest), eq(request.getReadOffset()), eq((long) content.size()), observerCaptor.capture(), eq(RequestMetadata.getDefaultInstance()));
    StreamObserver<ByteString> responseObserver = observerCaptor.getValue();
    // supply entire content
    responseObserver.onNext(content);
    responseObserver.onCompleted();
    assertThat(readObserver.isCompleted()).isTrue();
    assertThat(readObserver.getData()).isEqualTo(content);
    List<Integer> sizes = readObserver.getSizesList();
    // 10 + 1 incomplete chunk
    assertThat(sizes.size()).isEqualTo(11);
    assertThat(sizes.stream().filter((responseSize) -> responseSize > CHUNK_SIZE).collect(Collectors.toList())).isEmpty();
}

Example 7 with ReadRequest

use of com.google.bytestream.ByteStreamProto.ReadRequest in project bazel-buildfarm by bazelbuild.

the class ByteStreamServiceTest method missingBlobReadIsNotFound.

@Test
public void missingBlobReadIsNotFound() {
    ByteString helloWorld = ByteString.copyFromUtf8("Hello, World!");
    Digest digest = DIGEST_UTIL.compute(helloWorld);
    Channel channel = InProcessChannelBuilder.forName(fakeServerName).directExecutor().build();
    ByteStreamBlockingStub service = ByteStreamGrpc.newBlockingStub(channel);
    when(simpleBlobStore.get(eq(digest.getHash()), any(OutputStream.class))).thenReturn(immediateFuture(false));
    ReadRequest request = ReadRequest.newBuilder().setResourceName(createBlobDownloadResourceName(digest)).build();
    StatusRuntimeException notFoundException = null;
    try {
        if (service.read(request).hasNext()) {
            fail("no responses should be available");
        }
    } catch (StatusRuntimeException e) {
        assertThat(Status.fromThrowable(e).getCode()).isEqualTo(Code.NOT_FOUND);
        notFoundException = e;
    }
    assertThat(notFoundException).isNotNull();
}

Example 8 with ReadRequest

use of com.google.bytestream.ByteStreamProto.ReadRequest in project bazel-buildfarm by bazelbuild.

the class ByteStreamServiceTest method skippedInputIsNotInResponse.

@Test
public void skippedInputIsNotInResponse() throws ExecutionException, IOException, InterruptedException {
    ByteString helloWorld = ByteString.copyFromUtf8("Hello, World!");
    Digest digest = DIGEST_UTIL.compute(helloWorld);
    Channel channel = InProcessChannelBuilder.forName(fakeServerName).directExecutor().build();
    ByteStreamStub service = ByteStreamGrpc.newStub(channel);
    SettableFuture<Boolean> getComplete = SettableFuture.create();
    when(simpleBlobStore.get(eq(digest.getHash()), any(OutputStream.class))).thenReturn(getComplete);
    ArgumentCaptor<OutputStream> outputStreamCaptor = ArgumentCaptor.forClass(OutputStream.class);
    ReadRequest request = ReadRequest.newBuilder().setResourceName(createBlobDownloadResourceName(digest)).setReadOffset(6).build();
    SettableFuture<ByteString> readComplete = SettableFuture.create();
    service.read(request, new StreamObserver<ReadResponse>() {

        ByteString content = ByteString.EMPTY;

        @Override
        public void onNext(ReadResponse response) {
            content = content.concat(response.getData());
        }

        @Override
        public void onError(Throwable t) {
            readComplete.setException(t);
        }

        @Override
        public void onCompleted() {
            readComplete.set(content);
        }
    });
    verify(simpleBlobStore, times(1)).get(eq(digest.getHash()), outputStreamCaptor.capture());
    try (OutputStream outputStream = outputStreamCaptor.getValue()) {
        outputStream.write(helloWorld.toByteArray());
        getComplete.set(true);
    }
    assertThat(readComplete.get()).isEqualTo(helloWorld.substring(6));
}

Example 9 with ReadRequest

use of com.google.bytestream.ByteStreamProto.ReadRequest in project bazel-buildfarm by bazelbuild.

the class ByteStreamHelper method newInput.

@SuppressWarnings("Guava")
public static InputStream newInput(String resourceName, long offset, Supplier<ByteStreamStub> bsStubSupplier, Supplier<Backoff> backoffSupplier, Predicate<Status> isRetriable, @Nullable ListeningScheduledExecutorService retryService) throws IOException {
    ReadRequest request = ReadRequest.newBuilder().setResourceName(resourceName).setReadOffset(offset).build();
    BlockingQueue<ByteString> queue = new ArrayBlockingQueue<>(1);
    ByteStringQueueInputStream inputStream = new ByteStringQueueInputStream(queue);
    // this interface needs to operate similar to open, where it
    // throws an exception on creation. We will need to wait around
    // for the response to come back in order to supply the stream or
    // throw the exception it receives
    SettableFuture<InputStream> streamReadyFuture = SettableFuture.create();
    StreamObserver<ReadResponse> responseObserver = new StreamObserver<ReadResponse>() {

        long requestOffset = offset;

        long currentOffset = offset;

        Backoff backoff = backoffSupplier.get();

        @Override
        public void onNext(ReadResponse response) {
            streamReadyFuture.set(inputStream);
            ByteString data = response.getData();
            try {
                queue.put(data);
                currentOffset += data.size();
            } catch (InterruptedException e) {
                // cancel context?
                inputStream.setException(e);
            }
        }

        private void retryRequest() {
            requestOffset = currentOffset;
            bsStubSupplier.get().read(request.toBuilder().setReadOffset(requestOffset).build(), this);
        }

        @Override
        public void onError(Throwable t) {
            Status status = Status.fromThrowable(t);
            long nextDelayMillis = backoff.nextDelayMillis();
            if (status.getCode() == Status.Code.DEADLINE_EXCEEDED && currentOffset != requestOffset) {
                backoff = backoffSupplier.get();
                retryRequest();
            } else if (retryService == null || nextDelayMillis < 0 || !isRetriable.test(status)) {
                streamReadyFuture.setException(t);
                inputStream.setException(t);
            } else {
                try {
                    ListenableFuture<?> schedulingResult = retryService.schedule(this::retryRequest, nextDelayMillis, TimeUnit.MILLISECONDS);
                    schedulingResult.addListener(() -> {
                        try {
                            schedulingResult.get();
                        } catch (ExecutionException e) {
                            inputStream.setException(e.getCause());
                        } catch (InterruptedException e) {
                            inputStream.setException(e);
                        }
                    }, MoreExecutors.directExecutor());
                } catch (RejectedExecutionException e) {
                    inputStream.setException(e);
                }
            }
        }

        @Override
        public void onCompleted() {
            inputStream.setCompleted();
        }
    };
    bsStubSupplier.get().read(request, responseObserver);
    // perfectly reasonable to be used as a wait point
    try {
        return streamReadyFuture.get();
    } catch (InterruptedException e) {
        try {
            inputStream.close();
        } catch (RuntimeException closeEx) {
            e.addSuppressed(e);
        }
        IOException ioEx = new ClosedByInterruptException();
        ioEx.addSuppressed(e);
        throw ioEx;
    } catch (ExecutionException e) {
        Throwable cause = e.getCause();
        Status status = Status.fromThrowable(cause);
        if (status.getCode() == Status.Code.NOT_FOUND) {
            IOException ioEx = new NoSuchFileException(resourceName);
            ioEx.addSuppressed(cause);
            throw ioEx;
        }
        Throwables.throwIfInstanceOf(cause, IOException.class);
        throw new IOException(cause);
    }
}

Example 10 with ReadRequest

use of com.google.bytestream.ByteStreamProto.ReadRequest in project bazel-buildfarm by bazelbuild.

the class StubInstanceTest method inputStreamThrowsOnDeadlineExceededWithoutProgress.

@Test
public void inputStreamThrowsOnDeadlineExceededWithoutProgress() throws IOException, InterruptedException {
    serviceRegistry.addService(new ByteStreamImplBase() {

        @Override
        public void read(ReadRequest request, StreamObserver<ReadResponse> responseObserver) {
            responseObserver.onError(Status.DEADLINE_EXCEEDED.asException());
        }
    });
    OutputStream out = mock(OutputStream.class);
    IOException ioException = null;
    Instance instance = newStubInstance("input-stream-deadline-exceeded");
    Digest timeoutDigest = Digest.newBuilder().setHash("timeout-blob-name").setSizeBytes(1).build();
    try (InputStream in = instance.newBlobInput(timeoutDigest, 0, 1, SECONDS, RequestMetadata.getDefaultInstance())) {
        ByteStreams.copy(in, out);
    } catch (IOException e) {
        ioException = e;
    }
    assertThat(ioException).isNotNull();
    Status status = Status.fromThrowable(ioException);
    assertThat(status.getCode()).isEqualTo(Code.DEADLINE_EXCEEDED);
    verifyZeroInteractions(out);
    instance.stop();
}