Examples with FindMissingBlobsResponse build.bazel.remote.execution.v2.FindMissingBlobsResponse used on opensource projects

Example 1 with FindMissingBlobsResponse

use of build.bazel.remote.execution.v2.FindMissingBlobsResponse in project bazel-buildfarm by bazelbuild.

the class Executor method findMissingBlobs.

private static List<Digest> findMissingBlobs(String instanceName, Path blobsDir, ContentAddressableStorageBlockingStub casStub) throws IOException {
    FindMissingBlobsRequest.Builder request = FindMissingBlobsRequest.newBuilder().setInstanceName(instanceName);
    int size = 0;
    ImmutableList.Builder<Digest> missingDigests = ImmutableList.builder();
    System.out.println("Looking for missing blobs");
    final int messagesPerRequest = 2 * 1024 * 1024 / 80;
    System.out.println("Looking for missing blobs");
    Stopwatch stopwatch = Stopwatch.createUnstarted();
    FileStore fileStore = Files.getFileStore(blobsDir);
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(blobsDir)) {
        for (Path file : stream) {
            FileStatus stat = stat(file, /* followSymlinks=*/
            false, fileStore);
            Digest digest = DigestUtil.buildDigest(file.getFileName().toString().split("_")[0], stat.getSize());
            request.addBlobDigests(digest);
            size++;
            if (size == messagesPerRequest) {
                stopwatch.reset().start();
                FindMissingBlobsResponse response = casStub.findMissingBlobs(request.build());
                System.out.println("Found " + response.getMissingBlobDigestsCount() + " missing digests in " + (stopwatch.elapsed(MICROSECONDS) / 1000.0) + "ms");
                missingDigests.addAll(response.getMissingBlobDigestsList());
                request = FindMissingBlobsRequest.newBuilder().setInstanceName(instanceName);
                size = 0;
            }
        }
    }
    if (size > 0) {
        FindMissingBlobsResponse response = casStub.findMissingBlobs(request.build());
        System.out.println("Found " + response.getMissingBlobDigestsCount() + " missing digests");
        missingDigests.addAll(response.getMissingBlobDigestsList());
    }
    return missingDigests.build();
}

Example 2 with FindMissingBlobsResponse

use of build.bazel.remote.execution.v2.FindMissingBlobsResponse in project bazel-buildfarm by bazelbuild.

the class ContentAddressableStorageService method findMissingBlobs.

@Override
public void findMissingBlobs(FindMissingBlobsRequest request, StreamObserver<FindMissingBlobsResponse> responseObserver) {
    FindMissingBlobsResponse.Builder responseBuilder = FindMissingBlobsResponse.newBuilder();
    try {
        for (Digest blobDigest : request.getBlobDigestsList()) {
            if (!simpleBlobStore.containsKey(blobDigest.getHash())) {
                responseBuilder.addMissingBlobDigests(blobDigest);
            }
        }
        responseObserver.onNext(responseBuilder.build());
        responseObserver.onCompleted();
    } catch (IOException e) {
        responseObserver.onError(Status.fromThrowable(e).asException());
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
}

Example 3 with FindMissingBlobsResponse

use of build.bazel.remote.execution.v2.FindMissingBlobsResponse in project bazel-buildfarm by bazelbuild.

the class BuildFarmServerTest method findMissingBlobs.

@Test
public void findMissingBlobs() {
    DigestUtil digestUtil = new DigestUtil(HashFunction.SHA256);
    ByteString content = ByteString.copyFromUtf8("Hello, World!");
    Iterable<Digest> digests = Collections.singleton(digestUtil.compute(content));
    FindMissingBlobsRequest request = FindMissingBlobsRequest.newBuilder().setInstanceName(INSTANCE_NAME).addAllBlobDigests(digests).build();
    ContentAddressableStorageGrpc.ContentAddressableStorageBlockingStub stub = ContentAddressableStorageGrpc.newBlockingStub(inProcessChannel);
    FindMissingBlobsResponse response = stub.findMissingBlobs(request);
    assertThat(response.getMissingBlobDigestsList()).containsExactlyElementsIn(digests);
}

Example 4 with FindMissingBlobsResponse

use of build.bazel.remote.execution.v2.FindMissingBlobsResponse in project bazel-buildfarm by bazelbuild.

the class StubInstanceTest method findMissingBlobsCallsFindMissingBlobs.

@Test
public void findMissingBlobsCallsFindMissingBlobs() throws ExecutionException, InterruptedException {
    AtomicReference<FindMissingBlobsRequest> reference = new AtomicReference<>();
    serviceRegistry.addService(new ContentAddressableStorageImplBase() {

        @Override
        public void findMissingBlobs(FindMissingBlobsRequest request, StreamObserver<FindMissingBlobsResponse> responseObserver) {
            reference.set(request);
            responseObserver.onNext(FindMissingBlobsResponse.getDefaultInstance());
            responseObserver.onCompleted();
        }
    });
    Instance instance = newStubInstance("findMissingBlobs-test");
    Iterable<Digest> digests = ImmutableList.of(Digest.newBuilder().setHash("present").setSizeBytes(1).build());
    assertThat(instance.findMissingBlobs(digests, RequestMetadata.getDefaultInstance()).get()).isEmpty();
    instance.stop();
}

Example 5 with FindMissingBlobsResponse

use of build.bazel.remote.execution.v2.FindMissingBlobsResponse in project bazel-buildfarm by bazelbuild.

the class ContentAddressableStorageService method instanceFindMissingBlobs.

void instanceFindMissingBlobs(Instance instance, FindMissingBlobsRequest request, StreamObserver<FindMissingBlobsResponse> responseObserver) {
    Stopwatch stopwatch = Stopwatch.createStarted();
    FindMissingBlobsResponse.Builder builder = FindMissingBlobsResponse.newBuilder();
    ListenableFuture<FindMissingBlobsResponse.Builder> responseFuture = transform(instance.findMissingBlobs(request.getBlobDigestsList(), TracingMetadataUtils.fromCurrentContext()), builder::addAllMissingBlobDigests, directExecutor());
    addCallback(responseFuture, new FutureCallback<FindMissingBlobsResponse.Builder>() {

        @Override
        public void onSuccess(FindMissingBlobsResponse.Builder builder) {
            try {
                FindMissingBlobsResponse response = builder.build();
                responseObserver.onNext(response);
                responseObserver.onCompleted();
                long elapsedMicros = stopwatch.elapsed(MICROSECONDS);
                missingBlobs.observe(request.getBlobDigestsList().size());
                logger.log(Level.FINE, "FindMissingBlobs(" + instance.getName() + ") for " + request.getBlobDigestsList().size() + " blobs in " + elapsedMicros / 1000.0);
            } catch (Throwable t) {
                onFailure(t);
            }
        }

        @SuppressWarnings("NullableProblems")
        @Override
        public void onFailure(Throwable t) {
            Status status = Status.fromThrowable(t);
            if (status.getCode() != Code.CANCELLED) {
                logger.log(Level.SEVERE, format("findMissingBlobs(%s): %d", request.getInstanceName(), request.getBlobDigestsCount()), t);
                responseObserver.onError(t);
            }
        }
    }, directExecutor());
}