Examples with ContentDigest com.google.devtools.build.lib.remote.RemoteProtocol.ContentDigest used on opensource projects

Example 16 with ContentDigest

use of com.google.devtools.build.lib.remote.RemoteProtocol.ContentDigest in project bazel by bazelbuild.

the class GrpcActionCache method uploadBlobs.

@Override
public ImmutableList<ContentDigest> uploadBlobs(Iterable<byte[]> blobs) throws InterruptedException {
    ArrayList<ContentDigest> digests = new ArrayList<>();
    for (byte[] blob : blobs) {
        digests.add(ContentDigests.computeDigest(blob));
    }
    ImmutableSet<ContentDigest> missing = getMissingDigests(digests);
    try {
        if (!missing.isEmpty()) {
            uploadChunks(missing.size(), new BlobChunkInlineIterator(missing, blobs.iterator()));
        }
        return ImmutableList.copyOf(digests);
    } catch (IOException e) {
        // This will never happen.
        throw new RuntimeException(e);
    }
}

Example 17 with ContentDigest

use of com.google.devtools.build.lib.remote.RemoteProtocol.ContentDigest in project bazel by bazelbuild.

the class GrpcActionCache method downloadBlobs.

@Override
public ImmutableList<byte[]> downloadBlobs(Iterable<ContentDigest> digests) throws CacheNotFoundException {
    // Send all the file requests in a single synchronous batch.
    // TODO(olaola): profile to maybe replace with separate concurrent requests.
    CasDownloadBlobRequest.Builder request = CasDownloadBlobRequest.newBuilder();
    for (ContentDigest digest : digests) {
        if (digest.getSizeBytes() > 0) {
            // We handle empty blobs locally.
            request.addDigest(digest);
        }
    }
    Iterator<CasDownloadReply> replies = null;
    Map<ContentDigest, byte[]> results = new HashMap<>();
    int digestCount = request.getDigestCount();
    if (digestCount > 0) {
        replies = getBlockingStub().downloadBlob(request.build());
        while (digestCount-- > 0) {
            Preconditions.checkArgument(replies.hasNext());
            CasDownloadReply reply = replies.next();
            if (reply.hasStatus()) {
                handleDownloadStatus(reply.getStatus());
            }
            BlobChunk chunk = reply.getData();
            ContentDigest digest = chunk.getDigest();
            // This is not enough, but better than nothing.
            Preconditions.checkArgument(digest.getSizeBytes() / 1000.0 < MAX_MEMORY_KBYTES);
            byte[] result = new byte[(int) digest.getSizeBytes()];
            ByteString data = chunk.getData();
            data.copyTo(result, 0);
            int offset = data.size();
            while (offset < result.length) {
                Preconditions.checkArgument(replies.hasNext());
                reply = replies.next();
                if (reply.hasStatus()) {
                    handleDownloadStatus(reply.getStatus());
                }
                chunk = reply.getData();
                Preconditions.checkArgument(!chunk.hasDigest());
                Preconditions.checkArgument(chunk.getOffset() == offset);
                data = chunk.getData();
                data.copyTo(result, offset);
                offset += data.size();
            }
            results.put(digest, result);
        }
    }
    ArrayList<byte[]> result = new ArrayList<>();
    for (ContentDigest digest : digests) {
        if (digest.getSizeBytes() == 0) {
            result.add(new byte[0]);
            continue;
        }
        if (!results.containsKey(digest)) {
            throw new CacheNotFoundException(digest);
        }
        result.add(results.get(digest));
    }
    return ImmutableList.copyOf(result);
}

Example 18 with ContentDigest

use of com.google.devtools.build.lib.remote.RemoteProtocol.ContentDigest in project bazel by bazelbuild.

the class GrpcActionCacheTest method testDownloadBlobsBatchChunk.

@Test
public void testDownloadBlobsBatchChunk() throws Exception {
    RemoteOptions options = Options.getDefaults(RemoteOptions.class);
    options.grpcMaxBatchInputs = 10;
    options.grpcMaxChunkSizeBytes = 2;
    options.grpcMaxBatchSizeBytes = 10;
    options.grpcTimeoutSeconds = 10;
    GrpcActionCache client = new GrpcActionCache(channel, options);
    ContentDigest fooDigest = fakeRemoteCacheService.put("fooooooo".getBytes(UTF_8));
    ContentDigest barDigest = fakeRemoteCacheService.put("baaaar".getBytes(UTF_8));
    ContentDigest s1Digest = fakeRemoteCacheService.put("1".getBytes(UTF_8));
    ContentDigest s2Digest = fakeRemoteCacheService.put("2".getBytes(UTF_8));
    ContentDigest s3Digest = fakeRemoteCacheService.put("3".getBytes(UTF_8));
    ImmutableList<byte[]> results = client.downloadBlobs(ImmutableList.<ContentDigest>of(fooDigest, barDigest, s1Digest, s2Digest, s3Digest));
    assertThat(new String(results.get(0), UTF_8)).isEqualTo("fooooooo");
    assertThat(new String(results.get(1), UTF_8)).isEqualTo("baaaar");
    assertThat(new String(results.get(2), UTF_8)).isEqualTo("1");
    assertThat(new String(results.get(3), UTF_8)).isEqualTo("2");
    assertThat(new String(results.get(4), UTF_8)).isEqualTo("3");
}

Example 19 with ContentDigest

use of com.google.devtools.build.lib.remote.RemoteProtocol.ContentDigest in project bazel by bazelbuild.

the class GrpcActionCacheTest method testUploadBlobsBatchChunk.

@Test
public void testUploadBlobsBatchChunk() throws Exception {
    RemoteOptions options = Options.getDefaults(RemoteOptions.class);
    options.grpcMaxBatchInputs = 10;
    options.grpcMaxChunkSizeBytes = 2;
    options.grpcMaxBatchSizeBytes = 10;
    options.grpcTimeoutSeconds = 10;
    GrpcActionCache client = new GrpcActionCache(channel, options);
    byte[] foo = "fooooooo".getBytes(UTF_8);
    byte[] bar = "baaaar".getBytes(UTF_8);
    byte[] s1 = "1".getBytes(UTF_8);
    byte[] s2 = "2".getBytes(UTF_8);
    byte[] s3 = "3".getBytes(UTF_8);
    ContentDigest fooDigest = ContentDigests.computeDigest(foo);
    ContentDigest barDigest = ContentDigests.computeDigest(bar);
    ContentDigest s1Digest = ContentDigests.computeDigest(s1);
    ContentDigest s2Digest = ContentDigests.computeDigest(s2);
    ContentDigest s3Digest = ContentDigests.computeDigest(s3);
    ImmutableList<ContentDigest> digests = client.uploadBlobs(ImmutableList.<byte[]>of(foo, bar, s1, s2, s3));
    assertThat(digests).containsExactly(fooDigest, barDigest, s1Digest, s2Digest, s3Digest);
    assertThat(fakeRemoteCacheService.get(fooDigest)).isEqualTo(foo);
    assertThat(fakeRemoteCacheService.get(barDigest)).isEqualTo(bar);
    assertThat(fakeRemoteCacheService.get(s1Digest)).isEqualTo(s1);
    assertThat(fakeRemoteCacheService.get(s2Digest)).isEqualTo(s2);
    assertThat(fakeRemoteCacheService.get(s3Digest)).isEqualTo(s3);
}

Example 20 with ContentDigest

use of com.google.devtools.build.lib.remote.RemoteProtocol.ContentDigest in project bazel by bazelbuild.

the class GrpcActionCacheTest method testDownloadAllResults.

@Test
public void testDownloadAllResults() throws Exception {
    GrpcActionCache client = new GrpcActionCache(channel, Options.getDefaults(RemoteOptions.class));
    ContentDigest fooDigest = fakeRemoteCacheService.put("foo".getBytes(UTF_8));
    ContentDigest barDigest = fakeRemoteCacheService.put("bar".getBytes(UTF_8));
    ContentDigest emptyDigest = ContentDigests.computeDigest(new byte[0]);
    ActionResult.Builder result = ActionResult.newBuilder();
    result.addOutputBuilder().setPath("a/foo").getFileMetadataBuilder().setDigest(fooDigest);
    result.addOutputBuilder().setPath("b/empty").getFileMetadataBuilder().setDigest(emptyDigest);
    result.addOutputBuilder().setPath("a/bar").getFileMetadataBuilder().setDigest(barDigest);
    client.downloadAllResults(result.build(), rootDir.getPath());
    Path fooFile = rootDir.getPath().getRelative("a/foo");
    Path emptyFile = rootDir.getPath().getRelative("b/empty");
    Path barFile = rootDir.getPath().getRelative("a/bar");
    assertThat(ContentDigests.computeDigest(fooFile)).isEqualTo(fooDigest);
    assertThat(ContentDigests.computeDigest(emptyFile)).isEqualTo(emptyDigest);
    assertThat(ContentDigests.computeDigest(barFile)).isEqualTo(barDigest);
}