Examples with CasDownloadReply com.google.devtools.build.lib.remote.RemoteProtocol.CasDownloadReply used on opensource projects

Example 1 with CasDownloadReply

use of com.google.devtools.build.lib.remote.RemoteProtocol.CasDownloadReply in project bazel by bazelbuild.

the class GrpcActionCache method downloadAllResults.

/**
   * Download all results of a remotely executed action locally. TODO(olaola): will need to amend to
   * include the {@link com.google.devtools.build.lib.remote.TreeNodeRepository} for updating.
   */
@Override
public void downloadAllResults(ActionResult result, Path execRoot) throws IOException, CacheNotFoundException {
    // Send all the file requests in a single synchronous batch.
    // TODO(olaola): profile to maybe replace with separate concurrent requests.
    CasDownloadBlobRequest.Builder request = CasDownloadBlobRequest.newBuilder();
    Map<ContentDigest, Pair<Path, FileMetadata>> metadataMap = new HashMap<>();
    for (Output output : result.getOutputList()) {
        Path path = execRoot.getRelative(output.getPath());
        if (output.getContentCase() == ContentCase.FILE_METADATA) {
            FileMetadata fileMetadata = output.getFileMetadata();
            ContentDigest digest = fileMetadata.getDigest();
            if (digest.getSizeBytes() > 0) {
                request.addDigest(digest);
                metadataMap.put(digest, Pair.of(path, fileMetadata));
            } else {
                // Handle empty file locally.
                FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
                FileSystemUtils.writeContent(path, new byte[0]);
            }
        } else {
            downloadTree(output.getDigest(), path);
        }
    }
    Iterator<CasDownloadReply> replies = getBlockingStub().downloadBlob(request.build());
    Set<ContentDigest> results = new HashSet<>();
    while (replies.hasNext()) {
        results.add(createFileFromStream(metadataMap, replies));
    }
    for (ContentDigest digest : metadataMap.keySet()) {
        if (!results.contains(digest)) {
            throw new CacheNotFoundException(digest);
        }
    }
}

Example 2 with CasDownloadReply

use of com.google.devtools.build.lib.remote.RemoteProtocol.CasDownloadReply in project bazel by bazelbuild.

the class GrpcActionCache method createFileFromStream.

private ContentDigest createFileFromStream(Map<ContentDigest, Pair<Path, FileMetadata>> metadataMap, Iterator<CasDownloadReply> replies) throws IOException, CacheNotFoundException {
    Preconditions.checkArgument(replies.hasNext());
    CasDownloadReply reply = replies.next();
    if (reply.hasStatus()) {
        handleDownloadStatus(reply.getStatus());
    }
    BlobChunk chunk = reply.getData();
    ContentDigest digest = chunk.getDigest();
    Preconditions.checkArgument(metadataMap.containsKey(digest));
    Pair<Path, FileMetadata> metadata = metadataMap.get(digest);
    Path path = metadata.first;
    FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
    try (OutputStream stream = path.getOutputStream()) {
        ByteString data = chunk.getData();
        data.writeTo(stream);
        long bytesLeft = digest.getSizeBytes() - data.size();
        while (bytesLeft > 0) {
            Preconditions.checkArgument(replies.hasNext());
            reply = replies.next();
            if (reply.hasStatus()) {
                handleDownloadStatus(reply.getStatus());
            }
            chunk = reply.getData();
            data = chunk.getData();
            Preconditions.checkArgument(!chunk.hasDigest());
            Preconditions.checkArgument(chunk.getOffset() == digest.getSizeBytes() - bytesLeft);
            data.writeTo(stream);
            bytesLeft -= data.size();
        }
        path.setExecutable(metadata.second.getExecutable());
    }
    return digest;
}

Example 3 with CasDownloadReply

use of com.google.devtools.build.lib.remote.RemoteProtocol.CasDownloadReply in project bazel by bazelbuild.

the class GrpcActionCache method downloadFileContents.

/**
   * Download a blob keyed by the given digest and write it to the specified path. Set the
   * executable parameter to the specified value.
   */
@Override
public void downloadFileContents(ContentDigest digest, Path dest, boolean executable) throws IOException, CacheNotFoundException {
    // Send all the file requests in a single synchronous batch.
    // TODO(olaola): profile to maybe replace with separate concurrent requests.
    CasDownloadBlobRequest.Builder request = CasDownloadBlobRequest.newBuilder().addDigest(digest);
    Iterator<CasDownloadReply> replies = getBlockingStub().downloadBlob(request.build());
    FileMetadata fileMetadata = FileMetadata.newBuilder().setDigest(digest).setExecutable(executable).build();
    Map<ContentDigest, Pair<Path, FileMetadata>> metadataMap = new HashMap<>();
    metadataMap.put(digest, Pair.of(dest, fileMetadata));
    createFileFromStream(metadataMap, replies);
}

Example 4 with CasDownloadReply

use of com.google.devtools.build.lib.remote.RemoteProtocol.CasDownloadReply in project bazel by bazelbuild.

the class GrpcActionCache method downloadBlobs.

@Override
public ImmutableList<byte[]> downloadBlobs(Iterable<ContentDigest> digests) throws CacheNotFoundException {
    // Send all the file requests in a single synchronous batch.
    // TODO(olaola): profile to maybe replace with separate concurrent requests.
    CasDownloadBlobRequest.Builder request = CasDownloadBlobRequest.newBuilder();
    for (ContentDigest digest : digests) {
        if (digest.getSizeBytes() > 0) {
            // We handle empty blobs locally.
            request.addDigest(digest);
        }
    }
    Iterator<CasDownloadReply> replies = null;
    Map<ContentDigest, byte[]> results = new HashMap<>();
    int digestCount = request.getDigestCount();
    if (digestCount > 0) {
        replies = getBlockingStub().downloadBlob(request.build());
        while (digestCount-- > 0) {
            Preconditions.checkArgument(replies.hasNext());
            CasDownloadReply reply = replies.next();
            if (reply.hasStatus()) {
                handleDownloadStatus(reply.getStatus());
            }
            BlobChunk chunk = reply.getData();
            ContentDigest digest = chunk.getDigest();
            // This is not enough, but better than nothing.
            Preconditions.checkArgument(digest.getSizeBytes() / 1000.0 < MAX_MEMORY_KBYTES);
            byte[] result = new byte[(int) digest.getSizeBytes()];
            ByteString data = chunk.getData();
            data.copyTo(result, 0);
            int offset = data.size();
            while (offset < result.length) {
                Preconditions.checkArgument(replies.hasNext());
                reply = replies.next();
                if (reply.hasStatus()) {
                    handleDownloadStatus(reply.getStatus());
                }
                chunk = reply.getData();
                Preconditions.checkArgument(!chunk.hasDigest());
                Preconditions.checkArgument(chunk.getOffset() == offset);
                data = chunk.getData();
                data.copyTo(result, offset);
                offset += data.size();
            }
            results.put(digest, result);
        }
    }
    ArrayList<byte[]> result = new ArrayList<>();
    for (ContentDigest digest : digests) {
        if (digest.getSizeBytes() == 0) {
            result.add(new byte[0]);
            continue;
        }
        if (!results.containsKey(digest)) {
            throw new CacheNotFoundException(digest);
        }
        result.add(results.get(digest));
    }
    return ImmutableList.copyOf(result);
}