Examples with FileMetadata com.google.devtools.build.lib.remote.RemoteProtocol.FileMetadata used on opensource projects

Example 1 with FileMetadata

use of com.google.devtools.build.lib.remote.RemoteProtocol.FileMetadata in project bazel by bazelbuild.

the class ConcurrentMapActionCache method downloadAllResults.

@Override
public void downloadAllResults(ActionResult result, Path execRoot) throws IOException, CacheNotFoundException {
    for (Output output : result.getOutputList()) {
        if (output.getContentCase() == ContentCase.FILE_METADATA) {
            FileMetadata m = output.getFileMetadata();
            downloadFileContents(m.getDigest(), execRoot.getRelative(output.getPath()), m.getExecutable());
        } else {
            downloadTree(output.getDigest(), execRoot.getRelative(output.getPath()));
        }
    }
}

Example 2 with FileMetadata

use of com.google.devtools.build.lib.remote.RemoteProtocol.FileMetadata in project bazel by bazelbuild.

the class ConcurrentMapActionCache method downloadTree.

@Override
public void downloadTree(ContentDigest rootDigest, Path rootLocation) throws IOException, CacheNotFoundException {
    FileNode fileNode = FileNode.parseFrom(downloadBlob(rootDigest));
    if (fileNode.hasFileMetadata()) {
        FileMetadata meta = fileNode.getFileMetadata();
        downloadFileContents(meta.getDigest(), rootLocation, meta.getExecutable());
    }
    for (FileNode.Child child : fileNode.getChildList()) {
        downloadTree(child.getDigest(), rootLocation.getRelative(child.getPath()));
    }
}

Example 3 with FileMetadata

use of com.google.devtools.build.lib.remote.RemoteProtocol.FileMetadata in project bazel by bazelbuild.

the class GrpcActionCache method downloadAllResults.

/**
   * Download all results of a remotely executed action locally. TODO(olaola): will need to amend to
   * include the {@link com.google.devtools.build.lib.remote.TreeNodeRepository} for updating.
   */
@Override
public void downloadAllResults(ActionResult result, Path execRoot) throws IOException, CacheNotFoundException {
    // Send all the file requests in a single synchronous batch.
    // TODO(olaola): profile to maybe replace with separate concurrent requests.
    CasDownloadBlobRequest.Builder request = CasDownloadBlobRequest.newBuilder();
    Map<ContentDigest, Pair<Path, FileMetadata>> metadataMap = new HashMap<>();
    for (Output output : result.getOutputList()) {
        Path path = execRoot.getRelative(output.getPath());
        if (output.getContentCase() == ContentCase.FILE_METADATA) {
            FileMetadata fileMetadata = output.getFileMetadata();
            ContentDigest digest = fileMetadata.getDigest();
            if (digest.getSizeBytes() > 0) {
                request.addDigest(digest);
                metadataMap.put(digest, Pair.of(path, fileMetadata));
            } else {
                // Handle empty file locally.
                FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
                FileSystemUtils.writeContent(path, new byte[0]);
            }
        } else {
            downloadTree(output.getDigest(), path);
        }
    }
    Iterator<CasDownloadReply> replies = getBlockingStub().downloadBlob(request.build());
    Set<ContentDigest> results = new HashSet<>();
    while (replies.hasNext()) {
        results.add(createFileFromStream(metadataMap, replies));
    }
    for (ContentDigest digest : metadataMap.keySet()) {
        if (!results.contains(digest)) {
            throw new CacheNotFoundException(digest);
        }
    }
}

Example 4 with FileMetadata

use of com.google.devtools.build.lib.remote.RemoteProtocol.FileMetadata in project bazel by bazelbuild.

the class GrpcActionCache method createFileFromStream.

private ContentDigest createFileFromStream(Map<ContentDigest, Pair<Path, FileMetadata>> metadataMap, Iterator<CasDownloadReply> replies) throws IOException, CacheNotFoundException {
    Preconditions.checkArgument(replies.hasNext());
    CasDownloadReply reply = replies.next();
    if (reply.hasStatus()) {
        handleDownloadStatus(reply.getStatus());
    }
    BlobChunk chunk = reply.getData();
    ContentDigest digest = chunk.getDigest();
    Preconditions.checkArgument(metadataMap.containsKey(digest));
    Pair<Path, FileMetadata> metadata = metadataMap.get(digest);
    Path path = metadata.first;
    FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
    try (OutputStream stream = path.getOutputStream()) {
        ByteString data = chunk.getData();
        data.writeTo(stream);
        long bytesLeft = digest.getSizeBytes() - data.size();
        while (bytesLeft > 0) {
            Preconditions.checkArgument(replies.hasNext());
            reply = replies.next();
            if (reply.hasStatus()) {
                handleDownloadStatus(reply.getStatus());
            }
            chunk = reply.getData();
            data = chunk.getData();
            Preconditions.checkArgument(!chunk.hasDigest());
            Preconditions.checkArgument(chunk.getOffset() == digest.getSizeBytes() - bytesLeft);
            data.writeTo(stream);
            bytesLeft -= data.size();
        }
        path.setExecutable(metadata.second.getExecutable());
    }
    return digest;
}

Example 5 with FileMetadata

use of com.google.devtools.build.lib.remote.RemoteProtocol.FileMetadata in project bazel by bazelbuild.

the class GrpcActionCache method downloadFileContents.

/**
   * Download a blob keyed by the given digest and write it to the specified path. Set the
   * executable parameter to the specified value.
   */
@Override
public void downloadFileContents(ContentDigest digest, Path dest, boolean executable) throws IOException, CacheNotFoundException {
    // Send all the file requests in a single synchronous batch.
    // TODO(olaola): profile to maybe replace with separate concurrent requests.
    CasDownloadBlobRequest.Builder request = CasDownloadBlobRequest.newBuilder().addDigest(digest);
    Iterator<CasDownloadReply> replies = getBlockingStub().downloadBlob(request.build());
    FileMetadata fileMetadata = FileMetadata.newBuilder().setDigest(digest).setExecutable(executable).build();
    Map<ContentDigest, Pair<Path, FileMetadata>> metadataMap = new HashMap<>();
    metadataMap.put(digest, Pair.of(dest, fileMetadata));
    createFileFromStream(metadataMap, replies);
}