Examples with Chunker build.buildfarm.instance.stub.Chunker used on opensource projects

Example 1 with Chunker

use of build.buildfarm.instance.stub.Chunker in project bazel-buildfarm by bazelbuild.

the class OperationQueueWorkerContext method uploadManifest.

private static void uploadManifest(UploadManifest manifest, ByteStreamUploader uploader) throws IOException, InterruptedException {
    Map<HashCode, Chunker> filesToUpload = Maps.newHashMap();
    Map<Digest, Path> digestToFile = manifest.getDigestToFile();
    Map<Digest, Chunker> digestToChunkers = manifest.getDigestToChunkers();
    Collection<Digest> digests = new ArrayList<>();
    digests.addAll(digestToFile.keySet());
    digests.addAll(digestToChunkers.keySet());
    for (Digest digest : digests) {
        Chunker chunker;
        Path file = digestToFile.get(digest);
        if (file != null) {
            chunker = Chunker.builder().setInput(digest.getSizeBytes(), file).build();
        } else {
            chunker = digestToChunkers.get(digest);
            if (chunker == null) {
                String message = "FindMissingBlobs call returned an unknown digest: " + digest;
                throw new IOException(message);
            }
        }
        filesToUpload.put(HashCode.fromString(digest.getHash()), chunker);
    }
    if (!filesToUpload.isEmpty()) {
        uploader.uploadBlobs(filesToUpload);
    }
}

Example 2 with Chunker

use of build.buildfarm.instance.stub.Chunker in project bazel-buildfarm by bazelbuild.

the class UploadOutputsTest method uploadOutputsUploadsFiles.

@SuppressWarnings("unchecked")
@Test
public void uploadOutputsUploadsFiles() throws IOException, StatusException, InterruptedException {
    Path topdir = root.resolve("foo");
    Files.createDirectory(topdir);
    Path file = topdir.resolve("bar");
    Files.createFile(file);
    // maybe make some files...
    uploadOutputs(ImmutableList.of(), ImmutableList.of("foo"));
    Tree tree = Tree.newBuilder().setRoot(Directory.newBuilder().addFiles(FileNode.newBuilder().setName("bar").setDigest(DIGEST_UTIL.empty()).setIsExecutable(Files.isExecutable(file)).build()).build()).build();
    ByteString treeBlob = tree.toByteString();
    ArgumentCaptor<Map<HashCode, Chunker>> uploadCaptor = ArgumentCaptor.forClass(Map.class);
    verify(mockUploader).uploadBlobs(uploadCaptor.capture());
    Map<HashCode, Chunker> upload = uploadCaptor.getValue();
    Chunker emptyChunker = upload.get(DIGEST_UTIL.computeHash(ByteString.EMPTY));
    assertThat(emptyChunker.next().getData()).isEqualTo(ByteString.EMPTY);
    Chunker treeChunker = upload.get(DIGEST_UTIL.computeHash(treeBlob));
    assertThat(treeChunker.next().getData()).isEqualTo(treeBlob);
    assertThat(resultBuilder.getOutputDirectoriesList()).containsExactly(OutputDirectory.newBuilder().setPath("foo").setTreeDigest(DIGEST_UTIL.compute(tree)).build());
}

Example 3 with Chunker

use of build.buildfarm.instance.stub.Chunker in project bazel-buildfarm by bazelbuild.

the class UploadOutputsTest method uploadOutputsUploadsNestedDirectories.

@SuppressWarnings("unchecked")
@Test
public void uploadOutputsUploadsNestedDirectories() throws IOException, StatusException, InterruptedException {
    Path topdir = root.resolve("foo");
    Files.createDirectory(topdir);
    Path subdir = topdir.resolve("bar");
    Files.createDirectory(subdir);
    Path file = subdir.resolve("baz");
    Files.createFile(file);
    // maybe make some files...
    uploadOutputs(ImmutableList.of(), ImmutableList.of("foo"));
    Directory subDirectory = Directory.newBuilder().addFiles(FileNode.newBuilder().setName("baz").setDigest(DIGEST_UTIL.empty()).setIsExecutable(Files.isExecutable(file)).build()).build();
    Tree tree = Tree.newBuilder().setRoot(Directory.newBuilder().addDirectories(DirectoryNode.newBuilder().setName("bar").setDigest(DIGEST_UTIL.compute(subDirectory)).build()).build()).addChildren(subDirectory).build();
    ByteString treeBlob = tree.toByteString();
    ArgumentCaptor<Map<HashCode, Chunker>> uploadCaptor = ArgumentCaptor.forClass(Map.class);
    verify(mockUploader).uploadBlobs(uploadCaptor.capture());
    Map<HashCode, Chunker> upload = uploadCaptor.getValue();
    Chunker emptyChunker = upload.get(DIGEST_UTIL.computeHash(ByteString.EMPTY));
    assertThat(emptyChunker.next().getData()).isEqualTo(ByteString.EMPTY);
    Chunker treeChunker = upload.get(DIGEST_UTIL.computeHash(treeBlob));
    assertThat(treeChunker.next().getData()).isEqualTo(treeBlob);
    assertThat(resultBuilder.getOutputDirectoriesList()).containsExactly(OutputDirectory.newBuilder().setPath("foo").setTreeDigest(DIGEST_UTIL.compute(tree)).build());
}

Example 4 with Chunker

use of build.buildfarm.instance.stub.Chunker in project bazel-buildfarm by bazelbuild.

the class UploadOutputsTest method uploadOutputsUploadsEmptyOutputDirectories.

@SuppressWarnings("unchecked")
@Test
public void uploadOutputsUploadsEmptyOutputDirectories() throws IOException, StatusException, InterruptedException {
    Files.createDirectory(root.resolve("foo"));
    // maybe make some files...
    uploadOutputs(ImmutableList.of(), ImmutableList.of("foo"));
    Tree emptyTree = Tree.newBuilder().setRoot(Directory.getDefaultInstance()).build();
    ByteString emptyTreeBlob = emptyTree.toByteString();
    ArgumentCaptor<Map<HashCode, Chunker>> uploadCaptor = ArgumentCaptor.forClass(Map.class);
    verify(mockUploader).uploadBlobs(uploadCaptor.capture());
    Map<HashCode, Chunker> upload = uploadCaptor.getValue();
    Chunker chunker = upload.get(DIGEST_UTIL.computeHash(emptyTreeBlob));
    assertThat(chunker.next().getData()).isEqualTo(emptyTreeBlob);
    assertThat(resultBuilder.getOutputDirectoriesList()).containsExactly(OutputDirectory.newBuilder().setPath("foo").setTreeDigest(DIGEST_UTIL.compute(emptyTree)).build());
}

Example 5 with Chunker

use of build.buildfarm.instance.stub.Chunker in project bazel-buildfarm by bazelbuild.

the class UploadManifest method addDirectory.

private void addDirectory(Path dir) throws IllegalStateException, IOException {
    Tree.Builder tree = Tree.newBuilder();
    Directory root = computeDirectory(dir, tree);
    tree.setRoot(root);
    ByteString blob = tree.build().toByteString();
    Digest digest = digestUtil.compute(blob);
    Chunker chunker = Chunker.builder().setInput(blob).build();
    if (result != null) {
        result.addOutputDirectoriesBuilder().setPath(execRoot.relativize(dir).toString()).setTreeDigest(digest);
    }
    digestToChunkers.put(digest, chunker);
}