Examples with ContentAddressableStorage build.buildfarm.cas.ContentAddressableStorage used on opensource projects

Example 1 with ContentAddressableStorage

use of build.buildfarm.cas.ContentAddressableStorage in project bazel-buildfarm by bazelbuild.

the class CASFileCacheTest method newInputThrowsNoSuchFileExceptionWithoutDelegate.

@Test
public void newInputThrowsNoSuchFileExceptionWithoutDelegate() throws Exception {
    ContentAddressableStorage undelegatedCAS = new CASFileCache(root, /* maxSizeInBytes=*/
    1024, /* maxEntrySizeInBytes=*/
    1024, /* hexBucketLevels=*/
    1, storeFileDirsIndexInMemory, DIGEST_UTIL, expireService, /* accessRecorder=*/
    directExecutor(), storage, /* directoriesIndexDbName=*/
    ":memory:", /* onPut=*/
    digest -> {
    }, /* onExpire=*/
    digests -> {
    }, /* delegate=*/
    null) {

        @Override
        protected InputStream newExternalInput(Digest digest) throws IOException {
            ByteString content = blobs.get(digest);
            if (content == null) {
                return fileCache.newTransparentInput(digest, 0);
            }
            return content.substring((int) (long) 0).newInput();
        }
    };
    ByteString blob = ByteString.copyFromUtf8("Missing Entry");
    Digest blobDigest = DIGEST_UTIL.compute(blob);
    NoSuchFileException expected = null;
    try (InputStream in = undelegatedCAS.newInput(blobDigest, /* offset=*/
    0)) {
        fail("should not get here");
    } catch (NoSuchFileException e) {
        expected = e;
    }
    assertThat(expected).isNotNull();
}

Example 2 with ContentAddressableStorage

use of build.buildfarm.cas.ContentAddressableStorage in project bazel-buildfarm by bazelbuild.

the class AbstractServerInstanceTest method outputDirectoriesFilesAreEnsuredPresent.

@SuppressWarnings("unchecked")
@Test
public void outputDirectoriesFilesAreEnsuredPresent() throws Exception {
    // our test subjects - these should appear in the findMissingBlobs request
    Digest fileDigest = DIGEST_UTIL.compute(ByteString.copyFromUtf8("Output Directory Root File Content"));
    Digest childFileDigest = DIGEST_UTIL.compute(ByteString.copyFromUtf8("Output Directory Child File Content"));
    Digest otherFileDigest = DIGEST_UTIL.compute(ByteString.copyFromUtf8("Another Output Directory File Content"));
    // setup block and ensureOutputsPresent trigger
    RequestMetadata requestMetadata = RequestMetadata.newBuilder().setCorrelatedInvocationsId("https://localhost:12345/test/build?ENSURE_OUTPUTS_PRESENT=true#92af266a-c5bf-48ca-a723-344ae516a786").build();
    ContentAddressableStorage contentAddressableStorage = mock(ContentAddressableStorage.class);
    ActionCache actionCache = mock(ActionCache.class);
    AbstractServerInstance instance = new DummyServerInstance(contentAddressableStorage, actionCache);
    Tree tree = Tree.newBuilder().setRoot(Directory.newBuilder().addFiles(FileNode.newBuilder().setDigest(fileDigest)).build()).addChildren(Directory.newBuilder().addFiles(FileNode.newBuilder().setDigest(childFileDigest)).build()).build();
    Tree otherTree = Tree.newBuilder().setRoot(Directory.newBuilder().addFiles(FileNode.newBuilder().setDigest(otherFileDigest)).build()).build();
    Digest treeDigest = DIGEST_UTIL.compute(tree);
    doBlob(contentAddressableStorage, treeDigest, tree.toByteString(), requestMetadata);
    Digest otherTreeDigest = DIGEST_UTIL.compute(otherTree);
    doBlob(contentAddressableStorage, otherTreeDigest, otherTree.toByteString(), requestMetadata);
    ActionKey actionKey = DigestUtil.asActionKey(DIGEST_UTIL.compute(ByteString.copyFromUtf8("action")));
    ActionResult actionResult = ActionResult.newBuilder().addOutputDirectories(OutputDirectory.newBuilder().setTreeDigest(treeDigest).build()).addOutputDirectories(OutputDirectory.newBuilder().setTreeDigest(otherTreeDigest).build()).build();
    when(actionCache.get(actionKey)).thenReturn(immediateFuture(actionResult));
    // invocation
    assertThat(instance.getActionResult(actionKey, requestMetadata).get()).isEqualTo(actionResult);
    // validation
    ArgumentCaptor<Iterable<Digest>> findMissingBlobsCaptor = ArgumentCaptor.forClass(Iterable.class);
    verify(contentAddressableStorage, times(1)).get(eq(treeDigest), /* offset=*/
    eq(0L), eq(treeDigest.getSizeBytes()), any(ServerCallStreamObserver.class), eq(requestMetadata));
    verify(contentAddressableStorage, times(1)).findMissingBlobs(findMissingBlobsCaptor.capture());
    assertThat(findMissingBlobsCaptor.getValue()).containsAtLeast(fileDigest, childFileDigest, otherFileDigest);
}

Example 3 with ContentAddressableStorage

use of build.buildfarm.cas.ContentAddressableStorage in project bazel-buildfarm by bazelbuild.

the class Worker method createFuseExecFileSystem.

private ExecFileSystem createFuseExecFileSystem(InputStreamFactory remoteInputStreamFactory, ContentAddressableStorage storage) {
    InputStreamFactory storageInputStreamFactory = (digest, offset) -> storage.get(digest).getData().substring((int) offset).newInput();
    InputStreamFactory localPopulatingInputStreamFactory = (blobDigest, offset) -> {
        // FIXME use write
        ByteString content = ByteString.readFrom(remoteInputStreamFactory.newInput(blobDigest, offset));
        if (offset == 0) {
            // extra computations
            Blob blob = new Blob(content, digestUtil);
            // here's hoping that our digest matches...
            storage.put(blob);
        }
        return content.newInput();
    };
    return new FuseExecFileSystem(root, new FuseCAS(root, new EmptyInputStreamFactory(new FailoverInputStreamFactory(storageInputStreamFactory, localPopulatingInputStreamFactory))), storage);
}

Example 4 with ContentAddressableStorage

use of build.buildfarm.cas.ContentAddressableStorage in project bazel-buildfarm by bazelbuild.

the class Worker method createStorages.

private ContentAddressableStorage createStorages(InputStreamFactory remoteInputStreamFactory, ExecutorService removeDirectoryService, Executor accessRecorder, List<ContentAddressableStorageConfig> configs) throws ConfigurationException {
    ImmutableList.Builder<ContentAddressableStorage> storages = ImmutableList.builder();
    // must construct delegates first
    ContentAddressableStorage storage = null;
    ContentAddressableStorage delegate = null;
    for (ContentAddressableStorageConfig config : Lists.reverse(configs)) {
        storage = createStorage(remoteInputStreamFactory, removeDirectoryService, accessRecorder, config, delegate);
        storages.add(storage);
        delegate = storage;
    }
    return storage;
}