Examples with Digest com.google.cloud.kms.v1.Digest used on opensource projects

Example 36 with Digest

use of com.google.cloud.kms.v1.Digest in project bazel-buildfarm by bazelbuild.

the class CASFileCache method findMissingBlobs.

@Override
public Iterable<Digest> findMissingBlobs(Iterable<Digest> digests) throws InterruptedException {
    ImmutableList.Builder<Digest> builder = ImmutableList.builder();
    ImmutableList.Builder<String> found = ImmutableList.builder();
    for (Digest digest : digests) {
        if (digest.getSizeBytes() != 0 && !containsLocal(digest, null, found::add)) {
            builder.add(digest);
        }
    }
    List<String> foundDigests = found.build();
    if (!foundDigests.isEmpty()) {
        accessed(foundDigests);
    }
    ImmutableList<Digest> missingDigests = builder.build();
    return CasFallbackDelegate.findMissingBlobs(delegate, missingDigests);
}

Example 37 with Digest

use of com.google.cloud.kms.v1.Digest in project bazel-buildfarm by bazelbuild.

the class CASFileCache method charge.

@SuppressWarnings({ "ConstantConditions", "ResultOfMethodCallIgnored" })
private boolean charge(String key, long blobSizeInBytes, AtomicBoolean requiresDischarge) throws IOException, InterruptedException {
    boolean interrupted = false;
    Iterable<ListenableFuture<Digest>> expiredDigestsFutures;
    synchronized (this) {
        if (referenceIfExists(key)) {
            return false;
        }
        sizeInBytes += blobSizeInBytes;
        requiresDischarge.set(true);
        ImmutableList.Builder<ListenableFuture<Digest>> builder = ImmutableList.builder();
        try {
            while (!interrupted && sizeInBytes > maxSizeInBytes) {
                ListenableFuture<Entry> expiredFuture = expireEntry(blobSizeInBytes, expireService);
                interrupted = Thread.interrupted();
                if (expiredFuture != null) {
                    builder.add(transformAsync(expiredFuture, (expiredEntry) -> {
                        String expiredKey = expiredEntry.key;
                        try {
                            Files.delete(getPath(expiredKey));
                        } catch (NoSuchFileException eNoEnt) {
                            logger.log(Level.SEVERE, format("CASFileCache::putImpl: expired key %s did not exist to delete", expiredKey));
                        }
                        FileEntryKey fileEntryKey = parseFileEntryKey(expiredKey, expiredEntry.size);
                        if (fileEntryKey == null) {
                            logger.log(Level.SEVERE, format("error parsing expired key %s", expiredKey));
                        } else if (storage.containsKey(getKey(fileEntryKey.getDigest(), !fileEntryKey.getIsExecutable()))) {
                            return immediateFuture(null);
                        }
                        expiredKeyCounter.inc();
                        logger.log(Level.INFO, format("expired key %s", expiredKey));
                        return immediateFuture(fileEntryKey.getDigest());
                    }, expireService));
                }
            }
        } catch (InterruptedException e) {
            // clear interrupted flag
            Thread.interrupted();
            interrupted = true;
        }
        expiredDigestsFutures = builder.build();
    }
    ImmutableSet.Builder<Digest> builder = ImmutableSet.builder();
    for (ListenableFuture<Digest> expiredDigestFuture : expiredDigestsFutures) {
        Digest digest = getOrIOException(expiredDigestFuture);
        if (Thread.interrupted()) {
            interrupted = true;
        }
        if (digest != null) {
            builder.add(digest);
        }
    }
    Set<Digest> expiredDigests = builder.build();
    if (!expiredDigests.isEmpty()) {
        onExpire.accept(expiredDigests);
    }
    if (interrupted || Thread.currentThread().isInterrupted()) {
        throw new InterruptedException();
    }
    return true;
}

Example 38 with Digest

use of com.google.cloud.kms.v1.Digest in project bazel-buildfarm by bazelbuild.

the class CASFileCache method computeDirectories.

@SuppressWarnings("ConstantConditions")
private List<Path> computeDirectories(CacheScanResults cacheScanResults) throws InterruptedException {
    // create thread pool
    int nThreads = Runtime.getRuntime().availableProcessors();
    String threadNameFormat = "compute-cache-pool-%d";
    ExecutorService pool = Executors.newFixedThreadPool(nThreads, new ThreadFactoryBuilder().setNameFormat(threadNameFormat).build());
    ImmutableList.Builder<Path> invalidDirectories = new ImmutableList.Builder<>();
    for (Path path : cacheScanResults.computeDirs) {
        pool.execute(() -> {
            try {
                ImmutableList.Builder<String> inputsBuilder = ImmutableList.builder();
                List<NamedFileKey> sortedDirent = listDirentSorted(path, fileStore);
                Directory directory = computeDirectory(path, sortedDirent, cacheScanResults.fileKeys, inputsBuilder);
                Digest digest = directory == null ? null : digestUtil.compute(directory);
                // apply legacy rename if possible
                // Remove on major release or when #677 is closed
                Path dirPath = path;
                String basename = path.getFileName().toString();
                if (basename.equals(digest.getHash() + "_" + digest.getSizeBytes() + "_dir")) {
                    dirPath = getDirectoryPath(digest);
                    if (Files.exists(dirPath)) {
                        // destroy this directory if the destination already exists
                        digest = null;
                    } else {
                        Files.move(path, dirPath);
                    }
                }
                if (digest != null && getDirectoryPath(digest).equals(dirPath)) {
                    DirectoryEntry e = new DirectoryEntry(directory, Deadline.after(10, SECONDS));
                    directoriesIndex.put(digest, inputsBuilder.build());
                    directoryStorage.put(digest, e);
                } else {
                    synchronized (invalidDirectories) {
                        invalidDirectories.add(dirPath);
                    }
                }
            } catch (Exception e) {
                logger.log(Level.SEVERE, "error processing directory " + path.toString(), e);
            }
        });
    }
    joinThreads(pool, "Populating Directories...");
    return invalidDirectories.build();
}

Example 39 with Digest

use of com.google.cloud.kms.v1.Digest in project bazel-buildfarm by bazelbuild.

the class CASFileCache method decrementReferencesSynchronized.

@GuardedBy("this")
private void decrementReferencesSynchronized(Iterable<String> inputFiles, Iterable<Digest> inputDirectories) throws IOException {
    // decrement references and notify if any dropped to 0
    // insert after the last 0-reference count entry in list
    int entriesDereferenced = decrementInputReferences(inputFiles);
    for (Digest inputDirectory : inputDirectories) {
        DirectoryEntry dirEntry = directoryStorage.get(inputDirectory);
        if (dirEntry == null) {
            throw new IllegalStateException("inputDirectory " + DigestUtil.toString(inputDirectory) + " is not in directoryStorage");
        }
        entriesDereferenced += decrementInputReferences(directoriesIndex.directoryEntries(inputDirectory));
    }
    if (entriesDereferenced > 0) {
        notify();
    }
}

Example 40 with Digest

use of com.google.cloud.kms.v1.Digest in project bazel-buildfarm by bazelbuild.

the class ContentAddressableStorages method createFilesystemCAS.

public static ContentAddressableStorage createFilesystemCAS(FilesystemCASConfig config) throws ConfigurationException {
    String path = config.getPath();
    if (path.isEmpty()) {
        throw new ConfigurationException("filesystem cas path is empty");
    }
    long maxSizeBytes = config.getMaxSizeBytes();
    long maxEntrySizeBytes = config.getMaxEntrySizeBytes();
    int hexBucketLevels = config.getHexBucketLevels();
    boolean storeFileDirsIndexInMemory = config.getFileDirectoriesIndexInMemory();
    if (maxSizeBytes <= 0) {
        throw new ConfigurationException("filesystem cas max_size_bytes <= 0");
    }
    if (maxEntrySizeBytes <= 0) {
        throw new ConfigurationException("filesystem cas max_entry_size_bytes <= 0");
    }
    if (maxEntrySizeBytes > maxSizeBytes) {
        throw new ConfigurationException("filesystem cas max_entry_size_bytes > maxSizeBytes");
    }
    if (hexBucketLevels < 0) {
        throw new ConfigurationException("filesystem cas hex_bucket_levels <= 0");
    }
    CASFileCache cas = new CASFileCache(Paths.get(path), maxSizeBytes, maxEntrySizeBytes, hexBucketLevels, storeFileDirsIndexInMemory, DigestUtil.forHash("SHA256"), /* expireService=*/
    newDirectExecutorService(), /* accessRecorder=*/
    directExecutor()) {

        @Override
        protected InputStream newExternalInput(Digest digest) throws IOException {
            throw new NoSuchFileException(digest.getHash());
        }
    };
    try {
        cas.start(false);
    } catch (IOException | InterruptedException e) {
        throw new RuntimeException("error starting filesystem cas", e);
    }
    return cas;
}