Examples with FileStatus build.buildfarm.common.io.FileStatus used on opensource projects

Example 1 with FileStatus

use of build.buildfarm.common.io.FileStatus in project bazel-buildfarm by bazelbuild.

the class Executor method findMissingBlobs.

private static List<Digest> findMissingBlobs(String instanceName, Path blobsDir, ContentAddressableStorageBlockingStub casStub) throws IOException {
    FindMissingBlobsRequest.Builder request = FindMissingBlobsRequest.newBuilder().setInstanceName(instanceName);
    int size = 0;
    ImmutableList.Builder<Digest> missingDigests = ImmutableList.builder();
    System.out.println("Looking for missing blobs");
    final int messagesPerRequest = 2 * 1024 * 1024 / 80;
    System.out.println("Looking for missing blobs");
    Stopwatch stopwatch = Stopwatch.createUnstarted();
    FileStore fileStore = Files.getFileStore(blobsDir);
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(blobsDir)) {
        for (Path file : stream) {
            FileStatus stat = stat(file, /* followSymlinks=*/
            false, fileStore);
            Digest digest = DigestUtil.buildDigest(file.getFileName().toString().split("_")[0], stat.getSize());
            request.addBlobDigests(digest);
            size++;
            if (size == messagesPerRequest) {
                stopwatch.reset().start();
                FindMissingBlobsResponse response = casStub.findMissingBlobs(request.build());
                System.out.println("Found " + response.getMissingBlobDigestsCount() + " missing digests in " + (stopwatch.elapsed(MICROSECONDS) / 1000.0) + "ms");
                missingDigests.addAll(response.getMissingBlobDigestsList());
                request = FindMissingBlobsRequest.newBuilder().setInstanceName(instanceName);
                size = 0;
            }
        }
    }
    if (size > 0) {
        FindMissingBlobsResponse response = casStub.findMissingBlobs(request.build());
        System.out.println("Found " + response.getMissingBlobDigestsCount() + " missing digests");
        missingDigests.addAll(response.getMissingBlobDigestsList());
    }
    return missingDigests.build();
}

Example 2 with FileStatus

use of build.buildfarm.common.io.FileStatus in project bazel-buildfarm by bazelbuild.

the class CASFileCache method processRootFile.

@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
private void processRootFile(Consumer<Digest> onStartPut, Path file, ImmutableList.Builder<Path> computeDirs, ImmutableList.Builder<Path> deleteFiles, ImmutableMap.Builder<Object, Entry> fileKeys) throws IOException {
    String basename = file.getFileName().toString();
    FileStatus stat = stat(file, false, fileStore);
    // mark directory for later key compute
    if (file.toString().endsWith("_dir")) {
        if (stat.isDirectory()) {
            synchronized (computeDirs) {
                computeDirs.add(file);
            }
        } else {
            synchronized (deleteFiles) {
                deleteFiles.add(file);
            }
        }
    } else if (stat.isDirectory()) {
        synchronized (deleteFiles) {
            deleteFiles.add(file);
        }
    } else {
        // if cas is full or entry is oversized or empty, mark file for later deletion.
        long size = stat.getSize();
        if (sizeInBytes + size > maxSizeInBytes || size > maxEntrySizeInBytes || size == 0) {
            synchronized (deleteFiles) {
                deleteFiles.add(file);
            }
        } else {
            // get the key entry from the file name.
            FileEntryKey fileEntryKey = parseFileEntryKey(basename, stat.getSize());
            // if key entry file name cannot be parsed, mark file for later deletion.
            if (fileEntryKey == null || stat.isReadOnlyExecutable() != fileEntryKey.getIsExecutable()) {
                synchronized (deleteFiles) {
                    deleteFiles.add(file);
                }
            } else {
                String key = fileEntryKey.getKey();
                Path keyPath = getPath(key);
                // remove/refactor when #677 is closed
                if (fileEntryKey.isLegacy()) {
                    Files.move(file, keyPath);
                }
                // populate key it is not currently stored.
                Entry e = new Entry(key, size, Deadline.after(10, SECONDS));
                Object fileKey = getFileKey(keyPath, stat);
                synchronized (fileKeys) {
                    fileKeys.put(fileKey, e);
                }
                storage.put(e.key, e);
                onStartPut.accept(fileEntryKey.getDigest());
                synchronized (this) {
                    if (e.decrementReference(header)) {
                        unreferencedEntryCount++;
                    }
                }
                sizeInBytes += size;
            }
        }
    }
}