Examples with NamedFileKey build.buildfarm.common.io.NamedFileKey used on opensource projects

Example 1 with NamedFileKey

use of build.buildfarm.common.io.NamedFileKey in project bazel-buildfarm by bazelbuild.

the class CASFileCache method computeDirectories.

@SuppressWarnings("ConstantConditions")
private List<Path> computeDirectories(CacheScanResults cacheScanResults) throws InterruptedException {
    // create thread pool
    int nThreads = Runtime.getRuntime().availableProcessors();
    String threadNameFormat = "compute-cache-pool-%d";
    ExecutorService pool = Executors.newFixedThreadPool(nThreads, new ThreadFactoryBuilder().setNameFormat(threadNameFormat).build());
    ImmutableList.Builder<Path> invalidDirectories = new ImmutableList.Builder<>();
    for (Path path : cacheScanResults.computeDirs) {
        pool.execute(() -> {
            try {
                ImmutableList.Builder<String> inputsBuilder = ImmutableList.builder();
                List<NamedFileKey> sortedDirent = listDirentSorted(path, fileStore);
                Directory directory = computeDirectory(path, sortedDirent, cacheScanResults.fileKeys, inputsBuilder);
                Digest digest = directory == null ? null : digestUtil.compute(directory);
                // apply legacy rename if possible
                // Remove on major release or when #677 is closed
                Path dirPath = path;
                String basename = path.getFileName().toString();
                if (basename.equals(digest.getHash() + "_" + digest.getSizeBytes() + "_dir")) {
                    dirPath = getDirectoryPath(digest);
                    if (Files.exists(dirPath)) {
                        // destroy this directory if the destination already exists
                        digest = null;
                    } else {
                        Files.move(path, dirPath);
                    }
                }
                if (digest != null && getDirectoryPath(digest).equals(dirPath)) {
                    DirectoryEntry e = new DirectoryEntry(directory, Deadline.after(10, SECONDS));
                    directoriesIndex.put(digest, inputsBuilder.build());
                    directoryStorage.put(digest, e);
                } else {
                    synchronized (invalidDirectories) {
                        invalidDirectories.add(dirPath);
                    }
                }
            } catch (Exception e) {
                logger.log(Level.SEVERE, "error processing directory " + path.toString(), e);
            }
        });
    }
    joinThreads(pool, "Populating Directories...");
    return invalidDirectories.build();
}

Example 2 with NamedFileKey

use of build.buildfarm.common.io.NamedFileKey in project bazel-buildfarm by bazelbuild.

the class CASFileCache method computeDirectory.

private Directory computeDirectory(Path path, List<NamedFileKey> sortedDirent, Map<Object, Entry> fileKeys, ImmutableList.Builder<String> inputsBuilder) throws IOException {
    Directory.Builder b = Directory.newBuilder();
    for (NamedFileKey dirent : sortedDirent) {
        String name = dirent.getName();
        Path entryPath = path.resolve(name);
        if (dirent.getFileStatus().isSymbolicLink()) {
            b.addSymlinksBuilder().setName(name).setTarget(Files.readSymbolicLink(entryPath).toString());
        // TODO symlink properties
        } else {
            Entry e = fileKeys.get(dirent.getFileKey());
            // decide if file is a directory or empty/non-empty file
            boolean isDirectory = dirent.getFileStatus().isDirectory();
            boolean isEmptyFile = false;
            if (e == null && !isDirectory) {
                if (dirent.getFileStatus().getSize() == 0) {
                    isEmptyFile = true;
                } else {
                    // no entry, not a directory, will NPE
                    b.addFilesBuilder().setName(name + "-MISSING");
                    // continue here to hopefully result in invalid directory
                    break;
                }
            }
            // directory
            if (isDirectory) {
                List<NamedFileKey> childDirent = listDirentSorted(entryPath, fileStore);
                Directory dir = computeDirectory(entryPath, childDirent, fileKeys, inputsBuilder);
                b.addDirectoriesBuilder().setName(name).setDigest(digestUtil.compute(dir));
            } else if (isEmptyFile) {
                // empty file
                boolean isExecutable = isReadOnlyExecutable(entryPath, fileStore);
                b.addFilesBuilder().setName(name).setDigest(digestUtil.empty()).setIsExecutable(isExecutable);
            } else {
                // non-empty file
                inputsBuilder.add(e.key);
                Digest digest = CASFileCache.keyToDigest(e.key, e.size, digestUtil);
                boolean isExecutable = e.key.endsWith("_exec");
                b.addFilesBuilder().setName(name).setDigest(digest).setIsExecutable(isExecutable);
            }
        }
    }
    return b.build();
}