Examples with DirectoryNode build.bazel.remote.execution.v2.DirectoryNode used on opensource projects

Example 1 with DirectoryNode

use of build.bazel.remote.execution.v2.DirectoryNode in project tools_remote by bazelbuild.

the class RemoteClient method listDirectory.

// Recursively list directory files/subdirectories with digests. Returns the number of files
// listed.
private int listDirectory(Path path, Directory dir, Map<Digest, Directory> childrenMap, int limit) throws IOException {
    // Try to list the files in this directory before listing the directories.
    int numFilesListed = listFileNodes(path, dir, limit);
    if (numFilesListed >= limit) {
        return numFilesListed;
    }
    for (DirectoryNode child : dir.getDirectoriesList()) {
        Path childPath = path.resolve(child.getName());
        printDirectoryNodeDetails(child, childPath);
        Digest childDigest = child.getDigest();
        Directory childDir = childrenMap.get(childDigest);
        numFilesListed += listDirectory(childPath, childDir, childrenMap, limit - numFilesListed);
        if (numFilesListed >= limit) {
            return numFilesListed;
        }
    }
    return numFilesListed;
}

Example 2 with DirectoryNode

use of build.bazel.remote.execution.v2.DirectoryNode in project tools_remote by bazelbuild.

the class AbstractRemoteActionCache method addChildDirectories.

/**
 * Recursively add all child directories of the given directory to the given list of directories.
 */
private void addChildDirectories(Directory dir, List<Directory> directories) throws IOException {
    for (DirectoryNode childNode : dir.getDirectoriesList()) {
        Directory childDir = Directory.parseFrom(downloadBlob(childNode.getDigest()));
        directories.add(childDir);
        addChildDirectories(childDir, directories);
    }
}

Example 3 with DirectoryNode

use of build.bazel.remote.execution.v2.DirectoryNode in project bazel-buildfarm by bazelbuild.

the class ShardInstance method getTreeFuture.

@Override
protected ListenableFuture<Tree> getTreeFuture(String reason, Digest inputRoot, ExecutorService service, RequestMetadata requestMetadata) {
    SettableFuture<Void> future = SettableFuture.create();
    Tree.Builder tree = Tree.newBuilder().setRootDigest(inputRoot);
    Set<Digest> digests = Sets.newConcurrentHashSet();
    Queue<Digest> remaining = new ConcurrentLinkedQueue();
    remaining.offer(inputRoot);
    Context ctx = Context.current();
    TreeCallback callback = new TreeCallback(future) {

        @Override
        protected void onDirectory(Digest digest, Directory directory) {
            tree.putDirectories(digest.getHash(), directory);
            for (DirectoryNode childNode : directory.getDirectoriesList()) {
                Digest child = childNode.getDigest();
                if (digests.add(child)) {
                    remaining.offer(child);
                }
            }
        }

        @Override
        boolean next() {
            Digest nextDigest = remaining.poll();
            if (!future.isDone() && nextDigest != null) {
                ctx.run(() -> addCallback(transform(expectDirectory(reason, nextDigest, requestMetadata), directory -> new DirectoryEntry(nextDigest, directory), service), this, service));
                return true;
            }
            return false;
        }
    };
    callback.next();
    return transform(future, (result) -> tree.build(), service);
}

Example 4 with DirectoryNode

use of build.bazel.remote.execution.v2.DirectoryNode in project bazel-buildfarm by bazelbuild.

the class Extract method directoryGetter.

static Runnable directoryGetter(Path root, String instanceName, Digest digest, Set<Digest> visitedDirectories, Set<Digest> visitedDigests, ByteStreamStub bsStub, Executor executor, AtomicLong outstandingOperations, ListeningScheduledExecutorService retryService) {
    return new Runnable() {

        @Override
        public void run() {
            try {
                runInterruptibly();
            } catch (InterruptedException e) {
                e.printStackTrace();
            } finally {
                outstandingOperations.getAndDecrement();
            }
        }

        void runInterruptibly() throws InterruptedException {
            try {
                ByteString content = getBlobIntoFile("directory", instanceName, digest, bsStub, root);
                handleDirectory(Directory.parseFrom(content));
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        void handleDirectory(Directory directory) {
            for (FileNode fileNode : directory.getFilesList()) {
                Digest fileDigest = fileNode.getDigest();
                if (!visitedDigests.contains(fileDigest)) {
                    visitedDigests.add(fileDigest);
                    outstandingOperations.getAndIncrement();
                    executor.execute(blobGetter(root, instanceName, fileDigest, bsStub, outstandingOperations, retryService));
                }
            }
            for (DirectoryNode directoryNode : directory.getDirectoriesList()) {
                Digest directoryDigest = directoryNode.getDigest();
                // we may have seen this digest, but now we will have seen it as a directory
                if (!visitedDirectories.contains(directoryDigest)) {
                    // probably won't collide with other writers, with single thread
                    visitedDigests.add(directoryDigest);
                    visitedDirectories.add(directoryDigest);
                    outstandingOperations.getAndIncrement();
                    executor.execute(directoryGetter(root, instanceName, directoryDigest, visitedDirectories, visitedDigests, bsStub, executor, outstandingOperations, retryService));
                }
            }
        }
    };
}

Example 5 with DirectoryNode

use of build.bazel.remote.execution.v2.DirectoryNode in project bazel-buildfarm by bazelbuild.

the class OperationQueueWorkerContext method fetchInputs.

private void fetchInputs(Path execDir, Digest inputRoot, Map<Digest, Directory> directoriesIndex, OutputDirectory outputDirectory, ImmutableList.Builder<String> inputFiles, ImmutableList.Builder<Digest> inputDirectories) throws IOException, InterruptedException {
    Directory directory;
    if (inputRoot.getSizeBytes() == 0) {
        directory = Directory.getDefaultInstance();
    } else {
        directory = directoriesIndex.get(inputRoot);
        if (directory == null) {
            throw new IOException("Directory " + DigestUtil.toString(inputRoot) + " is not in input index");
        }
    }
    getInterruptiblyOrIOException(allAsList(fileCache.putFiles(directory.getFilesList(), directory.getSymlinksList(), execDir, inputFiles, newDirectExecutorService())));
    for (DirectoryNode directoryNode : directory.getDirectoriesList()) {
        Digest digest = directoryNode.getDigest();
        String name = directoryNode.getName();
        OutputDirectory childOutputDirectory = outputDirectory != null ? outputDirectory.getChild(name) : null;
        Path dirPath = execDir.resolve(name);
        if (childOutputDirectory != null || !config.getLinkInputDirectories()) {
            Files.createDirectories(dirPath);
            fetchInputs(dirPath, digest, directoriesIndex, childOutputDirectory, inputFiles, inputDirectories);
        } else {
            inputDirectories.add(digest);
            linkDirectory(dirPath, digest, directoriesIndex);
        }
    }
}