Examples with DirectoryNode build.bazel.remote.execution.v2.DirectoryNode used on opensource projects

Example 11 with DirectoryNode

use of build.bazel.remote.execution.v2.DirectoryNode in project bazel-buildfarm by bazelbuild.

the class AbstractServerInstance method validateActionInputDirectory.

@VisibleForTesting
public static void validateActionInputDirectory(String directoryPath, Directory directory, Stack<Digest> pathDigests, Set<Digest> visited, Map<Digest, Directory> directoriesIndex, Consumer<String> onInputFile, Consumer<String> onInputDirectory, Consumer<Digest> onInputDigest, PreconditionFailure.Builder preconditionFailure) {
    Set<String> entryNames = new HashSet<>();
    String lastFileName = "";
    for (FileNode fileNode : directory.getFilesList()) {
        String fileName = fileNode.getName();
        if (entryNames.contains(fileName)) {
            preconditionFailure.addViolationsBuilder().setType(VIOLATION_TYPE_INVALID).setSubject("/" + directoryPath + ": " + fileName).setDescription(DUPLICATE_DIRENT);
        } else if (lastFileName.compareTo(fileName) > 0) {
            preconditionFailure.addViolationsBuilder().setType(VIOLATION_TYPE_INVALID).setSubject("/" + directoryPath + ": " + lastFileName + " > " + fileName).setDescription(DIRECTORY_NOT_SORTED);
        }
        // FIXME serverside validity check? regex?
        Preconditions.checkState(isValidFilename(), INVALID_FILE_NAME);
        lastFileName = fileName;
        entryNames.add(fileName);
        onInputDigest.accept(fileNode.getDigest());
        String filePath = directoryPath.isEmpty() ? fileName : (directoryPath + "/" + fileName);
        onInputFile.accept(filePath);
    }
    String lastSymlinkName = "";
    for (SymlinkNode symlinkNode : directory.getSymlinksList()) {
        String symlinkName = symlinkNode.getName();
        if (entryNames.contains(symlinkName)) {
            preconditionFailure.addViolationsBuilder().setType(VIOLATION_TYPE_INVALID).setSubject("/" + directoryPath + ": " + symlinkName).setDescription(DUPLICATE_DIRENT);
        } else if (lastSymlinkName.compareTo(symlinkName) > 0) {
            preconditionFailure.addViolationsBuilder().setType(VIOLATION_TYPE_INVALID).setSubject("/" + directoryPath + ": " + lastSymlinkName + " > " + symlinkName).setDescription(DIRECTORY_NOT_SORTED);
        }
        /* FIXME serverside validity check? regex?
      Preconditions.checkState(
          isValidFilename(symlinkName),
          INVALID_FILE_NAME);
      Preconditions.checkState(
          isValidFilename(symlinkNode.getTarget()),
          INVALID_FILE_NAME);
      // FIXME verify that any relative pathing for the target is within the input root
      */
        lastSymlinkName = symlinkName;
        entryNames.add(symlinkName);
    }
    String lastDirectoryName = "";
    for (DirectoryNode directoryNode : directory.getDirectoriesList()) {
        String directoryName = directoryNode.getName();
        if (entryNames.contains(directoryName)) {
            preconditionFailure.addViolationsBuilder().setType(VIOLATION_TYPE_INVALID).setSubject("/" + directoryPath + ": " + directoryName).setDescription(DUPLICATE_DIRENT);
        } else if (lastDirectoryName.compareTo(directoryName) > 0) {
            preconditionFailure.addViolationsBuilder().setType(VIOLATION_TYPE_INVALID).setSubject("/" + directoryPath + ": " + lastDirectoryName + " > " + directoryName).setDescription(DIRECTORY_NOT_SORTED);
        }
        /* FIXME serverside validity check? regex?
      Preconditions.checkState(
          isValidFilename(directoryName),
          INVALID_FILE_NAME);
      */
        lastDirectoryName = directoryName;
        entryNames.add(directoryName);
        Digest directoryDigest = directoryNode.getDigest();
        if (pathDigests.contains(directoryDigest)) {
            preconditionFailure.addViolationsBuilder().setType(VIOLATION_TYPE_INVALID).setSubject(DIRECTORY_CYCLE_DETECTED).setDescription("/" + directoryPath + ": " + directoryName);
        } else {
            String subDirectoryPath = directoryPath.isEmpty() ? directoryName : (directoryPath + "/" + directoryName);
            onInputDirectory.accept(subDirectoryPath);
            if (!visited.contains(directoryDigest)) {
                validateActionInputDirectoryDigest(subDirectoryPath, directoryDigest, pathDigests, visited, directoriesIndex, onInputFile, onInputDirectory, onInputDigest, preconditionFailure);
            } else {
                enumerateActionInputDirectory(subDirectoryPath, directoriesIndex.get(directoryDigest), directoriesIndex, onInputFile, onInputDirectory);
            }
        }
    }
}

Example 12 with DirectoryNode

use of build.bazel.remote.execution.v2.DirectoryNode in project bazel-buildfarm by bazelbuild.

the class AbstractServerInstance method enumerateActionInputDirectory.

private static void enumerateActionInputDirectory(String directoryPath, Directory directory, Map<Digest, Directory> directoriesIndex, Consumer<String> onInputFile, Consumer<String> onInputDirectory) {
    for (FileNode fileNode : directory.getFilesList()) {
        String fileName = fileNode.getName();
        String filePath = directoryPath.isEmpty() ? fileName : (directoryPath + "/" + fileName);
        onInputFile.accept(filePath);
    }
    for (DirectoryNode directoryNode : directory.getDirectoriesList()) {
        String directoryName = directoryNode.getName();
        Digest directoryDigest = directoryNode.getDigest();
        String subDirectoryPath = directoryPath.isEmpty() ? directoryName : (directoryPath + "/" + directoryName);
        onInputDirectory.accept(subDirectoryPath);
        enumerateActionInputDirectory(subDirectoryPath, directoriesIndex.get(directoryDigest), directoriesIndex, onInputFile, onInputDirectory);
    }
}

Example 13 with DirectoryNode

use of build.bazel.remote.execution.v2.DirectoryNode in project bazel-buildfarm by bazelbuild.

the class AbstractServerInstance method validateCommand.

@VisibleForTesting
void validateCommand(Command command, Digest inputRootDigest, Set<String> inputFiles, Set<String> inputDirectories, Map<Digest, Directory> directoriesIndex, PreconditionFailure.Builder preconditionFailure) {
    validatePlatform(command.getPlatform(), preconditionFailure);
    // FIXME should input/output collisions (through directories) be another
    // invalid action?
    filesUniqueAndSortedPrecondition(command.getOutputFilesList(), preconditionFailure);
    filesUniqueAndSortedPrecondition(command.getOutputDirectoriesList(), preconditionFailure);
    validateOutputs(inputFiles, inputDirectories, Sets.newHashSet(command.getOutputFilesList()), Sets.newHashSet(command.getOutputDirectoriesList()), preconditionFailure);
    environmentVariablesUniqueAndSortedPrecondition(command.getEnvironmentVariablesList(), preconditionFailure);
    if (command.getArgumentsList().isEmpty()) {
        preconditionFailure.addViolationsBuilder().setType(VIOLATION_TYPE_INVALID).setSubject(INVALID_COMMAND).setDescription("argument list is empty");
    }
    String workingDirectory = command.getWorkingDirectory();
    if (!workingDirectory.isEmpty()) {
        if (workingDirectory.startsWith("/")) {
            preconditionFailure.addViolationsBuilder().setType(VIOLATION_TYPE_INVALID).setSubject(INVALID_COMMAND).setDescription("working directory is absolute");
        } else {
            Directory directory = directoriesIndex.get(inputRootDigest);
            for (String segment : workingDirectory.split("/")) {
                Directory nextDirectory = directory;
                // linear for now
                for (DirectoryNode dirNode : directory.getDirectoriesList()) {
                    if (dirNode.getName().equals(segment)) {
                        nextDirectory = directoriesIndex.get(dirNode.getDigest());
                        break;
                    }
                }
                if (nextDirectory == directory) {
                    preconditionFailure.addViolationsBuilder().setType(VIOLATION_TYPE_INVALID).setSubject(INVALID_COMMAND).setDescription("working directory is not an input directory");
                    break;
                }
                directory = nextDirectory;
            }
        }
    }
}

Example 14 with DirectoryNode

use of build.bazel.remote.execution.v2.DirectoryNode in project bazel-buildfarm by bazelbuild.

the class CFCExecFileSystem method treeContainsPath.

private static boolean treeContainsPath(String directoryPath, Map<Digest, Directory> directoriesIndex, Digest rootDigest) {
    Directory directory = directoriesIndex.get(rootDigest);
    for (String name : directoryPath.split("/")) {
        List<DirectoryNode> subdirs = directory.getDirectoriesList();
        int index = Collections.binarySearch(Lists.transform(subdirs, DirectoryNode::getName), name);
        if (index < 0) {
            return false;
        }
        directory = directoriesIndex.get(subdirs.get(index).getDigest());
    }
    return true;
}

Example 15 with DirectoryNode

use of build.bazel.remote.execution.v2.DirectoryNode in project bazel-buildfarm by bazelbuild.

the class CFCExecFileSystem method fetchInputs.

private Iterable<ListenableFuture<Void>> fetchInputs(Path path, Digest directoryDigest, Map<Digest, Directory> directoriesIndex, OutputDirectory outputDirectory, ImmutableList.Builder<String> inputFiles, ImmutableList.Builder<Digest> inputDirectories) throws IOException {
    Directory directory = directoriesIndex.get(directoryDigest);
    if (directory == null) {
        // not quite IO...
        throw new IOException("Directory " + DigestUtil.toString(directoryDigest) + " is not in directories index");
    }
    Iterable<ListenableFuture<Void>> downloads = directory.getFilesList().stream().map(fileNode -> put(path, fileNode, inputFiles)).collect(ImmutableList.toImmutableList());
    downloads = concat(downloads, directory.getSymlinksList().stream().map(symlinkNode -> putSymlink(path, symlinkNode)).collect(ImmutableList.toImmutableList()));
    for (DirectoryNode directoryNode : directory.getDirectoriesList()) {
        Digest digest = directoryNode.getDigest();
        String name = directoryNode.getName();
        OutputDirectory childOutputDirectory = outputDirectory != null ? outputDirectory.getChild(name) : null;
        Path dirPath = path.resolve(name);
        if (childOutputDirectory != null || !linkInputDirectories) {
            Files.createDirectories(dirPath);
            downloads = concat(downloads, fetchInputs(dirPath, digest, directoriesIndex, childOutputDirectory, inputFiles, inputDirectories));
        } else {
            downloads = concat(downloads, ImmutableList.of(transform(linkDirectory(dirPath, digest, directoriesIndex), (result) -> {
                // we saw null entries in the built immutable list without synchronization
                synchronized (inputDirectories) {
                    inputDirectories.add(digest);
                }
                return null;
            }, fetchService)));
        }
        if (Thread.currentThread().isInterrupted()) {
            break;
        }
    }
    return downloads;
}