Examples with ResolvedFile com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile used on opensource projects

Example 1 with ResolvedFile

use of com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile in project bazel by bazelbuild.

the class RecursiveFilesystemTraversalFunctionTest method traverseAndAssertFiles.

/**
   * Asserts that the requested SkyValue can be built and results in the expected set of files.
   *
   * <p>The metadata of files is ignored in comparing the actual results with the expected ones.
   * The returned object however contains the actual metadata.
   */
@SafeVarargs
private final RecursiveFilesystemTraversalValue traverseAndAssertFiles(TraversalRequest params, ResolvedFile... expectedFilesIgnoringMetadata) throws Exception {
    RecursiveFilesystemTraversalValue result = evalTraversalRequest(params);
    Set<ResolvedFile> actual = new HashSet<>();
    for (ResolvedFile act : result.getTransitiveFiles()) {
        // Strip metadata so only the type and path of the objects are compared.
        actual.add(act.stripMetadataForTesting());
    }
    // First just assert equality of the keys, so in case of a mismatch the error message is easier
    // to read.
    assertThat(actual).containsExactly((Object[]) expectedFilesIgnoringMetadata);
    // The returned object still has the unstripped metadata.
    return result;
}

Example 2 with ResolvedFile

use of com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile in project bazel by bazelbuild.

the class RecursiveFilesystemTraversalFunctionTest method assertTraverseSubpackages.

private void assertTraverseSubpackages(PackageBoundaryMode traverseSubpackages) throws Exception {
    Artifact pkgDirArtifact = sourceArtifact("pkg1/foo");
    Artifact subpkgDirArtifact = sourceArtifact("pkg1/foo/subdir/subpkg");
    RootedPath pkgBuildFile = childOf(pkgDirArtifact, "BUILD");
    RootedPath subpkgBuildFile = childOf(subpkgDirArtifact, "BUILD");
    scratch.dir(rootedPath(pkgDirArtifact).asPath().getPathString());
    scratch.dir(rootedPath(subpkgDirArtifact).asPath().getPathString());
    createFile(pkgBuildFile);
    createFile(subpkgBuildFile);
    TraversalRequest traversalRoot = pkgRoot(parentOf(pkgBuildFile), traverseSubpackages);
    ResolvedFile expected1 = regularFileForTesting(pkgBuildFile);
    ResolvedFile expected2 = regularFileForTesting(subpkgBuildFile);
    switch(traverseSubpackages) {
        case CROSS:
            traverseAndAssertFiles(traversalRoot, expected1, expected2);
            break;
        case DONT_CROSS:
            traverseAndAssertFiles(traversalRoot, expected1);
            break;
        case REPORT_ERROR:
            SkyKey key = rftvSkyKey(traversalRoot);
            EvaluationResult<SkyValue> result = eval(key);
            assertThat(result.hasError()).isTrue();
            assertThat(result.getError().getException().getMessage()).contains("crosses package boundary into package rooted at");
            break;
        default:
            throw new IllegalStateException(traverseSubpackages.toString());
    }
}

Example 3 with ResolvedFile

use of com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile in project bazel by bazelbuild.

the class RecursiveFilesystemTraversalFunctionTest method assertTraversalOfDirectory.

private void assertTraversalOfDirectory(Artifact directoryArtifact) throws Exception {
    // Create files under the directory.
    // Use the root + root-relative path of the rootArtifact to create these files, rather than
    // using the rootDirectory + execpath of the rootArtifact. The resulting paths are the same
    // but the RootedPaths are different:
    // in the 1st case, it is: RootedPath(/root/execroot, relative), in the second it is
    // in the 2nd case, it is: RootedPath(/root, execroot/relative).
    // Creating the files will also create the parent directories.
    RootedPath file1 = createFile(childOf(directoryArtifact, "bar.txt"));
    RootedPath file2 = createFile(childOf(directoryArtifact, "baz/qux.txt"));
    TraversalRequest traversalRoot = fileLikeRoot(directoryArtifact, DONT_CROSS);
    // Assert that the SkyValue is built and looks right.
    ResolvedFile expected1 = regularFileForTesting(file1);
    ResolvedFile expected2 = regularFileForTesting(file2);
    RecursiveFilesystemTraversalValue v1 = traverseAndAssertFiles(traversalRoot, expected1, expected2);
    assertThat(progressReceiver.invalidations).isEmpty();
    assertThat(progressReceiver.evaluations).contains(v1);
    progressReceiver.clear();
    // Add a new file to the directory and see that the value is rebuilt.
    RootedPath file3 = createFile(childOf(directoryArtifact, "foo.txt"));
    invalidateDirectory(directoryArtifact);
    ResolvedFile expected3 = regularFileForTesting(file3);
    RecursiveFilesystemTraversalValue v2 = traverseAndAssertFiles(traversalRoot, expected1, expected2, expected3);
    assertThat(progressReceiver.invalidations).contains(rftvSkyKey(traversalRoot));
    assertThat(progressReceiver.evaluations).contains(v2);
    // Directories always have the same hash code, but that is fine because their contents are also
    // part of the RecursiveFilesystemTraversalValue, so v1 and v2 are unequal.
    assertThat(v2).isNotEqualTo(v1);
    assertTraversalRootHashesAreEqual(v1, v2);
    progressReceiver.clear();
    // Edit a file in the directory and see that the value is rebuilt.
    appendToFile(file1, "bar");
    RecursiveFilesystemTraversalValue v3 = traverseAndAssertFiles(traversalRoot, expected1, expected2, expected3);
    assertThat(progressReceiver.invalidations).contains(rftvSkyKey(traversalRoot));
    assertThat(progressReceiver.evaluations).contains(v3);
    assertThat(v3).isNotEqualTo(v2);
    // Directories always have the same hash code, but that is fine because their contents are also
    // part of the RecursiveFilesystemTraversalValue, so v2 and v3 are unequal.
    assertTraversalRootHashesAreEqual(v2, v3);
    progressReceiver.clear();
    // Add a new file *outside* of the directory and see that the value is *not* rebuilt.
    Artifact someFile = sourceArtifact("somewhere/else/a.file");
    createFile(someFile, "new file");
    appendToFile(someFile, "not all changes are treated equal");
    RecursiveFilesystemTraversalValue v4 = traverseAndAssertFiles(traversalRoot, expected1, expected2, expected3);
    assertThat(v4).isEqualTo(v3);
    assertTraversalRootHashesAreEqual(v3, v4);
    assertThat(progressReceiver.invalidations).doesNotContain(rftvSkyKey(traversalRoot));
}

Example 4 with ResolvedFile

use of com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile in project bazel by bazelbuild.

the class FilesetEntryFunction method compute.

@Override
public SkyValue compute(SkyKey key, Environment env) throws FilesetEntryFunctionException, InterruptedException {
    FilesetTraversalParams t = (FilesetTraversalParams) key.argument();
    Preconditions.checkState(t.getNestedTraversal().isPresent() != t.getDirectTraversal().isPresent(), "Exactly one of the nested and direct traversals must be specified: %s", t);
    // Create the set of excluded files. Only top-level files can be excluded, i.e. ones that are
    // directly under the root if the root is a directory.
    Set<String> exclusions = createExclusionSet(t.getExcludedFiles());
    // The map of output symlinks. Each key is the path of a output symlink that the Fileset must
    // create, relative to the Fileset.out directory, and each value specifies extra information
    // about the link (its target, associated metadata and again its name).
    Map<PathFragment, FilesetOutputSymlink> outputSymlinks = new LinkedHashMap<>();
    if (t.getNestedTraversal().isPresent()) {
        // The "nested" traversal parameters are present if and only if FilesetEntry.srcdir specifies
        // another Fileset (a "nested" one).
        FilesetEntryValue nested = (FilesetEntryValue) env.getValue(FilesetEntryValue.key(t.getNestedTraversal().get()));
        if (env.valuesMissing()) {
            return null;
        }
        for (FilesetOutputSymlink s : nested.getSymlinks()) {
            if (!exclusions.contains(s.name.getPathString())) {
                maybeStoreSymlink(s, t.getDestPath(), outputSymlinks);
            }
        }
    } else {
        // The "nested" traversal params are absent if and only if the "direct" traversal params are
        // present, which is the case when the FilesetEntry specifies a package's BUILD file, a
        // directory or a list of files.
        // The root of the direct traversal is defined as follows.
        //
        // If FilesetEntry.files is specified, then a TraversalRequest is created for each entry, the
        // root being the respective entry itself. These are all traversed for they may be
        // directories or symlinks to directories, and we need to establish Skyframe dependencies on
        // their contents for incremental correctness. If an entry is indeed a directory (but not when
        // it's a symlink to one) then we have to create symlinks to each of their childen.
        // (NB: there seems to be no good reason for this, it's just how legacy Fileset works. We may
        // want to consider creating a symlink just for the directory and not for its child elements.)
        //
        // If FilesetEntry.files is not specified, then srcdir refers to either a BUILD file or a
        // directory. For the former, the root will be the parent of the BUILD file. For the latter,
        // the root will be srcdir itself.
        DirectTraversal direct = t.getDirectTraversal().get();
        RecursiveFilesystemTraversalValue rftv;
        try {
            // Traverse the filesystem to establish skyframe dependencies.
            rftv = traverse(env, createErrorInfo(t), direct);
        } catch (MissingDepException e) {
            return null;
        }
        // The root can only be absent for the EMPTY rftv instance.
        if (!rftv.getResolvedRoot().isPresent()) {
            return FilesetEntryValue.EMPTY;
        }
        ResolvedFile resolvedRoot = rftv.getResolvedRoot().get();
        // Handle dangling symlinks gracefully be returning empty results.
        if (!resolvedRoot.getType().exists()) {
            return FilesetEntryValue.EMPTY;
        }
        // The prefix to remove is the entire path of the root. This is OK:
        // - when the root is a file, this removes the entire path, but the traversal's destination
        //   path is actually the name of the output symlink, so this works out correctly
        // - when the root is a directory or a symlink to one then we need to strip off the
        //   directory's path from every result (this is how the output symlinks must be created)
        //   before making them relative to the destination path
        PathFragment prefixToRemove = direct.getRoot().getRelativePart();
        Iterable<ResolvedFile> results = null;
        if (direct.isRecursive() || (resolvedRoot.getType().isDirectory() && !resolvedRoot.getType().isSymlink())) {
            // The traversal is recursive (requested for an entire FilesetEntry.srcdir) or it was
            // requested for a FilesetEntry.files entry which turned out to be a directory. We need to
            // create an output symlink for every file in it and all of its subdirectories. Only
            // exception is when the subdirectory is really a symlink to a directory -- no output
            // shall be created for the contents of those.
            // Now we create Dir objects to model the filesystem tree. The object employs a trick to
            // find directory symlinks: directory symlinks have corresponding ResolvedFile entries and
            // are added as files too, while their children, also added as files, contain the path of
            // the parent. Finding and discarding the children is easy if we traverse the tree from
            // root to leaf.
            DirectoryTree root = new DirectoryTree();
            for (ResolvedFile f : rftv.getTransitiveFiles().toCollection()) {
                PathFragment path = f.getNameInSymlinkTree().relativeTo(prefixToRemove);
                if (path.segmentCount() > 0) {
                    path = t.getDestPath().getRelative(path);
                    DirectoryTree dir = root;
                    for (int i = 0; i < path.segmentCount() - 1; ++i) {
                        dir = dir.addOrGetSubdir(path.getSegment(i));
                    }
                    dir.maybeAddFile(f);
                }
            }
            // Here's where the magic happens. The returned iterable will yield all files in the
            // directory that are not under symlinked directories, as well as all directory symlinks.
            results = root.iterateFiles();
        } else {
            // If we're on this branch then the traversal was done for just one entry in
            // FilesetEntry.files (which was not a directory, so it was either a file, a symlink to one
            // or a symlink to a directory), meaning we'll have only one output symlink.
            results = ImmutableList.of(resolvedRoot);
        }
        // Create one output symlink for each entry in the results.
        for (ResolvedFile f : results) {
            // The linkName has to be under the traversal's root, which is also the prefix to remove.
            PathFragment linkName = f.getNameInSymlinkTree().relativeTo(prefixToRemove);
            // It may be dangling, but excluding it is still fine.
            if (exclusions.contains(linkName.getPathString())) {
                continue;
            }
            PathFragment targetName;
            try {
                targetName = f.getTargetInSymlinkTree(direct.isFollowingSymlinks());
            } catch (DanglingSymlinkException e) {
                throw new FilesetEntryFunctionException(e);
            }
            // Metadata field must be present. It can only be absent when stripped by tests.
            String metadata = Integer.toHexString(f.getMetadataHash());
            maybeStoreSymlink(linkName, targetName, metadata, t.getDestPath(), outputSymlinks);
        }
    }
    return FilesetEntryValue.of(ImmutableSet.copyOf(outputSymlinks.values()));
}

Example 5 with ResolvedFile

use of com.google.devtools.build.lib.skyframe.RecursiveFilesystemTraversalValue.ResolvedFile in project bazel by bazelbuild.

the class RecursiveFilesystemTraversalFunction method resultForDirectory.

private static RecursiveFilesystemTraversalValue resultForDirectory(TraversalRequest traversal, FileInfo rootInfo, Collection<RecursiveFilesystemTraversalValue> subdirTraversals) {
    // Collect transitive closure of files in subdirectories.
    NestedSetBuilder<ResolvedFile> paths = NestedSetBuilder.stableOrder();
    for (RecursiveFilesystemTraversalValue child : subdirTraversals) {
        paths.addTransitive(child.getTransitiveFiles());
    }
    ResolvedFile root;
    if (rootInfo.type.isSymlink()) {
        NestedSet<ResolvedFile> children = paths.build();
        root = ResolvedFileFactory.symlinkToDirectory(rootInfo.realPath, traversal.path, rootInfo.unresolvedSymlinkTarget, hashDirectorySymlink(children, rootInfo.metadata.hashCode()));
        paths = NestedSetBuilder.<ResolvedFile>stableOrder().addTransitive(children).add(root);
    } else {
        root = ResolvedFileFactory.directory(rootInfo.realPath);
    }
    return RecursiveFilesystemTraversalValue.of(root, paths.build());
}