Examples with SkyValue com.google.devtools.build.skyframe.SkyValue used on opensource projects

Example 26 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class FileFunctionTest method testFilesOutsideRootIsReEvaluated.

@Test
public void testFilesOutsideRootIsReEvaluated() throws Exception {
    Path file = file("/outsideroot");
    SequentialBuildDriver driver = makeDriver();
    SkyKey key = skyKey("/outsideroot");
    EvaluationResult<SkyValue> result;
    result = driver.evaluate(ImmutableList.of(key), false, DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
    if (result.hasError()) {
        fail(String.format("Evaluation error for %s: %s", key, result.getError()));
    }
    FileValue oldValue = (FileValue) result.get(key);
    assertTrue(oldValue.exists());
    file.delete();
    differencer.invalidate(ImmutableList.of(fileStateSkyKey("/outsideroot")));
    result = driver.evaluate(ImmutableList.of(key), false, DEFAULT_THREAD_COUNT, NullEventHandler.INSTANCE);
    if (result.hasError()) {
        fail(String.format("Evaluation error for %s: %s", key, result.getError()));
    }
    FileValue newValue = (FileValue) result.get(key);
    assertNotSame(oldValue, newValue);
    assertFalse(newValue.exists());
}

Example 27 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class GlobFunction method compute.

@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws GlobFunctionException, InterruptedException {
    GlobDescriptor glob = (GlobDescriptor) skyKey.argument();
    // Note that the glob's package is assumed to exist which implies that the package's BUILD file
    // exists which implies that the package's directory exists.
    PathFragment globSubdir = glob.getSubdir();
    if (!globSubdir.equals(PathFragment.EMPTY_FRAGMENT)) {
        PackageLookupValue globSubdirPkgLookupValue = (PackageLookupValue) env.getValue(PackageLookupValue.key(PackageIdentifier.create(glob.getPackageId().getRepository(), glob.getPackageId().getPackageFragment().getRelative(globSubdir))));
        if (globSubdirPkgLookupValue == null) {
            return null;
        }
        if (globSubdirPkgLookupValue.packageExists()) {
            // defines another package, so glob expansion should not descend into that subdir.
            return GlobValue.EMPTY;
        }
    }
    String pattern = glob.getPattern();
    // Split off the first path component of the pattern.
    int slashPos = pattern.indexOf('/');
    String patternHead;
    String patternTail;
    if (slashPos == -1) {
        patternHead = pattern;
        patternTail = null;
    } else {
        // Substrings will share the backing array of the original glob string. That should be fine.
        patternHead = pattern.substring(0, slashPos);
        patternTail = pattern.substring(slashPos + 1);
    }
    NestedSetBuilder<PathFragment> matches = NestedSetBuilder.stableOrder();
    boolean globMatchesBareFile = patternTail == null;
    // "**" also matches an empty segment, so try the case where it is not present.
    if ("**".equals(patternHead)) {
        if (globMatchesBareFile) {
            // Recursive globs aren't supposed to match the package's directory.
            if (!glob.excludeDirs() && !globSubdir.equals(PathFragment.EMPTY_FRAGMENT)) {
                matches.add(globSubdir);
            }
        } else {
            SkyKey globKey = GlobValue.internalKey(glob.getPackageId(), glob.getPackageRoot(), globSubdir, patternTail, glob.excludeDirs());
            GlobValue globValue = (GlobValue) env.getValue(globKey);
            if (globValue == null) {
                return null;
            }
            matches.addTransitive(globValue.getMatches());
        }
    }
    PathFragment dirPathFragment = glob.getPackageId().getPackageFragment().getRelative(globSubdir);
    RootedPath dirRootedPath = RootedPath.toRootedPath(glob.getPackageRoot(), dirPathFragment);
    if (alwaysUseDirListing || containsGlobs(patternHead)) {
        String subdirPattern = "**".equals(patternHead) ? glob.getPattern() : patternTail;
        // Pattern contains globs, so a directory listing is required.
        //
        // Note that we have good reason to believe the directory exists: if this is the
        // top-level directory of the package, the package's existence implies the directory's
        // existence; if this is a lower-level directory in the package, then we got here from
        // previous directory listings. Filesystem operations concurrent with build could mean the
        // directory no longer exists, but DirectoryListingFunction handles that gracefully.
        DirectoryListingValue listingValue = (DirectoryListingValue) env.getValue(DirectoryListingValue.key(dirRootedPath));
        if (listingValue == null) {
            return null;
        }
        // In order to batch Skyframe requests, we do three passes over the directory:
        // (1) Process every dirent, keeping track of values we need to request if the dirent cannot
        //     be processed with current information (symlink targets and subdirectory globs/package
        //     lookups for some subdirectories).
        // (2) Get those values and process the symlinks, keeping track of subdirectory globs/package
        //     lookups we may need to request in case the symlink's target is a directory.
        // (3) Process the necessary subdirectories.
        int direntsSize = listingValue.getDirents().size();
        Map<SkyKey, Dirent> symlinkFileMap = Maps.newHashMapWithExpectedSize(direntsSize);
        Map<SkyKey, Dirent> subdirMap = Maps.newHashMapWithExpectedSize(direntsSize);
        Map<Dirent, Object> sortedResultMap = Maps.newTreeMap();
        // First pass: do normal files and collect SkyKeys to request for subdirectories and symlinks.
        for (Dirent dirent : listingValue.getDirents()) {
            Type direntType = dirent.getType();
            String fileName = dirent.getName();
            if (!UnixGlob.matches(patternHead, fileName, regexPatternCache)) {
                continue;
            }
            if (direntType == Dirent.Type.SYMLINK) {
                // TODO(bazel-team): Consider extracting the symlink resolution logic.
                // For symlinks, look up the corresponding FileValue. This ensures that if the symlink
                // changes and "switches types" (say, from a file to a directory), this value will be
                // invalidated. We also need the target's type to properly process the symlink.
                symlinkFileMap.put(FileValue.key(RootedPath.toRootedPath(glob.getPackageRoot(), dirPathFragment.getRelative(fileName))), dirent);
                continue;
            }
            if (direntType == Dirent.Type.DIRECTORY) {
                SkyKey keyToRequest = getSkyKeyForSubdir(fileName, glob, subdirPattern);
                if (keyToRequest != null) {
                    subdirMap.put(keyToRequest, dirent);
                }
            } else if (globMatchesBareFile) {
                sortedResultMap.put(dirent, glob.getSubdir().getRelative(fileName));
            }
        }
        Map<SkyKey, SkyValue> subdirAndSymlinksResult = env.getValues(Sets.union(subdirMap.keySet(), symlinkFileMap.keySet()));
        if (env.valuesMissing()) {
            return null;
        }
        Map<SkyKey, Dirent> symlinkSubdirMap = Maps.newHashMapWithExpectedSize(symlinkFileMap.size());
        // Also process any known directories.
        for (Map.Entry<SkyKey, SkyValue> lookedUpKeyAndValue : subdirAndSymlinksResult.entrySet()) {
            if (symlinkFileMap.containsKey(lookedUpKeyAndValue.getKey())) {
                FileValue symlinkFileValue = (FileValue) lookedUpKeyAndValue.getValue();
                if (!symlinkFileValue.isSymlink()) {
                    throw new GlobFunctionException(new InconsistentFilesystemException("readdir and stat disagree about whether " + ((RootedPath) lookedUpKeyAndValue.getKey().argument()).asPath() + " is a symlink."), Transience.TRANSIENT);
                }
                if (!symlinkFileValue.exists()) {
                    continue;
                }
                Dirent dirent = symlinkFileMap.get(lookedUpKeyAndValue.getKey());
                String fileName = dirent.getName();
                if (symlinkFileValue.isDirectory()) {
                    SkyKey keyToRequest = getSkyKeyForSubdir(fileName, glob, subdirPattern);
                    if (keyToRequest != null) {
                        symlinkSubdirMap.put(keyToRequest, dirent);
                    }
                } else if (globMatchesBareFile) {
                    sortedResultMap.put(dirent, glob.getSubdir().getRelative(fileName));
                }
            } else {
                processSubdir(lookedUpKeyAndValue, subdirMap, glob, sortedResultMap);
            }
        }
        Map<SkyKey, SkyValue> symlinkSubdirResult = env.getValues(symlinkSubdirMap.keySet());
        if (env.valuesMissing()) {
            return null;
        }
        // pass.
        for (Map.Entry<SkyKey, SkyValue> lookedUpKeyAndValue : symlinkSubdirResult.entrySet()) {
            processSubdir(lookedUpKeyAndValue, symlinkSubdirMap, glob, sortedResultMap);
        }
        for (Map.Entry<Dirent, Object> fileMatches : sortedResultMap.entrySet()) {
            addToMatches(fileMatches.getValue(), matches);
        }
    } else {
        // Pattern does not contain globs, so a direct stat is enough.
        String fileName = patternHead;
        RootedPath fileRootedPath = RootedPath.toRootedPath(glob.getPackageRoot(), dirPathFragment.getRelative(fileName));
        FileValue fileValue = (FileValue) env.getValue(FileValue.key(fileRootedPath));
        if (fileValue == null) {
            return null;
        }
        if (fileValue.exists()) {
            if (fileValue.isDirectory()) {
                SkyKey keyToRequest = getSkyKeyForSubdir(fileName, glob, patternTail);
                if (keyToRequest != null) {
                    SkyValue valueRequested = env.getValue(keyToRequest);
                    if (env.valuesMissing()) {
                        return null;
                    }
                    Object fileMatches = getSubdirMatchesFromSkyValue(fileName, glob, valueRequested);
                    if (fileMatches != null) {
                        addToMatches(fileMatches, matches);
                    }
                }
            } else if (globMatchesBareFile) {
                matches.add(glob.getSubdir().getRelative(fileName));
            }
        }
    }
    Preconditions.checkState(!env.valuesMissing(), skyKey);
    NestedSet<PathFragment> matchesBuilt = matches.build();
    // Use the same value to represent that we did not match anything.
    if (matchesBuilt.isEmpty()) {
        return GlobValue.EMPTY;
    }
    return new GlobValue(matchesBuilt);
}

Example 28 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class GraphBackedRecursivePackageProvider method bulkGetPackages.

@Override
public Map<PackageIdentifier, Package> bulkGetPackages(Iterable<PackageIdentifier> pkgIds) throws NoSuchPackageException, InterruptedException {
    Set<SkyKey> pkgKeys = ImmutableSet.copyOf(PackageValue.keys(pkgIds));
    ImmutableMap.Builder<PackageIdentifier, Package> pkgResults = ImmutableMap.builder();
    Map<SkyKey, SkyValue> packages = graph.getSuccessfulValues(pkgKeys);
    for (Map.Entry<SkyKey, SkyValue> pkgEntry : packages.entrySet()) {
        PackageIdentifier pkgId = (PackageIdentifier) pkgEntry.getKey().argument();
        PackageValue pkgValue = (PackageValue) pkgEntry.getValue();
        pkgResults.put(pkgId, Preconditions.checkNotNull(pkgValue.getPackage(), pkgId));
    }
    SetView<SkyKey> unknownKeys = Sets.difference(pkgKeys, packages.keySet());
    if (!Iterables.isEmpty(unknownKeys)) {
        LOGGER.warning("Unable to find " + unknownKeys + " in the batch lookup of " + pkgKeys + ". Successfully looked up " + packages.keySet());
    }
    for (Map.Entry<SkyKey, Exception> missingOrExceptionEntry : graph.getMissingAndExceptions(unknownKeys).entrySet()) {
        PackageIdentifier pkgIdentifier = (PackageIdentifier) missingOrExceptionEntry.getKey().argument();
        Exception exception = missingOrExceptionEntry.getValue();
        if (exception == null) {
            // package, because the SkyQuery environment has already loaded the universe.
            throw new BuildFileNotFoundException(pkgIdentifier, "Package not found");
        }
        Throwables.propagateIfInstanceOf(exception, NoSuchPackageException.class);
        Throwables.propagate(exception);
    }
    return pkgResults.build();
}

Example 29 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class GraphBackedRecursivePackageProvider method collectPackagesUnder.

private void collectPackagesUnder(ExtendedEventHandler eventHandler, final RepositoryName repository, Set<TraversalInfo> traversals, ImmutableList.Builder<PathFragment> builder) throws InterruptedException {
    Map<TraversalInfo, SkyKey> traversalToKeyMap = Maps.asMap(traversals, new Function<TraversalInfo, SkyKey>() {

        @Override
        public SkyKey apply(TraversalInfo traversalInfo) {
            return CollectPackagesUnderDirectoryValue.key(repository, traversalInfo.rootedDir, traversalInfo.excludedSubdirectories);
        }
    });
    Map<SkyKey, SkyValue> values = graph.getSuccessfulValues(traversalToKeyMap.values());
    ImmutableSet.Builder<TraversalInfo> subdirTraversalBuilder = ImmutableSet.builder();
    for (Map.Entry<TraversalInfo, SkyKey> entry : traversalToKeyMap.entrySet()) {
        TraversalInfo info = entry.getKey();
        SkyKey key = entry.getValue();
        SkyValue val = values.get(key);
        CollectPackagesUnderDirectoryValue collectPackagesValue = (CollectPackagesUnderDirectoryValue) val;
        if (collectPackagesValue != null) {
            if (collectPackagesValue.isDirectoryPackage()) {
                builder.add(info.rootedDir.getRelativePath());
            }
            if (collectPackagesValue.getErrorMessage() != null) {
                eventHandler.handle(Event.error(collectPackagesValue.getErrorMessage()));
            }
            ImmutableMap<RootedPath, Boolean> subdirectoryTransitivelyContainsPackages = collectPackagesValue.getSubdirectoryTransitivelyContainsPackagesOrErrors();
            for (RootedPath subdirectory : subdirectoryTransitivelyContainsPackages.keySet()) {
                if (subdirectoryTransitivelyContainsPackages.get(subdirectory)) {
                    PathFragment subdirectoryRelativePath = subdirectory.getRelativePath();
                    ImmutableSet<PathFragment> excludedSubdirectoriesBeneathThisSubdirectory = PathFragment.filterPathsStartingWith(info.excludedSubdirectories, subdirectoryRelativePath);
                    subdirTraversalBuilder.add(new TraversalInfo(subdirectory, excludedSubdirectoriesBeneathThisSubdirectory));
                }
            }
        }
    }
    ImmutableSet<TraversalInfo> subdirTraversals = subdirTraversalBuilder.build();
    if (!subdirTraversals.isEmpty()) {
        collectPackagesUnder(eventHandler, repository, subdirTraversals, builder);
    }
}

Example 30 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class SkyframeExecutor method getConfigurations.

/**
   * Retrieves the configurations needed for the given deps. If {@link
   * BuildConfiguration.Options#trimConfigurations()} is true, trims their fragments to only those
   * needed by their transitive closures. Else unconditionally includes all fragments.
   *
   * <p>Skips targets with loading phase errors.
   */
public Multimap<Dependency, BuildConfiguration> getConfigurations(ExtendedEventHandler eventHandler, BuildOptions fromOptions, Iterable<Dependency> keys) {
    Multimap<Dependency, BuildConfiguration> builder = ArrayListMultimap.<Dependency, BuildConfiguration>create();
    Set<Dependency> depsToEvaluate = new HashSet<>();
    // Check: if !Configuration.useDynamicConfigs then just return the original configs.
    Set<Class<? extends BuildConfiguration.Fragment>> allFragments = null;
    if (useUntrimmedDynamicConfigs(fromOptions)) {
        allFragments = ((ConfiguredRuleClassProvider) ruleClassProvider).getAllFragments();
    }
    // Get the fragments needed for dynamic configuration nodes.
    final List<SkyKey> transitiveFragmentSkyKeys = new ArrayList<>();
    Map<Label, Set<Class<? extends BuildConfiguration.Fragment>>> fragmentsMap = new HashMap<>();
    Set<Label> labelsWithErrors = new HashSet<>();
    for (Dependency key : keys) {
        if (key.hasStaticConfiguration()) {
            builder.put(key, key.getConfiguration());
        } else if (useUntrimmedDynamicConfigs(fromOptions)) {
            fragmentsMap.put(key.getLabel(), allFragments);
        } else {
            depsToEvaluate.add(key);
            transitiveFragmentSkyKeys.add(TransitiveTargetValue.key(key.getLabel()));
        }
    }
    EvaluationResult<SkyValue> fragmentsResult = evaluateSkyKeys(eventHandler, transitiveFragmentSkyKeys, /*keepGoing=*/
    true);
    for (Map.Entry<SkyKey, ErrorInfo> entry : fragmentsResult.errorMap().entrySet()) {
        reportCycles(eventHandler, entry.getValue().getCycleInfo(), entry.getKey());
    }
    for (Dependency key : keys) {
        if (!depsToEvaluate.contains(key)) {
        // No fragments to compute here.
        } else if (fragmentsResult.getError(TransitiveTargetValue.key(key.getLabel())) != null) {
            labelsWithErrors.add(key.getLabel());
        } else {
            TransitiveTargetValue ttv = (TransitiveTargetValue) fragmentsResult.get(TransitiveTargetValue.key(key.getLabel()));
            fragmentsMap.put(key.getLabel(), ttv.getTransitiveConfigFragments().toSet());
        }
    }
    // Now get the configurations.
    final List<SkyKey> configSkyKeys = new ArrayList<>();
    for (Dependency key : keys) {
        if (labelsWithErrors.contains(key.getLabel()) || key.hasStaticConfiguration()) {
            continue;
        }
        Set<Class<? extends BuildConfiguration.Fragment>> depFragments = fragmentsMap.get(key.getLabel());
        if (depFragments != null) {
            for (BuildOptions toOptions : ConfiguredTargetFunction.getDynamicTransitionOptions(fromOptions, key.getTransition(), depFragments, ruleClassProvider, true)) {
                configSkyKeys.add(BuildConfigurationValue.key(depFragments, toOptions));
            }
        }
    }
    EvaluationResult<SkyValue> configsResult = evaluateSkyKeys(eventHandler, configSkyKeys, /*keepGoing=*/
    true);
    for (Dependency key : keys) {
        if (labelsWithErrors.contains(key.getLabel()) || key.hasStaticConfiguration()) {
            continue;
        }
        Set<Class<? extends BuildConfiguration.Fragment>> depFragments = fragmentsMap.get(key.getLabel());
        if (depFragments != null) {
            for (BuildOptions toOptions : ConfiguredTargetFunction.getDynamicTransitionOptions(fromOptions, key.getTransition(), depFragments, ruleClassProvider, true)) {
                SkyKey configKey = BuildConfigurationValue.key(depFragments, toOptions);
                builder.put(key, ((BuildConfigurationValue) configsResult.get(configKey)).getConfiguration());
            }
        }
    }
    return builder;
}