Examples with SkyValue com.google.devtools.build.skyframe.SkyValue used on opensource projects

Example 11 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class ActionExecutionFunction method addDiscoveredInputs.

private static void addDiscoveredInputs(Map<Artifact, FileArtifactValue> inputData, Map<Artifact, Collection<Artifact>> expandedArtifacts, Iterable<Artifact> discoveredInputs, Environment env) throws InterruptedException {
    // We do not do a getValuesOrThrow() call for the following reasons:
    // 1. No exceptions can be thrown for non-mandatory inputs;
    // 2. Any derived inputs must be in the transitive closure of this action's inputs. Therefore,
    // if there was an error building one of them, then that exception would have percolated up to
    // this action already, through one of its declared inputs, and we would not have reached input
    // discovery.
    // Therefore there is no need to catch and rethrow exceptions as there is with #checkInputs.
    Map<SkyKey, SkyValue> nonMandatoryDiscovered = env.getValues(newlyDiscoveredInputsToSkyKeys(discoveredInputs, inputData.keySet()));
    if (!env.valuesMissing()) {
        for (Entry<SkyKey, SkyValue> entry : nonMandatoryDiscovered.entrySet()) {
            Artifact input = ArtifactSkyKey.artifact(entry.getKey());
            if (entry.getValue() instanceof TreeArtifactValue) {
                TreeArtifactValue treeValue = (TreeArtifactValue) entry.getValue();
                expandedArtifacts.put(input, ImmutableSet.<Artifact>copyOf(treeValue.getChildren()));
                inputData.putAll(treeValue.getChildValues());
                inputData.put(input, treeValue.getSelfData());
            } else {
                inputData.put(input, (FileArtifactValue) entry.getValue());
            }
        }
    }
}

Example 12 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class ActionExecutionFunction method checkInputs.

/**
   * Declare dependency on all known inputs of action. Throws exception if any are known to be
   * missing. Some inputs may not yet be in the graph, in which case the builder should abort.
   */
private Pair<Map<Artifact, FileArtifactValue>, Map<Artifact, Collection<Artifact>>> checkInputs(Environment env, Action action, Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps) throws ActionExecutionException {
    int missingCount = 0;
    int actionFailures = 0;
    boolean catastrophe = false;
    // Only populate input data if we have the input values, otherwise they'll just go unused.
    // We still want to loop through the inputs to collect missing deps errors. During the
    // evaluator "error bubbling", we may get one last chance at reporting errors even though
    // some deps are still missing.
    boolean populateInputData = !env.valuesMissing();
    NestedSetBuilder<Cause> rootCauses = NestedSetBuilder.stableOrder();
    Map<Artifact, FileArtifactValue> inputArtifactData = new HashMap<>(populateInputData ? inputDeps.size() : 0);
    Map<Artifact, Collection<Artifact>> expandedArtifacts = new HashMap<>(populateInputData ? 128 : 0);
    ActionExecutionException firstActionExecutionException = null;
    for (Map.Entry<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> depsEntry : inputDeps.entrySet()) {
        Artifact input = ArtifactSkyKey.artifact(depsEntry.getKey());
        try {
            SkyValue value = depsEntry.getValue().get();
            if (populateInputData) {
                if (value instanceof AggregatingArtifactValue) {
                    AggregatingArtifactValue aggregatingValue = (AggregatingArtifactValue) value;
                    for (Pair<Artifact, FileArtifactValue> entry : aggregatingValue.getInputs()) {
                        inputArtifactData.put(entry.first, entry.second);
                    }
                    // We have to cache the "digest" of the aggregating value itself,
                    // because the action cache checker may want it.
                    inputArtifactData.put(input, aggregatingValue.getSelfData());
                    ImmutableList.Builder<Artifact> expansionBuilder = ImmutableList.builder();
                    for (Pair<Artifact, FileArtifactValue> pair : aggregatingValue.getInputs()) {
                        expansionBuilder.add(pair.first);
                    }
                    expandedArtifacts.put(input, expansionBuilder.build());
                } else if (value instanceof TreeArtifactValue) {
                    TreeArtifactValue treeValue = (TreeArtifactValue) value;
                    expandedArtifacts.put(input, ImmutableSet.<Artifact>copyOf(treeValue.getChildren()));
                    inputArtifactData.putAll(treeValue.getChildValues());
                    // Again, we cache the "digest" of the value for cache checking.
                    inputArtifactData.put(input, treeValue.getSelfData());
                } else {
                    Preconditions.checkState(value instanceof FileArtifactValue, depsEntry);
                    inputArtifactData.put(input, (FileArtifactValue) value);
                }
            }
        } catch (MissingInputFileException e) {
            missingCount++;
            if (input.getOwner() != null) {
                rootCauses.add(new LabelCause(input.getOwner()));
            }
        } catch (ActionExecutionException e) {
            actionFailures++;
            if (firstActionExecutionException == null) {
                firstActionExecutionException = e;
            }
            catastrophe = catastrophe || e.isCatastrophe();
            rootCauses.addTransitive(e.getRootCauses());
        }
    }
    // We need to rethrow first exception because it can contain useful error message
    if (firstActionExecutionException != null) {
        if (missingCount == 0 && actionFailures == 1) {
            // having to copy the root causes to the upwards transitive closure.
            throw firstActionExecutionException;
        }
        throw new ActionExecutionException(firstActionExecutionException.getMessage(), firstActionExecutionException.getCause(), action, rootCauses.build(), catastrophe, firstActionExecutionException.getExitCode());
    }
    if (missingCount > 0) {
        for (Cause missingInput : rootCauses.build()) {
            env.getListener().handle(Event.error(action.getOwner().getLocation(), String.format("%s: missing input file '%s'", action.getOwner().getLabel(), missingInput.getLabel())));
        }
        throw new ActionExecutionException(missingCount + " input file(s) do not exist", action, rootCauses.build(), /*catastrophe=*/
        false);
    }
    return Pair.of(inputArtifactData, expandedArtifacts);
}

Example 13 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class SkyframeExecutorTestUtils method getAllExistingConfiguredTargets.

/**
   * Returns all configured targets currently in the graph.
   */
public static Iterable<ConfiguredTarget> getAllExistingConfiguredTargets(SkyframeExecutor skyframeExecutor) {
    Collection<SkyValue> values = Maps.filterKeys(skyframeExecutor.getEvaluatorForTesting().getValues(), SkyFunctions.isSkyFunction(SkyFunctions.CONFIGURED_TARGET)).values();
    List<ConfiguredTarget> cts = Lists.newArrayList();
    for (SkyValue value : values) {
        if (value != null) {
            cts.add(((ConfiguredTargetValue) value).getConfiguredTarget());
        }
    }
    return cts;
}

Example 14 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class TreeArtifactMetadataTest method evaluateArtifactValue.

private SkyValue evaluateArtifactValue(Artifact artifact, boolean mandatory) throws Exception {
    SkyKey key = ArtifactSkyKey.key(artifact, mandatory);
    EvaluationResult<SkyValue> result = evaluate(key);
    if (result.hasError()) {
        throw result.getError().getException();
    }
    return result.get(key);
}

Example 15 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class GlobFunction method compute.

@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws GlobFunctionException, InterruptedException {
    GlobDescriptor glob = (GlobDescriptor) skyKey.argument();
    // Note that the glob's package is assumed to exist which implies that the package's BUILD file
    // exists which implies that the package's directory exists.
    PathFragment globSubdir = glob.getSubdir();
    if (!globSubdir.equals(PathFragment.EMPTY_FRAGMENT)) {
        PackageLookupValue globSubdirPkgLookupValue = (PackageLookupValue) env.getValue(PackageLookupValue.key(PackageIdentifier.create(glob.getPackageId().getRepository(), glob.getPackageId().getPackageFragment().getRelative(globSubdir))));
        if (globSubdirPkgLookupValue == null) {
            return null;
        }
        if (globSubdirPkgLookupValue.packageExists()) {
            // defines another package, so glob expansion should not descend into that subdir.
            return GlobValue.EMPTY;
        }
    }
    String pattern = glob.getPattern();
    // Split off the first path component of the pattern.
    int slashPos = pattern.indexOf('/');
    String patternHead;
    String patternTail;
    if (slashPos == -1) {
        patternHead = pattern;
        patternTail = null;
    } else {
        // Substrings will share the backing array of the original glob string. That should be fine.
        patternHead = pattern.substring(0, slashPos);
        patternTail = pattern.substring(slashPos + 1);
    }
    NestedSetBuilder<PathFragment> matches = NestedSetBuilder.stableOrder();
    boolean globMatchesBareFile = patternTail == null;
    // "**" also matches an empty segment, so try the case where it is not present.
    if ("**".equals(patternHead)) {
        if (globMatchesBareFile) {
            // Recursive globs aren't supposed to match the package's directory.
            if (!glob.excludeDirs() && !globSubdir.equals(PathFragment.EMPTY_FRAGMENT)) {
                matches.add(globSubdir);
            }
        } else {
            SkyKey globKey = GlobValue.internalKey(glob.getPackageId(), glob.getPackageRoot(), globSubdir, patternTail, glob.excludeDirs());
            GlobValue globValue = (GlobValue) env.getValue(globKey);
            if (globValue == null) {
                return null;
            }
            matches.addTransitive(globValue.getMatches());
        }
    }
    PathFragment dirPathFragment = glob.getPackageId().getPackageFragment().getRelative(globSubdir);
    RootedPath dirRootedPath = RootedPath.toRootedPath(glob.getPackageRoot(), dirPathFragment);
    if (alwaysUseDirListing || containsGlobs(patternHead)) {
        String subdirPattern = "**".equals(patternHead) ? glob.getPattern() : patternTail;
        // Pattern contains globs, so a directory listing is required.
        //
        // Note that we have good reason to believe the directory exists: if this is the
        // top-level directory of the package, the package's existence implies the directory's
        // existence; if this is a lower-level directory in the package, then we got here from
        // previous directory listings. Filesystem operations concurrent with build could mean the
        // directory no longer exists, but DirectoryListingFunction handles that gracefully.
        DirectoryListingValue listingValue = (DirectoryListingValue) env.getValue(DirectoryListingValue.key(dirRootedPath));
        if (listingValue == null) {
            return null;
        }
        // In order to batch Skyframe requests, we do three passes over the directory:
        // (1) Process every dirent, keeping track of values we need to request if the dirent cannot
        //     be processed with current information (symlink targets and subdirectory globs/package
        //     lookups for some subdirectories).
        // (2) Get those values and process the symlinks, keeping track of subdirectory globs/package
        //     lookups we may need to request in case the symlink's target is a directory.
        // (3) Process the necessary subdirectories.
        int direntsSize = listingValue.getDirents().size();
        Map<SkyKey, Dirent> symlinkFileMap = Maps.newHashMapWithExpectedSize(direntsSize);
        Map<SkyKey, Dirent> subdirMap = Maps.newHashMapWithExpectedSize(direntsSize);
        Map<Dirent, Object> sortedResultMap = Maps.newTreeMap();
        // First pass: do normal files and collect SkyKeys to request for subdirectories and symlinks.
        for (Dirent dirent : listingValue.getDirents()) {
            Type direntType = dirent.getType();
            String fileName = dirent.getName();
            if (!UnixGlob.matches(patternHead, fileName, regexPatternCache)) {
                continue;
            }
            if (direntType == Dirent.Type.SYMLINK) {
                // TODO(bazel-team): Consider extracting the symlink resolution logic.
                // For symlinks, look up the corresponding FileValue. This ensures that if the symlink
                // changes and "switches types" (say, from a file to a directory), this value will be
                // invalidated. We also need the target's type to properly process the symlink.
                symlinkFileMap.put(FileValue.key(RootedPath.toRootedPath(glob.getPackageRoot(), dirPathFragment.getRelative(fileName))), dirent);
                continue;
            }
            if (direntType == Dirent.Type.DIRECTORY) {
                SkyKey keyToRequest = getSkyKeyForSubdir(fileName, glob, subdirPattern);
                if (keyToRequest != null) {
                    subdirMap.put(keyToRequest, dirent);
                }
            } else if (globMatchesBareFile) {
                sortedResultMap.put(dirent, glob.getSubdir().getRelative(fileName));
            }
        }
        Map<SkyKey, SkyValue> subdirAndSymlinksResult = env.getValues(Sets.union(subdirMap.keySet(), symlinkFileMap.keySet()));
        if (env.valuesMissing()) {
            return null;
        }
        Map<SkyKey, Dirent> symlinkSubdirMap = Maps.newHashMapWithExpectedSize(symlinkFileMap.size());
        // Also process any known directories.
        for (Map.Entry<SkyKey, SkyValue> lookedUpKeyAndValue : subdirAndSymlinksResult.entrySet()) {
            if (symlinkFileMap.containsKey(lookedUpKeyAndValue.getKey())) {
                FileValue symlinkFileValue = (FileValue) lookedUpKeyAndValue.getValue();
                if (!symlinkFileValue.isSymlink()) {
                    throw new GlobFunctionException(new InconsistentFilesystemException("readdir and stat disagree about whether " + ((RootedPath) lookedUpKeyAndValue.getKey().argument()).asPath() + " is a symlink."), Transience.TRANSIENT);
                }
                if (!symlinkFileValue.exists()) {
                    continue;
                }
                Dirent dirent = symlinkFileMap.get(lookedUpKeyAndValue.getKey());
                String fileName = dirent.getName();
                if (symlinkFileValue.isDirectory()) {
                    SkyKey keyToRequest = getSkyKeyForSubdir(fileName, glob, subdirPattern);
                    if (keyToRequest != null) {
                        symlinkSubdirMap.put(keyToRequest, dirent);
                    }
                } else if (globMatchesBareFile) {
                    sortedResultMap.put(dirent, glob.getSubdir().getRelative(fileName));
                }
            } else {
                processSubdir(lookedUpKeyAndValue, subdirMap, glob, sortedResultMap);
            }
        }
        Map<SkyKey, SkyValue> symlinkSubdirResult = env.getValues(symlinkSubdirMap.keySet());
        if (env.valuesMissing()) {
            return null;
        }
        // pass.
        for (Map.Entry<SkyKey, SkyValue> lookedUpKeyAndValue : symlinkSubdirResult.entrySet()) {
            processSubdir(lookedUpKeyAndValue, symlinkSubdirMap, glob, sortedResultMap);
        }
        for (Map.Entry<Dirent, Object> fileMatches : sortedResultMap.entrySet()) {
            addToMatches(fileMatches.getValue(), matches);
        }
    } else {
        // Pattern does not contain globs, so a direct stat is enough.
        String fileName = patternHead;
        RootedPath fileRootedPath = RootedPath.toRootedPath(glob.getPackageRoot(), dirPathFragment.getRelative(fileName));
        FileValue fileValue = (FileValue) env.getValue(FileValue.key(fileRootedPath));
        if (fileValue == null) {
            return null;
        }
        if (fileValue.exists()) {
            if (fileValue.isDirectory()) {
                SkyKey keyToRequest = getSkyKeyForSubdir(fileName, glob, patternTail);
                if (keyToRequest != null) {
                    SkyValue valueRequested = env.getValue(keyToRequest);
                    if (env.valuesMissing()) {
                        return null;
                    }
                    Object fileMatches = getSubdirMatchesFromSkyValue(fileName, glob, valueRequested);
                    if (fileMatches != null) {
                        addToMatches(fileMatches, matches);
                    }
                }
            } else if (globMatchesBareFile) {
                matches.add(glob.getSubdir().getRelative(fileName));
            }
        }
    }
    Preconditions.checkState(!env.valuesMissing(), skyKey);
    NestedSet<PathFragment> matchesBuilt = matches.build();
    // Use the same value to represent that we did not match anything.
    if (matchesBuilt.isEmpty()) {
        return GlobValue.EMPTY;
    }
    return new GlobValue(matchesBuilt);
}