Examples with ValueOrException com.google.devtools.build.skyframe.ValueOrException used on opensource projects

Example 6 with ValueOrException

use of com.google.devtools.build.skyframe.ValueOrException in project bazel by bazelbuild.

the class ConfiguredTargetFunction method getDynamicConfigurations.

/**
   * Creates a dynamic configuration for each dep that's custom-fitted specifically for that dep.
   *
   * <p>More specifically: given a set of {@link Dependency} instances holding dynamic config
   * transition requests (e.g. {@link Dependency#hasStaticConfiguration()} == false}), returns
   * equivalent dependencies containing dynamically created configurations applying those
   * transitions. If {@link BuildConfiguration.Options#trimConfigurations()} is true, these
   * configurations only contain the fragments needed by the dep and its transitive closure. Else
   * the configurations unconditionally include all fragments.
   *
   * <p>This method is heavily performance-optimized. Because it, in aggregate, reads over every
   * edge in the configured target graph, small inefficiencies can have observable impact on
   * analysis time. Keep this in mind when making modifications and performance-test any changes you
   * make.
   *
   * @param env Skyframe evaluation environment
   * @param ctgValue the label and the configuration of the node
   * @param originalDeps the set of configuration transition requests for this target's attributes
   * @param hostConfiguration the host configuration
   * @param ruleClassProvider the rule class provider for determining the right configuration
   *    fragments to apply to deps
   *
   * @return a mapping from each attribute to the {@link BuildConfiguration}s and {@link Label}s
   *    to use for that attribute's deps. Returns null if not all Skyframe dependencies are
   *    available yet.
   */
@Nullable
static OrderedSetMultimap<Attribute, Dependency> getDynamicConfigurations(Environment env, TargetAndConfiguration ctgValue, OrderedSetMultimap<Attribute, Dependency> originalDeps, BuildConfiguration hostConfiguration, RuleClassProvider ruleClassProvider) throws DependencyEvaluationException, InterruptedException {
    // Maps each Skyframe-evaluated BuildConfiguration to the dependencies that need that
    // configuration. For cases where Skyframe isn't needed to get the configuration (e.g. when
    // we just re-used the original rule's configuration), we should skip this outright.
    Multimap<SkyKey, Map.Entry<Attribute, Dependency>> keysToEntries = LinkedListMultimap.create();
    // Stores the result of applying a dynamic transition to the current configuration using a
    // particular subset of fragments. By caching this, we save from redundantly computing the
    // same transition for every dependency edge that requests that transition. This can have
    // real effect on analysis time for commonly triggered transitions.
    //
    // Split transitions may map to multiple values. All other transitions map to one.
    Map<FragmentsAndTransition, List<BuildOptions>> transitionsMap = new LinkedHashMap<>();
    // The fragments used by the current target's configuration.
    Set<Class<? extends BuildConfiguration.Fragment>> ctgFragments = ctgValue.getConfiguration().fragmentClasses();
    BuildOptions ctgOptions = ctgValue.getConfiguration().getOptions();
    // Stores the dynamically configured versions of each dependency. This method must preserve the
    // original label ordering of each attribute. For example, if originalDeps.get("data") is
    // [":a", ":b"], the dynamic variant must also be [":a", ":b"] in the same order. Because we may
    // not actualize the results in order (some results need Skyframe-evaluated configurations while
    // others can be computed trivially), we dump them all into this map, then as a final step
    // iterate through the original list and pluck out values from here for the final value.
    //
    // For split transitions, originaldeps.get("data") = [":a", ":b"] can produce the output
    // [":a"<config1>, ":a"<config2>, ..., ":b"<config1>, ":b"<config2>, ...]. All instances of ":a"
    // still appear before all instances of ":b". But the [":a"<config1>, ":a"<config2>"] subset may
    // be in any (deterministic) order. In particular, this may not be the same order as
    // SplitTransition.split. If needed, this code can be modified to use that order, but that
    // involves more runtime in performance-critical code, so we won't make that change without a
    // clear need.
    //
    // This map is used heavily by all builds. Inserts and gets should be as fast as possible.
    Multimap<AttributeAndLabel, Dependency> dynamicDeps = LinkedHashMultimap.create();
    // Performance optimization: This method iterates over originalDeps twice. By storing
    // AttributeAndLabel instances in this list, we avoid having to recreate them the second time
    // (particularly avoid recomputing their hash codes). Profiling shows this shaves 25% off this
    // method's execution time (at the time of this comment).
    ArrayList<AttributeAndLabel> attributesAndLabels = new ArrayList<>(originalDeps.size());
    for (Map.Entry<Attribute, Dependency> depsEntry : originalDeps.entries()) {
        Dependency dep = depsEntry.getValue();
        AttributeAndLabel attributeAndLabel = new AttributeAndLabel(depsEntry.getKey(), dep.getLabel());
        attributesAndLabels.add(attributeAndLabel);
        // simple cc_binary show this saves ~1% of total analysis phase time.
        if (dep.hasStaticConfiguration()) {
            continue;
        }
        // Figure out the required fragments for this dep and its transitive closure.
        Set<Class<? extends BuildConfiguration.Fragment>> depFragments = getTransitiveFragments(env, dep.getLabel(), ctgValue.getConfiguration());
        if (depFragments == null) {
            return null;
        }
        // to 0.5% of total analysis time as profiled over a simple cc_binary).
        if (ctgValue.getConfiguration().trimConfigurations()) {
            checkForMissingFragments(env, ctgValue, attributeAndLabel.attribute.getName(), dep, depFragments);
        }
        boolean sameFragments = depFragments.equals(ctgFragments);
        Attribute.Transition transition = dep.getTransition();
        if (sameFragments) {
            if (transition == Attribute.ConfigurationTransition.NONE) {
                // The dep uses the same exact configuration.
                putOnlyEntry(dynamicDeps, attributeAndLabel, Dependency.withConfigurationAndAspects(dep.getLabel(), ctgValue.getConfiguration(), dep.getAspects()));
                continue;
            } else if (transition == HostTransition.INSTANCE) {
                // The current rule's host configuration can also be used for the dep. We short-circuit
                // the standard transition logic for host transitions because these transitions are
                // uniquely frequent. It's possible, e.g., for every node in the configured target graph
                // to incur multiple host transitions. So we aggressively optimize to avoid hurting
                // analysis time.
                putOnlyEntry(dynamicDeps, attributeAndLabel, Dependency.withConfigurationAndAspects(dep.getLabel(), hostConfiguration, dep.getAspects()));
                continue;
            }
        }
        // Apply the transition or use the cached result if it was already applied.
        FragmentsAndTransition transitionKey = new FragmentsAndTransition(depFragments, transition);
        List<BuildOptions> toOptions = transitionsMap.get(transitionKey);
        if (toOptions == null) {
            toOptions = getDynamicTransitionOptions(ctgOptions, transition, depFragments, ruleClassProvider, !sameFragments);
            transitionsMap.put(transitionKey, toOptions);
        }
        // configuration.
        if (sameFragments && toOptions.size() == 1 && Iterables.getOnlyElement(toOptions).equals(ctgOptions)) {
            putOnlyEntry(dynamicDeps, attributeAndLabel, Dependency.withConfigurationAndAspects(dep.getLabel(), ctgValue.getConfiguration(), dep.getAspects()));
            continue;
        }
        // If we get here, we have to get the configuration from Skyframe.
        for (BuildOptions options : toOptions) {
            keysToEntries.put(BuildConfigurationValue.key(depFragments, options), depsEntry);
        }
    }
    // Get all BuildConfigurations we need from Skyframe. While not every value might be available,
    // we don't call env.valuesMissing() here because that could be true from the earlier
    // resolver.dependentNodeMap call in computeDependencies, which also calls Skyframe. This method
    // doesn't need those missing values, but it still has to be called after
    // resolver.dependentNodeMap because it consumes that method's output. The reason the missing
    // values don't matter is because resolver.dependentNodeMap still returns "partial" results
    // and this method runs over whatever's available.
    //
    // While there would be no *correctness* harm in nulling out early, there's significant
    // *performance* harm. Profiling shows that putting "if (env.valuesMissing()) { return null; }"
    // here (or even after resolver.dependentNodeMap) produces a ~30% performance hit on the
    // analysis phase. That's because resolveConfiguredTargetDependencies and
    // resolveAspectDependencies don't get a chance to make their own Skyframe requests before
    // bailing out of this ConfiguredTargetFunction call. Ideally we could batch all requests
    // from all methods into a single Skyframe call, but there are enough subtle data flow
    // dependencies in ConfiguredTargetFucntion to make that impractical.
    Map<SkyKey, ValueOrException<InvalidConfigurationException>> depConfigValues = env.getValuesOrThrow(keysToEntries.keySet(), InvalidConfigurationException.class);
    // Now fill in the remaining unresolved deps with the now-resolved configurations.
    try {
        for (Map.Entry<SkyKey, ValueOrException<InvalidConfigurationException>> entry : depConfigValues.entrySet()) {
            SkyKey key = entry.getKey();
            ValueOrException<InvalidConfigurationException> valueOrException = entry.getValue();
            if (valueOrException.get() == null) {
                // null out on missing values from *this specific Skyframe request*.
                return null;
            }
            BuildConfigurationValue trimmedConfig = (BuildConfigurationValue) valueOrException.get();
            for (Map.Entry<Attribute, Dependency> info : keysToEntries.get(key)) {
                Dependency originalDep = info.getValue();
                AttributeAndLabel attr = new AttributeAndLabel(info.getKey(), originalDep.getLabel());
                Dependency resolvedDep = Dependency.withConfigurationAndAspects(originalDep.getLabel(), trimmedConfig.getConfiguration(), originalDep.getAspects());
                if (attr.attribute.hasSplitConfigurationTransition()) {
                    dynamicDeps.put(attr, resolvedDep);
                } else {
                    putOnlyEntry(dynamicDeps, attr, resolvedDep);
                }
            }
        }
    } catch (InvalidConfigurationException e) {
        throw new DependencyEvaluationException(e);
    }
    return sortDynamicallyConfiguredDeps(originalDeps, dynamicDeps, attributesAndLabels);
}

Example 7 with ValueOrException

use of com.google.devtools.build.skyframe.ValueOrException in project bazel by bazelbuild.

the class BuildConfigurationFunction method getConfigurationFragments.

private Set<Fragment> getConfigurationFragments(BuildConfigurationValue.Key key, Environment env) throws InvalidConfigurationException, InterruptedException {
    // Get SkyKeys for the fragments we need to load.
    Set<SkyKey> fragmentKeys = new LinkedHashSet<>();
    for (Class<? extends BuildConfiguration.Fragment> fragmentClass : key.getFragments()) {
        fragmentKeys.add(ConfigurationFragmentValue.key(key.getBuildOptions(), fragmentClass, ruleClassProvider));
    }
    // Load them as Skyframe deps.
    Map<SkyKey, ValueOrException<InvalidConfigurationException>> fragmentDeps = env.getValuesOrThrow(fragmentKeys, InvalidConfigurationException.class);
    if (env.valuesMissing()) {
        return null;
    }
    // Collect and return the results.
    ImmutableSet.Builder<Fragment> fragments = ImmutableSet.builder();
    for (ValueOrException<InvalidConfigurationException> value : fragmentDeps.values()) {
        BuildConfiguration.Fragment fragment = ((ConfigurationFragmentValue) value.get()).getFragment();
        if (fragment != null) {
            fragments.add(fragment);
        }
    }
    return fragments.build();
}

Example 8 with ValueOrException

use of com.google.devtools.build.skyframe.ValueOrException in project bazel by bazelbuild.

the class PrepareDepsOfPatternsFunction method compute.

/**
   * Given a {@link SkyKey} that contains a sequence of target patterns, when this function returns
   * {@link PrepareDepsOfPatternsValue}, then all targets matching that sequence, and those targets'
   * transitive dependencies, have been loaded.
   */
@Nullable
@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException {
    EventHandler eventHandler = env.getListener();
    ImmutableList<SkyKey> skyKeys = getSkyKeys(skyKey, eventHandler);
    Map<SkyKey, ValueOrException<TargetParsingException>> tokensByKey = env.getValuesOrThrow(skyKeys, TargetParsingException.class);
    if (env.valuesMissing()) {
        return null;
    }
    boolean handlerIsParseFailureListener = eventHandler instanceof ParseFailureListener;
    for (SkyKey key : skyKeys) {
        try {
            // The only exception type throwable by PrepareDepsOfPatternFunction is
            // TargetParsingException. Therefore all ValueOrException values in the map will either
            // be non-null or throw TargetParsingException when get is called.
            Preconditions.checkNotNull(tokensByKey.get(key).get());
        } catch (TargetParsingException e) {
            // If a target pattern can't be evaluated, notify the user of the problem and keep going.
            handleTargetParsingException(eventHandler, handlerIsParseFailureListener, key, e);
        }
    }
    return new PrepareDepsOfPatternsValue(getTargetPatternKeys(skyKeys));
}

Example 9 with ValueOrException

use of com.google.devtools.build.skyframe.ValueOrException in project bazel by bazelbuild.

the class RecursiveDirectoryTraversalFunction method visitDirectory.

/**
   * Looks in the directory specified by {@code recursivePkgKey} for a package, does some work as
   * specified by {@link Visitor} if such a package exists, then recursively does work in each
   * non-excluded subdirectory as specified by {@link #getSkyKeyForSubdirectory}, and finally
   * aggregates the {@link Visitor} value along with values from each subdirectory as specified by
   * {@link #aggregateWithSubdirectorySkyValues}, and returns that aggregation.
   *
   * <p>Returns null if {@code env.valuesMissing()} is true, checked after each call to one of
   * {@link RecursiveDirectoryTraversalFunction}'s abstract methods that were given {@code env}.
   * (And after each of {@code visitDirectory}'s own uses of {@code env}, of course.)
   */
TReturn visitDirectory(RecursivePkgKey recursivePkgKey, Environment env) throws InterruptedException {
    RootedPath rootedPath = recursivePkgKey.getRootedPath();
    ProcessPackageDirectoryResult packageExistenceAndSubdirDeps = processPackageDirectory.getPackageExistenceAndSubdirDeps(rootedPath, recursivePkgKey.getRepository(), env, recursivePkgKey.getExcludedPaths());
    if (env.valuesMissing()) {
        return null;
    }
    Iterable<SkyKey> childDeps = packageExistenceAndSubdirDeps.getChildDeps();
    TVisitor visitor = getInitialVisitor();
    Map<SkyKey, SkyValue> subdirectorySkyValues;
    if (packageExistenceAndSubdirDeps.packageExists()) {
        PathFragment rootRelativePath = rootedPath.getRelativePath();
        SkyKey packageKey = PackageValue.key(PackageIdentifier.create(recursivePkgKey.getRepository(), rootRelativePath));
        Map<SkyKey, ValueOrException<NoSuchPackageException>> dependentSkyValues = env.getValuesOrThrow(Iterables.concat(childDeps, ImmutableList.of(packageKey)), NoSuchPackageException.class);
        if (env.valuesMissing()) {
            return null;
        }
        Package pkg = null;
        try {
            PackageValue pkgValue = (PackageValue) dependentSkyValues.get(packageKey).get();
            if (pkgValue == null) {
                return null;
            }
            pkg = pkgValue.getPackage();
            if (pkg.containsErrors()) {
                env.getListener().handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
            }
        } catch (NoSuchPackageException e) {
            // The package had errors, but don't fail-fast as there might be subpackages below the
            // current directory.
            env.getListener().handle(Event.error(e.getMessage()));
            visitor.visitPackageError(e, env);
            if (env.valuesMissing()) {
                return null;
            }
        }
        if (pkg != null) {
            visitor.visitPackageValue(pkg, env);
            if (env.valuesMissing()) {
                return null;
            }
        }
        ImmutableMap.Builder<SkyKey, SkyValue> subdirectoryBuilder = ImmutableMap.builder();
        for (Map.Entry<SkyKey, ValueOrException<NoSuchPackageException>> entry : Maps.filterKeys(dependentSkyValues, Predicates.not(Predicates.equalTo(packageKey))).entrySet()) {
            try {
                subdirectoryBuilder.put(entry.getKey(), entry.getValue().get());
            } catch (NoSuchPackageException e) {
            // ignored.
            }
        }
        subdirectorySkyValues = subdirectoryBuilder.build();
    } else {
        subdirectorySkyValues = env.getValues(childDeps);
    }
    if (env.valuesMissing()) {
        return null;
    }
    return aggregateWithSubdirectorySkyValues(visitor, subdirectorySkyValues);
}