Examples with SkyValue com.google.devtools.build.skyframe.SkyValue used on opensource projects

Example 46 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class PackageFunction method fetchImportsFromBuildFile.

/**
   * Fetch the skylark loads for this BUILD file. If any of them haven't been computed yet,
   * returns null.
   */
@Nullable
static SkylarkImportResult fetchImportsFromBuildFile(Path buildFilePath, PackageIdentifier packageId, BuildFileAST buildFileAST, Environment env, SkylarkImportLookupFunction skylarkImportLookupFunctionForInlining) throws NoSuchPackageException, InterruptedException {
    Preconditions.checkArgument(!packageId.getRepository().isDefault());
    ImmutableList<SkylarkImport> imports = buildFileAST.getImports();
    Map<String, Extension> importMap = Maps.newHashMapWithExpectedSize(imports.size());
    ImmutableList.Builder<SkylarkFileDependency> fileDependencies = ImmutableList.builder();
    ImmutableMap<String, Label> importPathMap;
    // Find the labels corresponding to the load statements.
    Label labelForCurrBuildFile;
    try {
        labelForCurrBuildFile = Label.create(packageId, "BUILD");
    } catch (LabelSyntaxException e) {
        // Shouldn't happen; the Label is well-formed by construction.
        throw new IllegalStateException(e);
    }
    try {
        importPathMap = SkylarkImportLookupFunction.findLabelsForLoadStatements(imports, labelForCurrBuildFile, env);
        if (importPathMap == null) {
            return null;
        }
    } catch (SkylarkImportFailedException e) {
        throw new BuildFileContainsErrorsException(packageId, e.getMessage());
    }
    // Look up and load the imports.
    ImmutableCollection<Label> importLabels = importPathMap.values();
    List<SkyKey> importLookupKeys = Lists.newArrayListWithExpectedSize(importLabels.size());
    boolean inWorkspace = buildFilePath.getBaseName().endsWith("WORKSPACE");
    for (Label importLabel : importLabels) {
        importLookupKeys.add(SkylarkImportLookupValue.key(importLabel, inWorkspace));
    }
    Map<SkyKey, SkyValue> skylarkImportMap = Maps.newHashMapWithExpectedSize(importPathMap.size());
    boolean valuesMissing = false;
    try {
        if (skylarkImportLookupFunctionForInlining == null) {
            // Not inlining
            Map<SkyKey, ValueOrException2<SkylarkImportFailedException, InconsistentFilesystemException>> skylarkLookupResults = env.getValuesOrThrow(importLookupKeys, SkylarkImportFailedException.class, InconsistentFilesystemException.class);
            valuesMissing = env.valuesMissing();
            for (Map.Entry<SkyKey, ValueOrException2<SkylarkImportFailedException, InconsistentFilesystemException>> entry : skylarkLookupResults.entrySet()) {
                // Fetching the value will raise any deferred exceptions
                skylarkImportMap.put(entry.getKey(), entry.getValue().get());
            }
        } else {
            // Inlining calls to SkylarkImportLookupFunction
            LinkedHashMap<Label, SkylarkImportLookupValue> alreadyVisitedImports = Maps.newLinkedHashMapWithExpectedSize(importLookupKeys.size());
            for (SkyKey importLookupKey : importLookupKeys) {
                SkyValue skyValue = skylarkImportLookupFunctionForInlining.computeWithInlineCalls(importLookupKey, env, alreadyVisitedImports);
                if (skyValue == null) {
                    Preconditions.checkState(env.valuesMissing(), "no skylark import value for %s", importLookupKey);
                    // We continue making inline calls even if some requested values are missing, to
                    // maximize the number of dependent (non-inlined) SkyFunctions that are requested, thus
                    // avoiding a quadratic number of restarts.
                    valuesMissing = true;
                } else {
                    skylarkImportMap.put(importLookupKey, skyValue);
                }
            }
        }
    } catch (SkylarkImportFailedException e) {
        throw new BuildFileContainsErrorsException(packageId, e.getMessage());
    } catch (InconsistentFilesystemException e) {
        throw new NoSuchPackageException(packageId, e.getMessage(), e);
    }
    if (valuesMissing) {
        // Some imports are unavailable.
        return null;
    }
    // Process the loaded imports.
    for (Entry<String, Label> importEntry : importPathMap.entrySet()) {
        String importString = importEntry.getKey();
        Label importLabel = importEntry.getValue();
        SkyKey keyForLabel = SkylarkImportLookupValue.key(importLabel, inWorkspace);
        SkylarkImportLookupValue importLookupValue = (SkylarkImportLookupValue) skylarkImportMap.get(keyForLabel);
        importMap.put(importString, importLookupValue.getEnvironmentExtension());
        fileDependencies.add(importLookupValue.getDependency());
    }
    return new SkylarkImportResult(importMap, transitiveClosureOfLabels(fileDependencies.build()));
}

Example 47 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class ActionExecutionFunction method checkCacheAndExecuteIfNeeded.

private ActionExecutionValue checkCacheAndExecuteIfNeeded(Action action, ContinuationState state, Environment env, Map<String, String> clientEnv) throws ActionExecutionException, InterruptedException {
    // other action's value, provided here, since it is populated with metadata for the outputs.
    if (!state.hasArtifactData()) {
        return skyframeActionExecutor.executeAction(action, null, -1, null);
    }
    // This may be recreated if we discover inputs.
    ActionMetadataHandler metadataHandler = new ActionMetadataHandler(state.inputArtifactData, action.getOutputs(), tsgm.get());
    long actionStartTime = BlazeClock.nanoTime();
    // We only need to check the action cache if we haven't done it on a previous run.
    if (!state.hasCheckedActionCache()) {
        state.token = skyframeActionExecutor.checkActionCache(action, metadataHandler, actionStartTime, state.allInputs.actionCacheInputs, clientEnv);
    }
    if (state.token == null) {
        // We got a hit from the action cache -- no need to execute.
        return new ActionExecutionValue(metadataHandler.getOutputArtifactData(), metadataHandler.getOutputTreeArtifactData(), metadataHandler.getAdditionalOutputData());
    }
    // This may be recreated if we discover inputs.
    PerActionFileCache perActionFileCache = new PerActionFileCache(state.inputArtifactData);
    ActionExecutionContext actionExecutionContext = null;
    try {
        if (action.discoversInputs()) {
            if (state.discoveredInputs == null) {
                try {
                    state.discoveredInputs = skyframeActionExecutor.discoverInputs(action, perActionFileCache, metadataHandler, env);
                    Preconditions.checkState(state.discoveredInputs != null, "discoverInputs() returned null on action %s", action);
                } catch (MissingDepException e) {
                    Preconditions.checkState(env.valuesMissing(), action);
                    return null;
                }
            }
            addDiscoveredInputs(state.inputArtifactData, state.expandedArtifacts, state.discoveredInputs, env);
            if (env.valuesMissing()) {
                return null;
            }
            perActionFileCache = new PerActionFileCache(state.inputArtifactData);
            metadataHandler = new ActionMetadataHandler(state.inputArtifactData, action.getOutputs(), tsgm.get());
            // available.
            if (state.discoveredInputsStage2 == null) {
                state.discoveredInputsStage2 = action.discoverInputsStage2(env);
            }
            if (state.discoveredInputsStage2 != null) {
                addDiscoveredInputs(state.inputArtifactData, state.expandedArtifacts, state.discoveredInputsStage2, env);
                if (env.valuesMissing()) {
                    return null;
                }
                perActionFileCache = new PerActionFileCache(state.inputArtifactData);
                metadataHandler = new ActionMetadataHandler(state.inputArtifactData, action.getOutputs(), tsgm.get());
            }
        }
        actionExecutionContext = skyframeActionExecutor.getContext(perActionFileCache, metadataHandler, Collections.unmodifiableMap(state.expandedArtifacts));
        if (!state.hasExecutedAction()) {
            state.value = skyframeActionExecutor.executeAction(action, metadataHandler, actionStartTime, actionExecutionContext);
        }
    } finally {
        if (actionExecutionContext != null) {
            try {
                actionExecutionContext.getFileOutErr().close();
            } catch (IOException e) {
            // Nothing we can do here.
            }
        }
    }
    if (action.discoversInputs()) {
        Iterable<Artifact> newInputs = filterKnownInputs(action.getInputs(), state.inputArtifactData.keySet());
        Map<SkyKey, SkyValue> metadataFoundDuringActionExecution = env.getValues(toKeys(newInputs, action.getMandatoryInputs()));
        state.discoveredInputs = newInputs;
        if (env.valuesMissing()) {
            return null;
        }
        if (!Iterables.isEmpty(newInputs)) {
            // We are in the interesting case of an action that discovered its inputs during
            // execution, and found some new ones, but the new ones were already present in the graph.
            // We must therefore cache the metadata for those new ones.
            Map<Artifact, FileArtifactValue> inputArtifactData = new HashMap<>();
            inputArtifactData.putAll(state.inputArtifactData);
            for (Map.Entry<SkyKey, SkyValue> entry : metadataFoundDuringActionExecution.entrySet()) {
                inputArtifactData.put(ArtifactSkyKey.artifact(entry.getKey()), (FileArtifactValue) entry.getValue());
            }
            state.inputArtifactData = inputArtifactData;
            metadataHandler = new ActionMetadataHandler(state.inputArtifactData, action.getOutputs(), tsgm.get());
        }
    }
    Preconditions.checkState(!env.valuesMissing(), action);
    skyframeActionExecutor.afterExecution(action, metadataHandler, state.token, clientEnv);
    return state.value;
}

Example 48 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class ActionExecutionFunction method compute.

@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws ActionExecutionFunctionException, InterruptedException {
    Preconditions.checkArgument(skyKey.argument() instanceof Action);
    Action action = (Action) skyKey.argument();
    // BUILD_ID, forcing invalidation of upward transitive closure on each build.
    if ((action.isVolatile() && !(action instanceof SkyframeAwareAction)) || action instanceof NotifyOnActionCacheHit) {
        // Volatile build actions may need to execute even if none of their known inputs have changed.
        // Depending on the buildID ensure that these actions have a chance to execute.
        PrecomputedValue.BUILD_ID.get(env);
    }
    // Look up the parts of the environment that influence the action.
    Map<SkyKey, SkyValue> clientEnvLookup = env.getValues(Iterables.transform(action.getClientEnvironmentVariables(), VAR_TO_SKYKEY));
    if (env.valuesMissing()) {
        return null;
    }
    Map<String, String> clientEnv = new HashMap<>();
    for (Entry<SkyKey, SkyValue> entry : clientEnvLookup.entrySet()) {
        ClientEnvironmentValue envValue = (ClientEnvironmentValue) entry.getValue();
        if (envValue.getValue() != null) {
            clientEnv.put((String) entry.getKey().argument(), envValue.getValue());
        }
    }
    // For restarts of this ActionExecutionFunction we use a ContinuationState variable, below, to
    // avoid redoing work. However, if two actions are shared and the first one executes, when the
    // second one goes to execute, we should detect that and short-circuit, even without taking
    // ContinuationState into account.
    boolean sharedActionAlreadyRan = skyframeActionExecutor.probeActionExecution(action);
    ContinuationState state;
    if (action.discoversInputs()) {
        state = getState(action);
    } else {
        // Because this is a new state, all conditionals below about whether state has already done
        // something will return false, and so we will execute all necessary steps.
        state = new ContinuationState();
    }
    if (!state.hasCollectedInputs()) {
        state.allInputs = collectInputs(action, env);
        if (state.allInputs == null) {
            // Missing deps.
            return null;
        }
    } else if (state.allInputs.keysRequested != null) {
        // Preserve the invariant that we ask for the same deps each build.
        env.getValues(state.allInputs.keysRequested);
        Preconditions.checkState(!env.valuesMissing(), "%s %s", action, state);
    }
    Pair<Map<Artifact, FileArtifactValue>, Map<Artifact, Collection<Artifact>>> checkedInputs = null;
    try {
        // Declare deps on known inputs to action. We do this unconditionally to maintain our
        // invariant of asking for the same deps each build.
        Map<SkyKey, ValueOrException2<MissingInputFileException, ActionExecutionException>> inputDeps = env.getValuesOrThrow(toKeys(state.allInputs.getAllInputs(), action.discoversInputs() ? action.getMandatoryInputs() : null), MissingInputFileException.class, ActionExecutionException.class);
        if (!sharedActionAlreadyRan && !state.hasArtifactData()) {
            // Do we actually need to find our metadata?
            checkedInputs = checkInputs(env, action, inputDeps);
        }
    } catch (ActionExecutionException e) {
        // Remove action from state map in case it's there (won't be unless it discovers inputs).
        stateMap.remove(action);
        throw new ActionExecutionFunctionException(e);
    }
    if (env.valuesMissing()) {
        // of the action; see establishSkyframeDependencies why.
        return null;
    }
    try {
        establishSkyframeDependencies(env, action);
    } catch (ActionExecutionException e) {
        throw new ActionExecutionFunctionException(e);
    }
    if (env.valuesMissing()) {
        return null;
    }
    if (checkedInputs != null) {
        Preconditions.checkState(!state.hasArtifactData(), "%s %s", state, action);
        state.inputArtifactData = checkedInputs.first;
        state.expandedArtifacts = checkedInputs.second;
    }
    ActionExecutionValue result;
    try {
        result = checkCacheAndExecuteIfNeeded(action, state, env, clientEnv);
    } catch (ActionExecutionException e) {
        // Remove action from state map in case it's there (won't be unless it discovers inputs).
        stateMap.remove(action);
        // action. Label can be null in the case of, e.g., the SystemActionOwner (for build-info.txt).
        throw new ActionExecutionFunctionException(new AlreadyReportedActionExecutionException(e));
    }
    if (env.valuesMissing()) {
        Preconditions.checkState(stateMap.containsKey(action), action);
        return null;
    }
    // Remove action from state map in case it's there (won't be unless it discovers inputs).
    stateMap.remove(action);
    return result;
}

Example 49 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class RecursiveDirectoryTraversalFunction method visitDirectory.

/**
   * Looks in the directory specified by {@code recursivePkgKey} for a package, does some work as
   * specified by {@link Visitor} if such a package exists, then recursively does work in each
   * non-excluded subdirectory as specified by {@link #getSkyKeyForSubdirectory}, and finally
   * aggregates the {@link Visitor} value along with values from each subdirectory as specified by
   * {@link #aggregateWithSubdirectorySkyValues}, and returns that aggregation.
   *
   * <p>Returns null if {@code env.valuesMissing()} is true, checked after each call to one of
   * {@link RecursiveDirectoryTraversalFunction}'s abstract methods that were given {@code env}.
   * (And after each of {@code visitDirectory}'s own uses of {@code env}, of course.)
   */
TReturn visitDirectory(RecursivePkgKey recursivePkgKey, Environment env) throws InterruptedException {
    RootedPath rootedPath = recursivePkgKey.getRootedPath();
    ProcessPackageDirectoryResult packageExistenceAndSubdirDeps = processPackageDirectory.getPackageExistenceAndSubdirDeps(rootedPath, recursivePkgKey.getRepository(), env, recursivePkgKey.getExcludedPaths());
    if (env.valuesMissing()) {
        return null;
    }
    Iterable<SkyKey> childDeps = packageExistenceAndSubdirDeps.getChildDeps();
    TVisitor visitor = getInitialVisitor();
    Map<SkyKey, SkyValue> subdirectorySkyValues;
    if (packageExistenceAndSubdirDeps.packageExists()) {
        PathFragment rootRelativePath = rootedPath.getRelativePath();
        SkyKey packageKey = PackageValue.key(PackageIdentifier.create(recursivePkgKey.getRepository(), rootRelativePath));
        Map<SkyKey, ValueOrException<NoSuchPackageException>> dependentSkyValues = env.getValuesOrThrow(Iterables.concat(childDeps, ImmutableList.of(packageKey)), NoSuchPackageException.class);
        if (env.valuesMissing()) {
            return null;
        }
        Package pkg = null;
        try {
            PackageValue pkgValue = (PackageValue) dependentSkyValues.get(packageKey).get();
            if (pkgValue == null) {
                return null;
            }
            pkg = pkgValue.getPackage();
            if (pkg.containsErrors()) {
                env.getListener().handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
            }
        } catch (NoSuchPackageException e) {
            // The package had errors, but don't fail-fast as there might be subpackages below the
            // current directory.
            env.getListener().handle(Event.error(e.getMessage()));
            visitor.visitPackageError(e, env);
            if (env.valuesMissing()) {
                return null;
            }
        }
        if (pkg != null) {
            visitor.visitPackageValue(pkg, env);
            if (env.valuesMissing()) {
                return null;
            }
        }
        ImmutableMap.Builder<SkyKey, SkyValue> subdirectoryBuilder = ImmutableMap.builder();
        for (Map.Entry<SkyKey, ValueOrException<NoSuchPackageException>> entry : Maps.filterKeys(dependentSkyValues, Predicates.not(Predicates.equalTo(packageKey))).entrySet()) {
            try {
                subdirectoryBuilder.put(entry.getKey(), entry.getValue().get());
            } catch (NoSuchPackageException e) {
            // ignored.
            }
        }
        subdirectorySkyValues = subdirectoryBuilder.build();
    } else {
        subdirectorySkyValues = env.getValues(childDeps);
    }
    if (env.valuesMissing()) {
        return null;
    }
    return aggregateWithSubdirectorySkyValues(visitor, subdirectorySkyValues);
}

Example 50 with SkyValue

use of com.google.devtools.build.skyframe.SkyValue in project bazel by bazelbuild.

the class SequencedSkyframeExecutor method handleClientEnvironmentChanges.

/** Invalidates entries in the client environment that have changed since last sync. */
private void handleClientEnvironmentChanges() {
    Map<SkyKey, SkyValue> values = memoizingEvaluator.getValues();
    ImmutableMap.Builder<SkyKey, SkyValue> newValuesBuilder = ImmutableMap.builder();
    HashSet<String> envToCheck = new HashSet<>();
    if (previousClientEnvironment != null) {
        envToCheck.addAll(previousClientEnvironment);
    }
    envToCheck.addAll(clientEnv.get().keySet());
    previousClientEnvironment = clientEnv.get().keySet();
    for (String env : envToCheck) {
        SkyKey key = SkyKey.create(SkyFunctions.CLIENT_ENVIRONMENT_VARIABLE, env);
        if (values.containsKey(key)) {
            String value = ((ClientEnvironmentValue) values.get(key)).getValue();
            String newValue = clientEnv.get().get(env);
            if (!Objects.equal(newValue, value)) {
                newValuesBuilder.put(key, new ClientEnvironmentValue(newValue));
            }
        }
    }
    recordingDiffer.inject(newValuesBuilder.build());
}