Examples with ActionGraph com.google.devtools.build.lib.actions.ActionGraph used on opensource projects

Example 1 with ActionGraph

use of com.google.devtools.build.lib.actions.ActionGraph in project bazel by bazelbuild.

the class BuildView method createResult.

private AnalysisResult createResult(ExtendedEventHandler eventHandler, LoadingResult loadingResult, TopLevelArtifactContext topLevelOptions, BuildView.Options viewOptions, SkyframeAnalysisResult skyframeAnalysisResult) throws InterruptedException {
    Collection<Target> testsToRun = loadingResult.getTestsToRun();
    Collection<ConfiguredTarget> configuredTargets = skyframeAnalysisResult.getConfiguredTargets();
    Collection<AspectValue> aspects = skyframeAnalysisResult.getAspects();
    Collection<ConfiguredTarget> allTargetsToTest = null;
    if (testsToRun != null) {
        // Determine the subset of configured targets that are meant to be run as tests.
        // Do not remove <ConfiguredTarget>: workaround for Java 7 type inference.
        allTargetsToTest = Lists.<ConfiguredTarget>newArrayList(filterTestsByTargets(configuredTargets, Sets.newHashSet(testsToRun)));
    }
    Set<Artifact> artifactsToBuild = new HashSet<>();
    Set<ConfiguredTarget> parallelTests = new HashSet<>();
    Set<ConfiguredTarget> exclusiveTests = new HashSet<>();
    // build-info and build-changelist.
    Collection<Artifact> buildInfoArtifacts = skyframeExecutor.getWorkspaceStatusArtifacts(eventHandler);
    Preconditions.checkState(buildInfoArtifacts.size() == 2, buildInfoArtifacts);
    artifactsToBuild.addAll(buildInfoArtifacts);
    // Extra actions
    addExtraActionsIfRequested(viewOptions, configuredTargets, aspects, artifactsToBuild);
    // Coverage
    NestedSet<Artifact> baselineCoverageArtifacts = getBaselineCoverageArtifacts(configuredTargets);
    Iterables.addAll(artifactsToBuild, baselineCoverageArtifacts);
    if (coverageReportActionFactory != null) {
        CoverageReportActionsWrapper actionsWrapper;
        actionsWrapper = coverageReportActionFactory.createCoverageReportActionsWrapper(eventHandler, directories, allTargetsToTest, baselineCoverageArtifacts, getArtifactFactory(), CoverageReportValue.ARTIFACT_OWNER);
        if (actionsWrapper != null) {
            ImmutableList<ActionAnalysisMetadata> actions = actionsWrapper.getActions();
            skyframeExecutor.injectCoverageReportData(actions);
            artifactsToBuild.addAll(actionsWrapper.getCoverageOutputs());
        }
    }
    // Tests. This must come last, so that the exclusive tests are scheduled after everything else.
    scheduleTestsIfRequested(parallelTests, exclusiveTests, topLevelOptions, allTargetsToTest);
    String error = createErrorMessage(loadingResult, skyframeAnalysisResult);
    final WalkableGraph graph = skyframeAnalysisResult.getWalkableGraph();
    final ActionGraph actionGraph = new ActionGraph() {

        @Nullable
        @Override
        public ActionAnalysisMetadata getGeneratingAction(Artifact artifact) {
            ArtifactOwner artifactOwner = artifact.getArtifactOwner();
            if (artifactOwner instanceof ActionLookupValue.ActionLookupKey) {
                SkyKey key = ActionLookupValue.key((ActionLookupValue.ActionLookupKey) artifactOwner);
                ActionLookupValue val;
                try {
                    val = (ActionLookupValue) graph.getValue(key);
                } catch (InterruptedException e) {
                    throw new IllegalStateException("Interruption not expected from this graph: " + key, e);
                }
                return val == null ? null : val.getGeneratingAction(artifact);
            }
            return null;
        }
    };
    return new AnalysisResult(configuredTargets, aspects, allTargetsToTest, error, actionGraph, artifactsToBuild, parallelTests, exclusiveTests, topLevelOptions, skyframeAnalysisResult.getPackageRoots(), loadingResult.getWorkspaceName());
}

Example 2 with ActionGraph

use of com.google.devtools.build.lib.actions.ActionGraph in project bazel by bazelbuild.

the class SkyframeActionExecutor method findAndStoreArtifactConflicts.

/**
   * Find conflicts between generated artifacts. There are two ways to have conflicts. First, if
   * two (unshareable) actions generate the same output artifact, this will result in an {@link
   * ActionConflictException}. Second, if one action generates an artifact whose path is a prefix of
   * another artifact's path, those two artifacts cannot exist simultaneously in the output tree.
   * This causes an {@link ArtifactPrefixConflictException}. The relevant exceptions are stored in
   * the executor in {@code badActionMap}, and will be thrown immediately when that action is
   * executed. Those exceptions persist, so that even if the action is not executed this build, the
   * first time it is executed, the correct exception will be thrown.
   *
   * <p>This method must be called if a new action was added to the graph this build, so
   * whenever a new configured target was analyzed this build. It is somewhat expensive (~1s
   * range for a medium build as of 2014), so it should only be called when necessary.
   *
   * <p>Conflicts found may not be requested this build, and so we may overzealously throw an error.
   * For instance, if actions A and B generate the same artifact foo, and the user first requests
   * A' depending on A, and then in a subsequent build B' depending on B, we will fail the second
   * build, even though it would have succeeded if it had been the only build. However, since
   * Skyframe does not know the transitive dependencies of the request, we err on the conservative
   * side.
   *
   * <p>If the user first runs one action on the first build, and on the second build adds a
   * conflicting action, only the second action's error may be reported (because the first action
   * will be cached), whereas if both actions were requested for the first time, both errors would
   * be reported. However, the first time an action is added to the build, we are guaranteed to find
   * any conflicts it has, since this method will compare it against all other actions. So there is
   * no sequence of builds that can evade the error.
   */
void findAndStoreArtifactConflicts(Iterable<ActionLookupValue> actionLookupValues) throws InterruptedException {
    ConcurrentMap<ActionAnalysisMetadata, ConflictException> temporaryBadActionMap = new ConcurrentHashMap<>();
    Pair<ActionGraph, SortedMap<PathFragment, Artifact>> result;
    result = constructActionGraphAndPathMap(actionLookupValues, temporaryBadActionMap);
    ActionGraph actionGraph = result.first;
    SortedMap<PathFragment, Artifact> artifactPathMap = result.second;
    Map<ActionAnalysisMetadata, ArtifactPrefixConflictException> actionsWithArtifactPrefixConflict = Actions.findArtifactPrefixConflicts(actionGraph, artifactPathMap);
    for (Map.Entry<ActionAnalysisMetadata, ArtifactPrefixConflictException> actionExceptionPair : actionsWithArtifactPrefixConflict.entrySet()) {
        temporaryBadActionMap.put(actionExceptionPair.getKey(), new ConflictException(actionExceptionPair.getValue()));
    }
    this.badActionMap = ImmutableMap.copyOf(temporaryBadActionMap);
}

Example 3 with ActionGraph

use of com.google.devtools.build.lib.actions.ActionGraph in project bazel by bazelbuild.

the class SkyframeActionExecutor method constructActionGraphAndPathMap.

/**
   * Simultaneously construct an action graph for all the actions in Skyframe and a map from
   * {@link PathFragment}s to their respective {@link Artifact}s. We do this in a threadpool to save
   * around 1.5 seconds on a mid-sized build versus a single-threaded operation.
   */
private static Pair<ActionGraph, SortedMap<PathFragment, Artifact>> constructActionGraphAndPathMap(Iterable<ActionLookupValue> values, ConcurrentMap<ActionAnalysisMetadata, ConflictException> badActionMap) throws InterruptedException {
    MutableActionGraph actionGraph = new MapBasedActionGraph();
    ConcurrentNavigableMap<PathFragment, Artifact> artifactPathMap = new ConcurrentSkipListMap<>();
    // Action graph construction is CPU-bound.
    int numJobs = Runtime.getRuntime().availableProcessors();
    // No great reason for expecting 5000 action lookup values, but not worth counting size of
    // values.
    Sharder<ActionLookupValue> actionShards = new Sharder<>(numJobs, 5000);
    for (ActionLookupValue value : values) {
        actionShards.add(value);
    }
    ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper("SkyframeActionExecutor#constructActionGraphAndPathMap");
    ExecutorService executor = Executors.newFixedThreadPool(numJobs, new ThreadFactoryBuilder().setNameFormat("ActionLookupValue Processor %d").build());
    for (List<ActionLookupValue> shard : actionShards) {
        executor.execute(wrapper.wrap(actionRegistration(shard, actionGraph, artifactPathMap, badActionMap)));
    }
    boolean interrupted = ExecutorUtil.interruptibleShutdown(executor);
    Throwables.propagateIfPossible(wrapper.getFirstThrownError());
    if (interrupted) {
        throw new InterruptedException();
    }
    return Pair.<ActionGraph, SortedMap<PathFragment, Artifact>>of(actionGraph, artifactPathMap);
}

Example 4 with ActionGraph

use of com.google.devtools.build.lib.actions.ActionGraph in project bazel by bazelbuild.

the class ExecutionTool method executeBuild.

/**
   * Performs the execution phase (phase 3) of the build, in which the Builder
   * is applied to the action graph to bring the targets up to date. (This
   * function will return prior to execution-proper if --nobuild was specified.)
   * @param buildId UUID of the build id
   * @param analysisResult the analysis phase output
   * @param buildResult the mutable build result
   * @param packageRoots package roots collected from loading phase and BuildConfigurationCollection
   * creation
   */
void executeBuild(UUID buildId, AnalysisResult analysisResult, BuildResult buildResult, BuildConfigurationCollection configurations, ImmutableMap<PackageIdentifier, Path> packageRoots, TopLevelArtifactContext topLevelArtifactContext) throws BuildFailedException, InterruptedException, TestExecException, AbruptExitException {
    Stopwatch timer = Stopwatch.createStarted();
    prepare(packageRoots, analysisResult.getWorkspaceName());
    ActionGraph actionGraph = analysisResult.getActionGraph();
    // Get top-level artifacts.
    ImmutableSet<Artifact> additionalArtifacts = analysisResult.getAdditionalArtifactsToBuild();
    OutputService outputService = env.getOutputService();
    ModifiedFileSet modifiedOutputFiles = ModifiedFileSet.EVERYTHING_MODIFIED;
    if (outputService != null) {
        modifiedOutputFiles = outputService.startBuild(buildId, request.getBuildOptions().finalizeActions);
    } else {
        // TODO(bazel-team): this could be just another OutputService
        startLocalOutputBuild(analysisResult.getWorkspaceName());
    }
    List<BuildConfiguration> targetConfigurations = configurations.getTargetConfigurations();
    BuildConfiguration targetConfiguration = targetConfigurations.size() == 1 ? targetConfigurations.get(0) : null;
    if (targetConfigurations.size() == 1) {
        String productName = runtime.getProductName();
        String dirName = env.getWorkspaceName();
        String workspaceName = analysisResult.getWorkspaceName();
        OutputDirectoryLinksUtils.createOutputDirectoryLinks(dirName, env.getWorkspace(), env.getDirectories().getExecRoot(workspaceName), env.getDirectories().getOutputPath(workspaceName), getReporter(), targetConfiguration, request.getBuildOptions().getSymlinkPrefix(productName), productName);
    }
    ActionCache actionCache = getActionCache();
    SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
    Builder builder = createBuilder(request, actionCache, skyframeExecutor, modifiedOutputFiles);
    //
    // Execution proper.  All statements below are logically nested in
    // begin/end pairs.  No early returns or exceptions please!
    //
    Collection<ConfiguredTarget> configuredTargets = buildResult.getActualTargets();
    env.getEventBus().post(new ExecutionStartingEvent(configuredTargets));
    getReporter().handle(Event.progress("Building..."));
    // Conditionally record dependency-checker log:
    ExplanationHandler explanationHandler = installExplanationHandler(request.getBuildOptions().explanationPath, request.getOptionsDescription());
    Set<ConfiguredTarget> builtTargets = new HashSet<>();
    Collection<AspectValue> aspects = analysisResult.getAspects();
    Iterable<Artifact> allArtifactsForProviders = Iterables.concat(additionalArtifacts, TopLevelArtifactHelper.getAllArtifactsToBuild(analysisResult.getTargetsToBuild(), analysisResult.getTopLevelContext()).getAllArtifacts(), TopLevelArtifactHelper.getAllArtifactsToBuildFromAspects(aspects, analysisResult.getTopLevelContext()).getAllArtifacts(), //TODO(dslomov): Artifacts to test from aspects?
    TopLevelArtifactHelper.getAllArtifactsToTest(analysisResult.getTargetsToTest()));
    if (request.isRunningInEmacs()) {
        // The syntax of this message is tightly constrained by lisp/progmodes/compile.el in emacs
        request.getOutErr().printErrLn("blaze: Entering directory `" + getExecRoot() + "/'");
    }
    boolean buildCompleted = false;
    try {
        for (ActionContextProvider actionContextProvider : actionContextProviders) {
            actionContextProvider.executionPhaseStarting(actionGraph, allArtifactsForProviders);
        }
        executor.executionPhaseStarting();
        skyframeExecutor.drainChangedFiles();
        if (request.getViewOptions().discardAnalysisCache) {
            // Free memory by removing cache entries that aren't going to be needed. Note that in
            // skyframe full, this destroys the action graph as well, so we can only do it after the
            // action graph is no longer needed.
            env.getSkyframeBuildView().clearAnalysisCache(analysisResult.getTargetsToBuild());
        }
        configureResourceManager(request);
        Profiler.instance().markPhase(ProfilePhase.EXECUTE);
        builder.buildArtifacts(env.getReporter(), additionalArtifacts, analysisResult.getParallelTests(), analysisResult.getExclusiveTests(), analysisResult.getTargetsToBuild(), analysisResult.getAspects(), executor, builtTargets, request.getBuildOptions().explanationPath != null, env.getBlazeWorkspace().getLastExecutionTimeRange(), topLevelArtifactContext);
        buildCompleted = true;
    } catch (BuildFailedException | TestExecException e) {
        buildCompleted = true;
        throw e;
    } finally {
        env.recordLastExecutionTime();
        if (request.isRunningInEmacs()) {
            request.getOutErr().printErrLn("blaze: Leaving directory `" + getExecRoot() + "/'");
        }
        if (buildCompleted) {
            getReporter().handle(Event.progress("Building complete."));
        }
        env.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap.<String, Long>of(), 0L, skyframeExecutor.getOutputDirtyFilesAndClear(), skyframeExecutor.getModifiedFilesDuringPreviousBuildAndClear()));
        executor.executionPhaseEnding();
        for (ActionContextProvider actionContextProvider : actionContextProviders) {
            actionContextProvider.executionPhaseEnding();
        }
        Profiler.instance().markPhase(ProfilePhase.FINISH);
        if (buildCompleted) {
            saveCaches(actionCache);
        }
        try (AutoProfiler p = AutoProfiler.profiled("Show results", ProfilerTask.INFO)) {
            buildResult.setSuccessfulTargets(determineSuccessfulTargets(configuredTargets, builtTargets, timer));
            BuildResultPrinter buildResultPrinter = new BuildResultPrinter(env);
            buildResultPrinter.showBuildResult(request, buildResult, configuredTargets, analysisResult.getAspects());
        }
        try (AutoProfiler p = AutoProfiler.profiled("Show artifacts", ProfilerTask.INFO)) {
            if (request.getBuildOptions().showArtifacts) {
                BuildResultPrinter buildResultPrinter = new BuildResultPrinter(env);
                buildResultPrinter.showArtifacts(request, configuredTargets, analysisResult.getAspects());
            }
        }
        if (explanationHandler != null) {
            uninstallExplanationHandler(explanationHandler);
        }
        // code has already run.
        if (env.getOutputService() != null) {
            boolean isBuildSuccessful = buildResult.getSuccessfulTargets().size() == configuredTargets.size();
            env.getOutputService().finalizeBuild(isBuildSuccessful);
        }
    }
}