Examples with AutoProfiler com.google.devtools.build.lib.profiler.AutoProfiler used on opensource projects

Example 6 with AutoProfiler

use of com.google.devtools.build.lib.profiler.AutoProfiler in project bazel by bazelbuild.

the class SkyframeExecutor method findArtifactConflicts.

/**
   * Checks the actions in Skyframe for conflicts between their output artifacts. Delegates to
   * {@link SkyframeActionExecutor#findAndStoreArtifactConflicts} to do the work, since any
   * conflicts found will only be reported during execution.
   */
ImmutableMap<ActionAnalysisMetadata, SkyframeActionExecutor.ConflictException> findArtifactConflicts() throws InterruptedException {
    if (skyframeBuildView.isSomeConfiguredTargetEvaluated() || skyframeBuildView.isSomeConfiguredTargetInvalidated()) {
        // some way -- either we analyzed a new target or we invalidated an old one.
        try (AutoProfiler p = AutoProfiler.logged("discovering artifact conflicts", LOG)) {
            skyframeActionExecutor.findAndStoreArtifactConflicts(getActionLookupValues());
            skyframeBuildView.resetEvaluatedConfiguredTargetFlag();
        // The invalidated configured targets flag will be reset later in the evaluate() call.
        }
    }
    return skyframeActionExecutor.badActions();
}

Example 7 with AutoProfiler

use of com.google.devtools.build.lib.profiler.AutoProfiler in project bazel by bazelbuild.

the class ResourceManager method acquireResources.

/**
   * Acquires requested resource set. Will block if resource is not available.
   * NB! This method must be thread-safe!
   */
public ResourceHandle acquireResources(ActionExecutionMetadata owner, ResourceSet resources) throws InterruptedException {
    Preconditions.checkNotNull(resources, "acquireResources called with resources == NULL during %s", owner);
    Preconditions.checkState(!threadHasResources(), "acquireResources with existing resource lock during %s", owner);
    AutoProfiler p = profiled(owner, ProfilerTask.ACTION_LOCK);
    CountDownLatch latch = null;
    try {
        latch = acquire(resources);
        if (latch != null) {
            latch.await();
        }
    } catch (InterruptedException e) {
        // Synchronize on this to avoid any racing with #processWaitingThreads
        synchronized (this) {
            if (latch.getCount() == 0) {
                // Resources already acquired by other side. Release them, but not inside this
                // synchronized block to avoid deadlock.
                release(resources);
            } else {
                // Inform other side that resources shouldn't be acquired.
                latch.countDown();
            }
        }
        throw e;
    }
    threadLocked.set(true);
    // Profile acquisition only if it waited for resource to become available.
    if (latch != null) {
        p.complete();
    }
    return new ResourceHandle(this, owner, resources);
}

Example 8 with AutoProfiler

use of com.google.devtools.build.lib.profiler.AutoProfiler in project bazel by bazelbuild.

the class ExecutionTool method executeBuild.

/**
   * Performs the execution phase (phase 3) of the build, in which the Builder
   * is applied to the action graph to bring the targets up to date. (This
   * function will return prior to execution-proper if --nobuild was specified.)
   * @param buildId UUID of the build id
   * @param analysisResult the analysis phase output
   * @param buildResult the mutable build result
   * @param packageRoots package roots collected from loading phase and BuildConfigurationCollection
   * creation
   */
void executeBuild(UUID buildId, AnalysisResult analysisResult, BuildResult buildResult, BuildConfigurationCollection configurations, ImmutableMap<PackageIdentifier, Path> packageRoots, TopLevelArtifactContext topLevelArtifactContext) throws BuildFailedException, InterruptedException, TestExecException, AbruptExitException {
    Stopwatch timer = Stopwatch.createStarted();
    prepare(packageRoots, analysisResult.getWorkspaceName());
    ActionGraph actionGraph = analysisResult.getActionGraph();
    // Get top-level artifacts.
    ImmutableSet<Artifact> additionalArtifacts = analysisResult.getAdditionalArtifactsToBuild();
    OutputService outputService = env.getOutputService();
    ModifiedFileSet modifiedOutputFiles = ModifiedFileSet.EVERYTHING_MODIFIED;
    if (outputService != null) {
        modifiedOutputFiles = outputService.startBuild(buildId, request.getBuildOptions().finalizeActions);
    } else {
        // TODO(bazel-team): this could be just another OutputService
        startLocalOutputBuild(analysisResult.getWorkspaceName());
    }
    List<BuildConfiguration> targetConfigurations = configurations.getTargetConfigurations();
    BuildConfiguration targetConfiguration = targetConfigurations.size() == 1 ? targetConfigurations.get(0) : null;
    if (targetConfigurations.size() == 1) {
        String productName = runtime.getProductName();
        String dirName = env.getWorkspaceName();
        String workspaceName = analysisResult.getWorkspaceName();
        OutputDirectoryLinksUtils.createOutputDirectoryLinks(dirName, env.getWorkspace(), env.getDirectories().getExecRoot(workspaceName), env.getDirectories().getOutputPath(workspaceName), getReporter(), targetConfiguration, request.getBuildOptions().getSymlinkPrefix(productName), productName);
    }
    ActionCache actionCache = getActionCache();
    SkyframeExecutor skyframeExecutor = env.getSkyframeExecutor();
    Builder builder = createBuilder(request, actionCache, skyframeExecutor, modifiedOutputFiles);
    //
    // Execution proper.  All statements below are logically nested in
    // begin/end pairs.  No early returns or exceptions please!
    //
    Collection<ConfiguredTarget> configuredTargets = buildResult.getActualTargets();
    env.getEventBus().post(new ExecutionStartingEvent(configuredTargets));
    getReporter().handle(Event.progress("Building..."));
    // Conditionally record dependency-checker log:
    ExplanationHandler explanationHandler = installExplanationHandler(request.getBuildOptions().explanationPath, request.getOptionsDescription());
    Set<ConfiguredTarget> builtTargets = new HashSet<>();
    Collection<AspectValue> aspects = analysisResult.getAspects();
    Iterable<Artifact> allArtifactsForProviders = Iterables.concat(additionalArtifacts, TopLevelArtifactHelper.getAllArtifactsToBuild(analysisResult.getTargetsToBuild(), analysisResult.getTopLevelContext()).getAllArtifacts(), TopLevelArtifactHelper.getAllArtifactsToBuildFromAspects(aspects, analysisResult.getTopLevelContext()).getAllArtifacts(), //TODO(dslomov): Artifacts to test from aspects?
    TopLevelArtifactHelper.getAllArtifactsToTest(analysisResult.getTargetsToTest()));
    if (request.isRunningInEmacs()) {
        // The syntax of this message is tightly constrained by lisp/progmodes/compile.el in emacs
        request.getOutErr().printErrLn("blaze: Entering directory `" + getExecRoot() + "/'");
    }
    boolean buildCompleted = false;
    try {
        for (ActionContextProvider actionContextProvider : actionContextProviders) {
            actionContextProvider.executionPhaseStarting(actionGraph, allArtifactsForProviders);
        }
        executor.executionPhaseStarting();
        skyframeExecutor.drainChangedFiles();
        if (request.getViewOptions().discardAnalysisCache) {
            // Free memory by removing cache entries that aren't going to be needed. Note that in
            // skyframe full, this destroys the action graph as well, so we can only do it after the
            // action graph is no longer needed.
            env.getSkyframeBuildView().clearAnalysisCache(analysisResult.getTargetsToBuild());
        }
        configureResourceManager(request);
        Profiler.instance().markPhase(ProfilePhase.EXECUTE);
        builder.buildArtifacts(env.getReporter(), additionalArtifacts, analysisResult.getParallelTests(), analysisResult.getExclusiveTests(), analysisResult.getTargetsToBuild(), analysisResult.getAspects(), executor, builtTargets, request.getBuildOptions().explanationPath != null, env.getBlazeWorkspace().getLastExecutionTimeRange(), topLevelArtifactContext);
        buildCompleted = true;
    } catch (BuildFailedException | TestExecException e) {
        buildCompleted = true;
        throw e;
    } finally {
        env.recordLastExecutionTime();
        if (request.isRunningInEmacs()) {
            request.getOutErr().printErrLn("blaze: Leaving directory `" + getExecRoot() + "/'");
        }
        if (buildCompleted) {
            getReporter().handle(Event.progress("Building complete."));
        }
        env.getEventBus().post(new ExecutionFinishedEvent(ImmutableMap.<String, Long>of(), 0L, skyframeExecutor.getOutputDirtyFilesAndClear(), skyframeExecutor.getModifiedFilesDuringPreviousBuildAndClear()));
        executor.executionPhaseEnding();
        for (ActionContextProvider actionContextProvider : actionContextProviders) {
            actionContextProvider.executionPhaseEnding();
        }
        Profiler.instance().markPhase(ProfilePhase.FINISH);
        if (buildCompleted) {
            saveCaches(actionCache);
        }
        try (AutoProfiler p = AutoProfiler.profiled("Show results", ProfilerTask.INFO)) {
            buildResult.setSuccessfulTargets(determineSuccessfulTargets(configuredTargets, builtTargets, timer));
            BuildResultPrinter buildResultPrinter = new BuildResultPrinter(env);
            buildResultPrinter.showBuildResult(request, buildResult, configuredTargets, analysisResult.getAspects());
        }
        try (AutoProfiler p = AutoProfiler.profiled("Show artifacts", ProfilerTask.INFO)) {
            if (request.getBuildOptions().showArtifacts) {
                BuildResultPrinter buildResultPrinter = new BuildResultPrinter(env);
                buildResultPrinter.showArtifacts(request, configuredTargets, analysisResult.getAspects());
            }
        }
        if (explanationHandler != null) {
            uninstallExplanationHandler(explanationHandler);
        }
        // code has already run.
        if (env.getOutputService() != null) {
            boolean isBuildSuccessful = buildResult.getSuccessfulTargets().size() == configuredTargets.size();
            env.getOutputService().finalizeBuild(isBuildSuccessful);
        }
    }
}

Example 9 with AutoProfiler

use of com.google.devtools.build.lib.profiler.AutoProfiler in project bazel by bazelbuild.

the class FilesystemValueChecker method getDirtyValues.

private BatchDirtyResult getDirtyValues(ValueFetcher fetcher, Iterable<SkyKey> keys, final SkyValueDirtinessChecker checker, final boolean checkMissingValues) throws InterruptedException {
    ExecutorService executor = Executors.newFixedThreadPool(DIRTINESS_CHECK_THREADS, new ThreadFactoryBuilder().setNameFormat("FileSystem Value Invalidator %d").build());
    final BatchDirtyResult batchResult = new BatchDirtyResult();
    ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper("FilesystemValueChecker#getDirtyValues");
    final AtomicInteger numKeysScanned = new AtomicInteger(0);
    final AtomicInteger numKeysChecked = new AtomicInteger(0);
    ElapsedTimeReceiver elapsedTimeReceiver = new ElapsedTimeReceiver() {

        @Override
        public void accept(long elapsedTimeNanos) {
            if (elapsedTimeNanos > 0) {
                LOG.info(String.format("Spent %d ms checking %d filesystem nodes (%d scanned)", TimeUnit.MILLISECONDS.convert(elapsedTimeNanos, TimeUnit.NANOSECONDS), numKeysChecked.get(), numKeysScanned.get()));
            }
        }
    };
    try (AutoProfiler prof = AutoProfiler.create(elapsedTimeReceiver)) {
        for (final SkyKey key : keys) {
            numKeysScanned.incrementAndGet();
            if (!checker.applies(key)) {
                continue;
            }
            final SkyValue value = fetcher.get(key);
            if (!checkMissingValues && value == null) {
                continue;
            }
            executor.execute(wrapper.wrap(new Runnable() {

                @Override
                public void run() {
                    numKeysChecked.incrementAndGet();
                    DirtyResult result = checker.check(key, value, tsgm);
                    if (result.isDirty()) {
                        batchResult.add(key, value, result.getNewValue());
                    }
                }
            }));
        }
        boolean interrupted = ExecutorUtil.interruptibleShutdown(executor);
        Throwables.propagateIfPossible(wrapper.getFirstThrownError());
        if (interrupted) {
            throw new InterruptedException();
        }
    }
    return batchResult;
}