Examples with ThrowableRecordingRunnableWrapper com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper used on opensource projects

Example 1 with ThrowableRecordingRunnableWrapper

use of com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper in project bazel by bazelbuild.

the class FilesystemValueChecker method getDirtyActionValues.

/**
   * Return a collection of action values which have output files that are not in-sync with
   * the on-disk file value (were modified externally).
   */
Collection<SkyKey> getDirtyActionValues(Map<SkyKey, SkyValue> valuesMap, @Nullable final BatchStat batchStatter, ModifiedFileSet modifiedOutputFiles) throws InterruptedException {
    if (modifiedOutputFiles == ModifiedFileSet.NOTHING_MODIFIED) {
        LOG.info("Not checking for dirty actions since nothing was modified");
        return ImmutableList.of();
    }
    LOG.info("Accumulating dirty actions");
    final int numOutputJobs = Runtime.getRuntime().availableProcessors() * 4;
    final Set<SkyKey> actionSkyKeys = new HashSet<>();
    for (SkyKey key : valuesMap.keySet()) {
        if (ACTION_FILTER.apply(key)) {
            actionSkyKeys.add(key);
        }
    }
    final Sharder<Pair<SkyKey, ActionExecutionValue>> outputShards = new Sharder<>(numOutputJobs, actionSkyKeys.size());
    for (SkyKey key : actionSkyKeys) {
        outputShards.add(Pair.of(key, (ActionExecutionValue) valuesMap.get(key)));
    }
    LOG.info("Sharded action values for batching");
    ExecutorService executor = Executors.newFixedThreadPool(numOutputJobs, new ThreadFactoryBuilder().setNameFormat("FileSystem Output File Invalidator %d").build());
    Collection<SkyKey> dirtyKeys = Sets.newConcurrentHashSet();
    ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper("FileSystemValueChecker#getDirtyActionValues");
    modifiedOutputFilesCounter.set(0);
    modifiedOutputFilesIntraBuildCounter.set(0);
    final ImmutableSet<PathFragment> knownModifiedOutputFiles = modifiedOutputFiles == ModifiedFileSet.EVERYTHING_MODIFIED ? null : modifiedOutputFiles.modifiedSourceFiles();
    // Initialized lazily through a supplier because it is only used to check modified
    // TreeArtifacts, which are not frequently used in builds.
    Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles = Suppliers.memoize(new Supplier<NavigableSet<PathFragment>>() {

        @Override
        public NavigableSet<PathFragment> get() {
            if (knownModifiedOutputFiles == null) {
                return null;
            } else {
                return ImmutableSortedSet.copyOf(knownModifiedOutputFiles);
            }
        }
    });
    for (List<Pair<SkyKey, ActionExecutionValue>> shard : outputShards) {
        Runnable job = (batchStatter == null) ? outputStatJob(dirtyKeys, shard, knownModifiedOutputFiles, sortedKnownModifiedOutputFiles) : batchStatJob(dirtyKeys, shard, batchStatter, knownModifiedOutputFiles, sortedKnownModifiedOutputFiles);
        executor.submit(wrapper.wrap(job));
    }
    boolean interrupted = ExecutorUtil.interruptibleShutdown(executor);
    Throwables.propagateIfPossible(wrapper.getFirstThrownError());
    LOG.info("Completed output file stat checks");
    if (interrupted) {
        throw new InterruptedException();
    }
    return dirtyKeys;
}

Example 2 with ThrowableRecordingRunnableWrapper

use of com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper in project bazel by bazelbuild.

the class SkyframeActionExecutor method constructActionGraphAndPathMap.

/**
   * Simultaneously construct an action graph for all the actions in Skyframe and a map from
   * {@link PathFragment}s to their respective {@link Artifact}s. We do this in a threadpool to save
   * around 1.5 seconds on a mid-sized build versus a single-threaded operation.
   */
private static Pair<ActionGraph, SortedMap<PathFragment, Artifact>> constructActionGraphAndPathMap(Iterable<ActionLookupValue> values, ConcurrentMap<ActionAnalysisMetadata, ConflictException> badActionMap) throws InterruptedException {
    MutableActionGraph actionGraph = new MapBasedActionGraph();
    ConcurrentNavigableMap<PathFragment, Artifact> artifactPathMap = new ConcurrentSkipListMap<>();
    // Action graph construction is CPU-bound.
    int numJobs = Runtime.getRuntime().availableProcessors();
    // No great reason for expecting 5000 action lookup values, but not worth counting size of
    // values.
    Sharder<ActionLookupValue> actionShards = new Sharder<>(numJobs, 5000);
    for (ActionLookupValue value : values) {
        actionShards.add(value);
    }
    ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper("SkyframeActionExecutor#constructActionGraphAndPathMap");
    ExecutorService executor = Executors.newFixedThreadPool(numJobs, new ThreadFactoryBuilder().setNameFormat("ActionLookupValue Processor %d").build());
    for (List<ActionLookupValue> shard : actionShards) {
        executor.execute(wrapper.wrap(actionRegistration(shard, actionGraph, artifactPathMap, badActionMap)));
    }
    boolean interrupted = ExecutorUtil.interruptibleShutdown(executor);
    Throwables.propagateIfPossible(wrapper.getFirstThrownError());
    if (interrupted) {
        throw new InterruptedException();
    }
    return Pair.<ActionGraph, SortedMap<PathFragment, Artifact>>of(actionGraph, artifactPathMap);
}

Example 3 with ThrowableRecordingRunnableWrapper

use of com.google.devtools.build.lib.concurrent.ThrowableRecordingRunnableWrapper in project bazel by bazelbuild.

the class FilesystemValueChecker method getDirtyValues.

private BatchDirtyResult getDirtyValues(ValueFetcher fetcher, Iterable<SkyKey> keys, final SkyValueDirtinessChecker checker, final boolean checkMissingValues) throws InterruptedException {
    ExecutorService executor = Executors.newFixedThreadPool(DIRTINESS_CHECK_THREADS, new ThreadFactoryBuilder().setNameFormat("FileSystem Value Invalidator %d").build());
    final BatchDirtyResult batchResult = new BatchDirtyResult();
    ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper("FilesystemValueChecker#getDirtyValues");
    final AtomicInteger numKeysScanned = new AtomicInteger(0);
    final AtomicInteger numKeysChecked = new AtomicInteger(0);
    ElapsedTimeReceiver elapsedTimeReceiver = new ElapsedTimeReceiver() {

        @Override
        public void accept(long elapsedTimeNanos) {
            if (elapsedTimeNanos > 0) {
                LOG.info(String.format("Spent %d ms checking %d filesystem nodes (%d scanned)", TimeUnit.MILLISECONDS.convert(elapsedTimeNanos, TimeUnit.NANOSECONDS), numKeysChecked.get(), numKeysScanned.get()));
            }
        }
    };
    try (AutoProfiler prof = AutoProfiler.create(elapsedTimeReceiver)) {
        for (final SkyKey key : keys) {
            numKeysScanned.incrementAndGet();
            if (!checker.applies(key)) {
                continue;
            }
            final SkyValue value = fetcher.get(key);
            if (!checkMissingValues && value == null) {
                continue;
            }
            executor.execute(wrapper.wrap(new Runnable() {

                @Override
                public void run() {
                    numKeysChecked.incrementAndGet();
                    DirtyResult result = checker.check(key, value, tsgm);
                    if (result.isDirty()) {
                        batchResult.add(key, value, result.getNewValue());
                    }
                }
            }));
        }
        boolean interrupted = ExecutorUtil.interruptibleShutdown(executor);
        Throwables.propagateIfPossible(wrapper.getFirstThrownError());
        if (interrupted) {
            throw new InterruptedException();
        }
    }
    return batchResult;
}