Examples with Sharder com.google.devtools.build.lib.concurrent.Sharder used on opensource projects

Example 1 with Sharder

use of com.google.devtools.build.lib.concurrent.Sharder in project bazel by bazelbuild.

the class FilesystemValueChecker method getDirtyActionValues.

/**
   * Return a collection of action values which have output files that are not in-sync with
   * the on-disk file value (were modified externally).
   */
Collection<SkyKey> getDirtyActionValues(Map<SkyKey, SkyValue> valuesMap, @Nullable final BatchStat batchStatter, ModifiedFileSet modifiedOutputFiles) throws InterruptedException {
    if (modifiedOutputFiles == ModifiedFileSet.NOTHING_MODIFIED) {
        LOG.info("Not checking for dirty actions since nothing was modified");
        return ImmutableList.of();
    }
    LOG.info("Accumulating dirty actions");
    final int numOutputJobs = Runtime.getRuntime().availableProcessors() * 4;
    final Set<SkyKey> actionSkyKeys = new HashSet<>();
    for (SkyKey key : valuesMap.keySet()) {
        if (ACTION_FILTER.apply(key)) {
            actionSkyKeys.add(key);
        }
    }
    final Sharder<Pair<SkyKey, ActionExecutionValue>> outputShards = new Sharder<>(numOutputJobs, actionSkyKeys.size());
    for (SkyKey key : actionSkyKeys) {
        outputShards.add(Pair.of(key, (ActionExecutionValue) valuesMap.get(key)));
    }
    LOG.info("Sharded action values for batching");
    ExecutorService executor = Executors.newFixedThreadPool(numOutputJobs, new ThreadFactoryBuilder().setNameFormat("FileSystem Output File Invalidator %d").build());
    Collection<SkyKey> dirtyKeys = Sets.newConcurrentHashSet();
    ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper("FileSystemValueChecker#getDirtyActionValues");
    modifiedOutputFilesCounter.set(0);
    modifiedOutputFilesIntraBuildCounter.set(0);
    final ImmutableSet<PathFragment> knownModifiedOutputFiles = modifiedOutputFiles == ModifiedFileSet.EVERYTHING_MODIFIED ? null : modifiedOutputFiles.modifiedSourceFiles();
    // Initialized lazily through a supplier because it is only used to check modified
    // TreeArtifacts, which are not frequently used in builds.
    Supplier<NavigableSet<PathFragment>> sortedKnownModifiedOutputFiles = Suppliers.memoize(new Supplier<NavigableSet<PathFragment>>() {

        @Override
        public NavigableSet<PathFragment> get() {
            if (knownModifiedOutputFiles == null) {
                return null;
            } else {
                return ImmutableSortedSet.copyOf(knownModifiedOutputFiles);
            }
        }
    });
    for (List<Pair<SkyKey, ActionExecutionValue>> shard : outputShards) {
        Runnable job = (batchStatter == null) ? outputStatJob(dirtyKeys, shard, knownModifiedOutputFiles, sortedKnownModifiedOutputFiles) : batchStatJob(dirtyKeys, shard, batchStatter, knownModifiedOutputFiles, sortedKnownModifiedOutputFiles);
        executor.submit(wrapper.wrap(job));
    }
    boolean interrupted = ExecutorUtil.interruptibleShutdown(executor);
    Throwables.propagateIfPossible(wrapper.getFirstThrownError());
    LOG.info("Completed output file stat checks");
    if (interrupted) {
        throw new InterruptedException();
    }
    return dirtyKeys;
}

Example 2 with Sharder

use of com.google.devtools.build.lib.concurrent.Sharder in project bazel by bazelbuild.

the class SkyframeActionExecutor method constructActionGraphAndPathMap.

/**
   * Simultaneously construct an action graph for all the actions in Skyframe and a map from
   * {@link PathFragment}s to their respective {@link Artifact}s. We do this in a threadpool to save
   * around 1.5 seconds on a mid-sized build versus a single-threaded operation.
   */
private static Pair<ActionGraph, SortedMap<PathFragment, Artifact>> constructActionGraphAndPathMap(Iterable<ActionLookupValue> values, ConcurrentMap<ActionAnalysisMetadata, ConflictException> badActionMap) throws InterruptedException {
    MutableActionGraph actionGraph = new MapBasedActionGraph();
    ConcurrentNavigableMap<PathFragment, Artifact> artifactPathMap = new ConcurrentSkipListMap<>();
    // Action graph construction is CPU-bound.
    int numJobs = Runtime.getRuntime().availableProcessors();
    // No great reason for expecting 5000 action lookup values, but not worth counting size of
    // values.
    Sharder<ActionLookupValue> actionShards = new Sharder<>(numJobs, 5000);
    for (ActionLookupValue value : values) {
        actionShards.add(value);
    }
    ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper("SkyframeActionExecutor#constructActionGraphAndPathMap");
    ExecutorService executor = Executors.newFixedThreadPool(numJobs, new ThreadFactoryBuilder().setNameFormat("ActionLookupValue Processor %d").build());
    for (List<ActionLookupValue> shard : actionShards) {
        executor.execute(wrapper.wrap(actionRegistration(shard, actionGraph, artifactPathMap, badActionMap)));
    }
    boolean interrupted = ExecutorUtil.interruptibleShutdown(executor);
    Throwables.propagateIfPossible(wrapper.getFirstThrownError());
    if (interrupted) {
        throw new InterruptedException();
    }
    return Pair.<ActionGraph, SortedMap<PathFragment, Artifact>>of(actionGraph, artifactPathMap);
}