Examples with ConcurrencyLimit com.facebook.buck.util.concurrent.ConcurrencyLimit used on opensource projects

Example 1 with ConcurrencyLimit

use of com.facebook.buck.util.concurrent.ConcurrencyLimit in project buck by facebook.

the class JavaBuildGraphProcessor method run.

/**
   * Creates the appropriate target graph and other resources needed for the {@link Processor} and
   * runs it. This method will take responsibility for cleaning up the executor service after it
   * runs.
   */
static void run(final CommandRunnerParams params, final AbstractCommand command, final Processor processor) throws ExitCodeException, InterruptedException, IOException {
    final ConcurrencyLimit concurrencyLimit = command.getConcurrencyLimit(params.getBuckConfig());
    try (CommandThreadManager pool = new CommandThreadManager(command.getClass().getName(), concurrencyLimit)) {
        Cell cell = params.getCell();
        WeightedListeningExecutorService executorService = pool.getExecutor();
        // Ideally, we should be able to construct the TargetGraph quickly assuming most of it is
        // already in memory courtesy of buckd. Though we could make a performance optimization where
        // we pass an option to buck.py that tells it to ignore reading the BUCK.autodeps files when
        // parsing the BUCK files because we never need to consider the existing auto-generated deps
        // when creating the new auto-generated deps. If we did so, we would have to make sure to keep
        // the nodes for that version of the graph separate from the ones that are actually used for
        // building.
        TargetGraph graph;
        try {
            graph = params.getParser().buildTargetGraphForTargetNodeSpecs(params.getBuckEventBus(), cell, command.getEnableParserProfiling(), executorService, ImmutableList.of(TargetNodePredicateSpec.of(x -> true, BuildFileSpec.fromRecursivePath(Paths.get(""), cell.getRoot()))), /* ignoreBuckAutodepsFiles */
            true).getTargetGraph();
        } catch (BuildTargetException | BuildFileParseException e) {
            params.getBuckEventBus().post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e)));
            throw new ExitCodeException(1);
        }
        BuildRuleResolver buildRuleResolver = new BuildRuleResolver(graph, new DefaultTargetNodeToBuildRuleTransformer());
        CachingBuildEngineBuckConfig cachingBuildEngineBuckConfig = params.getBuckConfig().getView(CachingBuildEngineBuckConfig.class);
        LocalCachingBuildEngineDelegate cachingBuildEngineDelegate = new LocalCachingBuildEngineDelegate(params.getFileHashCache());
        BuildEngine buildEngine = new CachingBuildEngine(cachingBuildEngineDelegate, executorService, executorService, new DefaultStepRunner(), CachingBuildEngine.BuildMode.SHALLOW, cachingBuildEngineBuckConfig.getBuildDepFiles(), cachingBuildEngineBuckConfig.getBuildMaxDepFileCacheEntries(), cachingBuildEngineBuckConfig.getBuildArtifactCacheSizeLimit(), params.getObjectMapper(), buildRuleResolver, cachingBuildEngineBuckConfig.getResourceAwareSchedulingInfo(), new RuleKeyFactoryManager(params.getBuckConfig().getKeySeed(), fs -> cachingBuildEngineDelegate.getFileHashCache(), buildRuleResolver, cachingBuildEngineBuckConfig.getBuildInputRuleKeyFileSizeLimit(), new DefaultRuleKeyCache<>()));
        // Create a BuildEngine because we store symbol information as build artifacts.
        BuckEventBus eventBus = params.getBuckEventBus();
        ExecutionContext executionContext = ExecutionContext.builder().setConsole(params.getConsole()).setConcurrencyLimit(concurrencyLimit).setBuckEventBus(eventBus).setEnvironment(/* environment */
        ImmutableMap.of()).setExecutors(ImmutableMap.<ExecutorPool, ListeningExecutorService>of(ExecutorPool.CPU, executorService)).setJavaPackageFinder(params.getJavaPackageFinder()).setObjectMapper(params.getObjectMapper()).setPlatform(params.getPlatform()).setCellPathResolver(params.getCell().getCellPathResolver()).build();
        SourcePathResolver pathResolver = new SourcePathResolver(new SourcePathRuleFinder(buildRuleResolver));
        BuildEngineBuildContext buildContext = BuildEngineBuildContext.builder().setBuildContext(BuildContext.builder().setActionGraph(new ActionGraph(ImmutableList.of())).setSourcePathResolver(pathResolver).setJavaPackageFinder(executionContext.getJavaPackageFinder()).setEventBus(eventBus).build()).setClock(params.getClock()).setArtifactCache(params.getArtifactCacheFactory().newInstance()).setBuildId(eventBus.getBuildId()).setObjectMapper(params.getObjectMapper()).setEnvironment(executionContext.getEnvironment()).setKeepGoing(false).build();
        // Traverse the TargetGraph to find all of the auto-generated dependencies.
        JavaDepsFinder javaDepsFinder = JavaDepsFinder.createJavaDepsFinder(params.getBuckConfig(), params.getCell().getCellPathResolver(), params.getObjectMapper(), buildContext, executionContext, buildEngine);
        processor.process(graph, javaDepsFinder, executorService);
    }
}

Example 2 with ConcurrencyLimit

use of com.facebook.buck.util.concurrent.ConcurrencyLimit in project buck by facebook.

the class NdkBuildStep method getShellCommandInternal.

@Override
protected ImmutableList<String> getShellCommandInternal(ExecutionContext context) {
    Optional<Path> ndkRoot = context.getAndroidPlatformTarget().getNdkDirectory();
    if (!ndkRoot.isPresent()) {
        throw new HumanReadableException("Must set ANDROID_NDK to point to the absolute path of your Android NDK directory.");
    }
    Optional<Path> ndkBuild = new ExecutableFinder().getOptionalExecutable(Paths.get("ndk-build"), ndkRoot.get());
    if (!ndkBuild.isPresent()) {
        throw new HumanReadableException("Unable to find ndk-build");
    }
    ConcurrencyLimit concurrencyLimit = context.getConcurrencyLimit();
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    builder.add(ndkBuild.get().toAbsolutePath().toString(), "-j", // coordinate job concurrency.
    Integer.toString(concurrencyLimit.threadLimit), "-C", this.root.toString());
    if (concurrencyLimit.loadLimit < Double.POSITIVE_INFINITY) {
        builder.add("--load-average", Double.toString(concurrencyLimit.loadLimit));
    }
    Iterable<String> flags = Iterables.transform(this.flags, macroExpander);
    builder.addAll(flags);
    // We want relative, not absolute, paths in the debug-info for binaries we build using
    // ndk_library.  Absolute paths are machine-specific, but relative ones should be the
    // same everywhere.
    Path relativePathToProject = filesystem.resolve(root).relativize(filesystem.getRootPath());
    builder.add("APP_PROJECT_PATH=" + filesystem.resolve(buildArtifactsDirectory) + File.separatorChar, "APP_BUILD_SCRIPT=" + filesystem.resolve(makefile), "NDK_OUT=" + filesystem.resolve(buildArtifactsDirectory) + File.separatorChar, "NDK_LIBS_OUT=" + filesystem.resolve(binDirectory), "BUCK_PROJECT_DIR=" + relativePathToProject);
    // Suppress the custom build step messages (e.g. "Compile++ ...").
    if (Platform.detect() == Platform.WINDOWS) {
        builder.add("host-echo-build-step=@REM");
    } else {
        builder.add("host-echo-build-step=@#");
    }
    // If we're running verbosely, force all the subcommands from the ndk build to be printed out.
    if (context.getVerbosity().shouldPrintCommand()) {
        builder.add("V=1");
    // Otherwise, suppress everything, including the "make: entering directory..." messages.
    } else {
        builder.add("--silent");
    }
    return builder.build();
}

Example 3 with ConcurrencyLimit

use of com.facebook.buck.util.concurrent.ConcurrencyLimit in project buck by facebook.

the class CommandThreadManagerTest method throwsOnHang.

@Test
@SuppressWarnings("PMD.EmptyWhileStmt")
public void throwsOnHang() throws InterruptedException {
    exception.expect(RuntimeException.class);
    exception.expectMessage("Shutdown timed out for thread pool Test");
    exception.expectMessage("Thread Test-0");
    exception.expectMessage(this.getClass().getName());
    ConcurrencyLimit concurrencyLimit = new ConcurrencyLimit(/* threadLimit */
    1, /* loadLimit */
    Double.POSITIVE_INFINITY, ResourceAllocationFairness.FAIR, /* managedThreadCount */
    1, ResourceAmountsEstimator.DEFAULT_AMOUNTS, ResourceAmountsEstimator.DEFAULT_MAXIMUM_AMOUNTS.withCpu(1));
    try (CommandThreadManager pool = new CommandThreadManager("Test", concurrencyLimit, 250, TimeUnit.MILLISECONDS)) {
        pool.getExecutor().submit(new Runnable() {

            @Override
            public void run() {
                while (true) {
                }
            }
        });
    }
}