Examples with ListeningExecutorService com.google.common.util.concurrent.ListeningExecutorService used on opensource projects

Example 6 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project buck by facebook.

the class JavaBuildGraphProcessor method run.

/**
   * Creates the appropriate target graph and other resources needed for the {@link Processor} and
   * runs it. This method will take responsibility for cleaning up the executor service after it
   * runs.
   */
static void run(final CommandRunnerParams params, final AbstractCommand command, final Processor processor) throws ExitCodeException, InterruptedException, IOException {
    final ConcurrencyLimit concurrencyLimit = command.getConcurrencyLimit(params.getBuckConfig());
    try (CommandThreadManager pool = new CommandThreadManager(command.getClass().getName(), concurrencyLimit)) {
        Cell cell = params.getCell();
        WeightedListeningExecutorService executorService = pool.getExecutor();
        // Ideally, we should be able to construct the TargetGraph quickly assuming most of it is
        // already in memory courtesy of buckd. Though we could make a performance optimization where
        // we pass an option to buck.py that tells it to ignore reading the BUCK.autodeps files when
        // parsing the BUCK files because we never need to consider the existing auto-generated deps
        // when creating the new auto-generated deps. If we did so, we would have to make sure to keep
        // the nodes for that version of the graph separate from the ones that are actually used for
        // building.
        TargetGraph graph;
        try {
            graph = params.getParser().buildTargetGraphForTargetNodeSpecs(params.getBuckEventBus(), cell, command.getEnableParserProfiling(), executorService, ImmutableList.of(TargetNodePredicateSpec.of(x -> true, BuildFileSpec.fromRecursivePath(Paths.get(""), cell.getRoot()))), /* ignoreBuckAutodepsFiles */
            true).getTargetGraph();
        } catch (BuildTargetException | BuildFileParseException e) {
            params.getBuckEventBus().post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e)));
            throw new ExitCodeException(1);
        }
        BuildRuleResolver buildRuleResolver = new BuildRuleResolver(graph, new DefaultTargetNodeToBuildRuleTransformer());
        CachingBuildEngineBuckConfig cachingBuildEngineBuckConfig = params.getBuckConfig().getView(CachingBuildEngineBuckConfig.class);
        LocalCachingBuildEngineDelegate cachingBuildEngineDelegate = new LocalCachingBuildEngineDelegate(params.getFileHashCache());
        BuildEngine buildEngine = new CachingBuildEngine(cachingBuildEngineDelegate, executorService, executorService, new DefaultStepRunner(), CachingBuildEngine.BuildMode.SHALLOW, cachingBuildEngineBuckConfig.getBuildDepFiles(), cachingBuildEngineBuckConfig.getBuildMaxDepFileCacheEntries(), cachingBuildEngineBuckConfig.getBuildArtifactCacheSizeLimit(), params.getObjectMapper(), buildRuleResolver, cachingBuildEngineBuckConfig.getResourceAwareSchedulingInfo(), new RuleKeyFactoryManager(params.getBuckConfig().getKeySeed(), fs -> cachingBuildEngineDelegate.getFileHashCache(), buildRuleResolver, cachingBuildEngineBuckConfig.getBuildInputRuleKeyFileSizeLimit(), new DefaultRuleKeyCache<>()));
        // Create a BuildEngine because we store symbol information as build artifacts.
        BuckEventBus eventBus = params.getBuckEventBus();
        ExecutionContext executionContext = ExecutionContext.builder().setConsole(params.getConsole()).setConcurrencyLimit(concurrencyLimit).setBuckEventBus(eventBus).setEnvironment(/* environment */
        ImmutableMap.of()).setExecutors(ImmutableMap.<ExecutorPool, ListeningExecutorService>of(ExecutorPool.CPU, executorService)).setJavaPackageFinder(params.getJavaPackageFinder()).setObjectMapper(params.getObjectMapper()).setPlatform(params.getPlatform()).setCellPathResolver(params.getCell().getCellPathResolver()).build();
        SourcePathResolver pathResolver = new SourcePathResolver(new SourcePathRuleFinder(buildRuleResolver));
        BuildEngineBuildContext buildContext = BuildEngineBuildContext.builder().setBuildContext(BuildContext.builder().setActionGraph(new ActionGraph(ImmutableList.of())).setSourcePathResolver(pathResolver).setJavaPackageFinder(executionContext.getJavaPackageFinder()).setEventBus(eventBus).build()).setClock(params.getClock()).setArtifactCache(params.getArtifactCacheFactory().newInstance()).setBuildId(eventBus.getBuildId()).setObjectMapper(params.getObjectMapper()).setEnvironment(executionContext.getEnvironment()).setKeepGoing(false).build();
        // Traverse the TargetGraph to find all of the auto-generated dependencies.
        JavaDepsFinder javaDepsFinder = JavaDepsFinder.createJavaDepsFinder(params.getBuckConfig(), params.getCell().getCellPathResolver(), params.getObjectMapper(), buildContext, executionContext, buildEngine);
        processor.process(graph, javaDepsFinder, executorService);
    }
}

Example 7 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project buck by facebook.

the class ProjectCommand method generateWorkspacesForTargets.

@VisibleForTesting
static ImmutableSet<BuildTarget> generateWorkspacesForTargets(final CommandRunnerParams params, final TargetGraphAndTargets targetGraphAndTargets, ImmutableSet<BuildTarget> passedInTargetsSet, ImmutableSet<ProjectGenerator.Option> options, ImmutableList<String> buildWithBuckFlags, Optional<ImmutableSet<UnflavoredBuildTarget>> focusModules, Map<Path, ProjectGenerator> projectGenerators, boolean combinedProject, boolean buildWithBuck) throws IOException, InterruptedException {
    ImmutableSet<BuildTarget> targets;
    if (passedInTargetsSet.isEmpty()) {
        targets = targetGraphAndTargets.getProjectRoots().stream().map(TargetNode::getBuildTarget).collect(MoreCollectors.toImmutableSet());
    } else {
        targets = passedInTargetsSet;
    }
    LOG.debug("Generating workspace for config targets %s", targets);
    ImmutableSet.Builder<BuildTarget> requiredBuildTargetsBuilder = ImmutableSet.builder();
    for (final BuildTarget inputTarget : targets) {
        TargetNode<?, ?> inputNode = targetGraphAndTargets.getTargetGraph().get(inputTarget);
        XcodeWorkspaceConfigDescription.Arg workspaceArgs;
        if (inputNode.getDescription() instanceof XcodeWorkspaceConfigDescription) {
            TargetNode<XcodeWorkspaceConfigDescription.Arg, ?> castedWorkspaceNode = castToXcodeWorkspaceTargetNode(inputNode);
            workspaceArgs = castedWorkspaceNode.getConstructorArg();
        } else if (canGenerateImplicitWorkspaceForDescription(inputNode.getDescription())) {
            workspaceArgs = createImplicitWorkspaceArgs(inputNode);
        } else {
            throw new HumanReadableException("%s must be a xcode_workspace_config, apple_binary, apple_bundle, or apple_library", inputNode);
        }
        BuckConfig buckConfig = params.getBuckConfig();
        AppleConfig appleConfig = new AppleConfig(buckConfig);
        HalideBuckConfig halideBuckConfig = new HalideBuckConfig(buckConfig);
        CxxBuckConfig cxxBuckConfig = new CxxBuckConfig(buckConfig);
        SwiftBuckConfig swiftBuckConfig = new SwiftBuckConfig(buckConfig);
        CxxPlatform defaultCxxPlatform = params.getCell().getKnownBuildRuleTypes().getDefaultCxxPlatforms();
        WorkspaceAndProjectGenerator generator = new WorkspaceAndProjectGenerator(params.getCell(), targetGraphAndTargets.getTargetGraph(), workspaceArgs, inputTarget, options, combinedProject, buildWithBuck, buildWithBuckFlags, focusModules, !appleConfig.getXcodeDisableParallelizeBuild(), new ExecutableFinder(), params.getEnvironment(), params.getCell().getKnownBuildRuleTypes().getCxxPlatforms(), defaultCxxPlatform, params.getBuckConfig().getView(ParserConfig.class).getBuildFileName(), input -> ActionGraphCache.getFreshActionGraph(params.getBuckEventBus(), targetGraphAndTargets.getTargetGraph().getSubgraph(ImmutableSet.of(input))).getResolver(), params.getBuckEventBus(), halideBuckConfig, cxxBuckConfig, appleConfig, swiftBuckConfig);
        ListeningExecutorService executorService = params.getExecutors().get(ExecutorPool.PROJECT);
        Preconditions.checkNotNull(executorService, "CommandRunnerParams does not have executor for PROJECT pool");
        generator.generateWorkspaceAndDependentProjects(projectGenerators, executorService);
        ImmutableSet<BuildTarget> requiredBuildTargetsForWorkspace = generator.getRequiredBuildTargets();
        LOG.debug("Required build targets for workspace %s: %s", inputTarget, requiredBuildTargetsForWorkspace);
        requiredBuildTargetsBuilder.addAll(requiredBuildTargetsForWorkspace);
    }
    return requiredBuildTargetsBuilder.build();
}

Example 8 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project buck by facebook.

the class ArtifactCaches method createHttpArtifactCache.

private static ArtifactCache createHttpArtifactCache(HttpCacheEntry cacheDescription, final String hostToReportToRemote, final BuckEventBus buckEventBus, ProjectFilesystem projectFilesystem, ListeningExecutorService httpWriteExecutorService, ArtifactCacheBuckConfig config, NetworkCacheFactory factory, boolean distributedBuildModeEnabled) {
    // Setup the default client to use.
    OkHttpClient.Builder storeClientBuilder = new OkHttpClient.Builder();
    storeClientBuilder.networkInterceptors().add(chain -> chain.proceed(chain.request().newBuilder().addHeader("X-BuckCache-User", stripNonAscii(System.getProperty("user.name", "<unknown>"))).addHeader("X-BuckCache-Host", stripNonAscii(hostToReportToRemote)).build()));
    int timeoutSeconds = cacheDescription.getTimeoutSeconds();
    setTimeouts(storeClientBuilder, timeoutSeconds);
    storeClientBuilder.connectionPool(new ConnectionPool(/* maxIdleConnections */
    (int) config.getThreadPoolSize(), /* keepAliveDurationMs */
    config.getThreadPoolKeepAliveDurationMillis(), TimeUnit.MILLISECONDS));
    // The artifact cache effectively only connects to a single host at a time. We should allow as
    // many concurrent connections to that host as we allow threads.
    Dispatcher dispatcher = new Dispatcher();
    dispatcher.setMaxRequestsPerHost((int) config.getThreadPoolSize());
    storeClientBuilder.dispatcher(dispatcher);
    final ImmutableMap<String, String> readHeaders = cacheDescription.getReadHeaders();
    final ImmutableMap<String, String> writeHeaders = cacheDescription.getWriteHeaders();
    // If write headers are specified, add them to every default client request.
    if (!writeHeaders.isEmpty()) {
        storeClientBuilder.networkInterceptors().add(chain -> chain.proceed(addHeadersToBuilder(chain.request().newBuilder(), writeHeaders).build()));
    }
    OkHttpClient storeClient = storeClientBuilder.build();
    // For fetches, use a client with a read timeout.
    OkHttpClient.Builder fetchClientBuilder = storeClient.newBuilder();
    setTimeouts(fetchClientBuilder, timeoutSeconds);
    // If read headers are specified, add them to every read client request.
    if (!readHeaders.isEmpty()) {
        fetchClientBuilder.networkInterceptors().add(chain -> chain.proceed(addHeadersToBuilder(chain.request().newBuilder(), readHeaders).build()));
    }
    fetchClientBuilder.networkInterceptors().add((chain -> {
        Response originalResponse = chain.proceed(chain.request());
        return originalResponse.newBuilder().body(new ProgressResponseBody(originalResponse.body(), buckEventBus)).build();
    }));
    OkHttpClient fetchClient = fetchClientBuilder.build();
    HttpService fetchService;
    HttpService storeService;
    switch(config.getLoadBalancingType()) {
        case CLIENT_SLB:
            HttpLoadBalancer clientSideSlb = config.getSlbConfig().createClientSideSlb(new DefaultClock(), buckEventBus, new CommandThreadFactory("ArtifactCaches.HttpLoadBalancer", SLB_THREAD_PRIORITY));
            fetchService = new RetryingHttpService(buckEventBus, new LoadBalancedService(clientSideSlb, fetchClient, buckEventBus), config.getMaxFetchRetries());
            storeService = new LoadBalancedService(clientSideSlb, storeClient, buckEventBus);
            break;
        case SINGLE_SERVER:
            URI url = cacheDescription.getUrl();
            fetchService = new SingleUriService(url, fetchClient);
            storeService = new SingleUriService(url, storeClient);
            break;
        default:
            throw new IllegalArgumentException("Unknown HttpLoadBalancer type: " + config.getLoadBalancingType());
    }
    String cacheName = cacheDescription.getName().map(input -> "http-" + input).orElse("http");
    boolean doStore = cacheDescription.getCacheReadMode().isDoStore();
    return factory.newInstance(NetworkCacheArgs.builder().setThriftEndpointPath(config.getHybridThriftEndpoint()).setCacheName(cacheName).setRepository(config.getRepository()).setScheduleType(config.getScheduleType()).setFetchClient(fetchService).setStoreClient(storeService).setDoStore(doStore).setProjectFilesystem(projectFilesystem).setBuckEventBus(buckEventBus).setHttpWriteExecutorService(httpWriteExecutorService).setErrorTextTemplate(cacheDescription.getErrorMessageFormat()).setDistributedBuildModeEnabled(distributedBuildModeEnabled).build());
}

Example 9 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project buck by facebook.

the class ProjectBuildFileParserPoolTest method fuzzForConcurrentAccess.

@Test
public void fuzzForConcurrentAccess() throws Exception {
    final int parsersCount = 3;
    Cell cell = EasyMock.createMock(Cell.class);
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(4));
    try (ProjectBuildFileParserPool parserPool = new ProjectBuildFileParserPool(parsersCount, input -> {
        final AtomicInteger sleepCallCount = new AtomicInteger(0);
        return createMockParser(() -> {
            int numCalls = sleepCallCount.incrementAndGet();
            Preconditions.checkState(numCalls == 1);
            try {
                Thread.sleep(10);
            } finally {
                sleepCallCount.decrementAndGet();
            }
            return ImmutableList.of();
        });
    })) {
        Futures.allAsList(scheduleWork(cell, parserPool, executorService, 142)).get();
    } finally {
        executorService.shutdown();
    }
}

Example 10 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project buck by facebook.

the class ProjectBuildFileParserPoolTest method createsConstrainedNumberOfParsers.

@Test
public void createsConstrainedNumberOfParsers() throws Exception {
    ListeningExecutorService executorService = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(2));
    assertHowManyParserInstancesAreCreated(/* executor */
    executorService, /* maxParsers */
    2, /* requests */
    3, /* expectedCreateCount */
    2);
}