Examples with TargetGraph com.facebook.buck.rules.TargetGraph used on opensource projects

Example 36 with TargetGraph

use of com.facebook.buck.rules.TargetGraph in project buck by facebook.

the class JavaBuildGraphProcessor method run.

/**
   * Creates the appropriate target graph and other resources needed for the {@link Processor} and
   * runs it. This method will take responsibility for cleaning up the executor service after it
   * runs.
   */
static void run(final CommandRunnerParams params, final AbstractCommand command, final Processor processor) throws ExitCodeException, InterruptedException, IOException {
    final ConcurrencyLimit concurrencyLimit = command.getConcurrencyLimit(params.getBuckConfig());
    try (CommandThreadManager pool = new CommandThreadManager(command.getClass().getName(), concurrencyLimit)) {
        Cell cell = params.getCell();
        WeightedListeningExecutorService executorService = pool.getExecutor();
        // Ideally, we should be able to construct the TargetGraph quickly assuming most of it is
        // already in memory courtesy of buckd. Though we could make a performance optimization where
        // we pass an option to buck.py that tells it to ignore reading the BUCK.autodeps files when
        // parsing the BUCK files because we never need to consider the existing auto-generated deps
        // when creating the new auto-generated deps. If we did so, we would have to make sure to keep
        // the nodes for that version of the graph separate from the ones that are actually used for
        // building.
        TargetGraph graph;
        try {
            graph = params.getParser().buildTargetGraphForTargetNodeSpecs(params.getBuckEventBus(), cell, command.getEnableParserProfiling(), executorService, ImmutableList.of(TargetNodePredicateSpec.of(x -> true, BuildFileSpec.fromRecursivePath(Paths.get(""), cell.getRoot()))), /* ignoreBuckAutodepsFiles */
            true).getTargetGraph();
        } catch (BuildTargetException | BuildFileParseException e) {
            params.getBuckEventBus().post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e)));
            throw new ExitCodeException(1);
        }
        BuildRuleResolver buildRuleResolver = new BuildRuleResolver(graph, new DefaultTargetNodeToBuildRuleTransformer());
        CachingBuildEngineBuckConfig cachingBuildEngineBuckConfig = params.getBuckConfig().getView(CachingBuildEngineBuckConfig.class);
        LocalCachingBuildEngineDelegate cachingBuildEngineDelegate = new LocalCachingBuildEngineDelegate(params.getFileHashCache());
        BuildEngine buildEngine = new CachingBuildEngine(cachingBuildEngineDelegate, executorService, executorService, new DefaultStepRunner(), CachingBuildEngine.BuildMode.SHALLOW, cachingBuildEngineBuckConfig.getBuildDepFiles(), cachingBuildEngineBuckConfig.getBuildMaxDepFileCacheEntries(), cachingBuildEngineBuckConfig.getBuildArtifactCacheSizeLimit(), params.getObjectMapper(), buildRuleResolver, cachingBuildEngineBuckConfig.getResourceAwareSchedulingInfo(), new RuleKeyFactoryManager(params.getBuckConfig().getKeySeed(), fs -> cachingBuildEngineDelegate.getFileHashCache(), buildRuleResolver, cachingBuildEngineBuckConfig.getBuildInputRuleKeyFileSizeLimit(), new DefaultRuleKeyCache<>()));
        // Create a BuildEngine because we store symbol information as build artifacts.
        BuckEventBus eventBus = params.getBuckEventBus();
        ExecutionContext executionContext = ExecutionContext.builder().setConsole(params.getConsole()).setConcurrencyLimit(concurrencyLimit).setBuckEventBus(eventBus).setEnvironment(/* environment */
        ImmutableMap.of()).setExecutors(ImmutableMap.<ExecutorPool, ListeningExecutorService>of(ExecutorPool.CPU, executorService)).setJavaPackageFinder(params.getJavaPackageFinder()).setObjectMapper(params.getObjectMapper()).setPlatform(params.getPlatform()).setCellPathResolver(params.getCell().getCellPathResolver()).build();
        SourcePathResolver pathResolver = new SourcePathResolver(new SourcePathRuleFinder(buildRuleResolver));
        BuildEngineBuildContext buildContext = BuildEngineBuildContext.builder().setBuildContext(BuildContext.builder().setActionGraph(new ActionGraph(ImmutableList.of())).setSourcePathResolver(pathResolver).setJavaPackageFinder(executionContext.getJavaPackageFinder()).setEventBus(eventBus).build()).setClock(params.getClock()).setArtifactCache(params.getArtifactCacheFactory().newInstance()).setBuildId(eventBus.getBuildId()).setObjectMapper(params.getObjectMapper()).setEnvironment(executionContext.getEnvironment()).setKeepGoing(false).build();
        // Traverse the TargetGraph to find all of the auto-generated dependencies.
        JavaDepsFinder javaDepsFinder = JavaDepsFinder.createJavaDepsFinder(params.getBuckConfig(), params.getCell().getCellPathResolver(), params.getObjectMapper(), buildContext, executionContext, buildEngine);
        processor.process(graph, javaDepsFinder, executorService);
    }
}

Example 37 with TargetGraph

use of com.facebook.buck.rules.TargetGraph in project buck by facebook.

the class AuditInputCommand method runWithoutHelp.

@Override
public int runWithoutHelp(final CommandRunnerParams params) throws IOException, InterruptedException {
    // Create a TargetGraph that is composed of the transitive closure of all of the dependent
    // TargetNodes for the specified BuildTargets.
    final ImmutableSet<String> fullyQualifiedBuildTargets = ImmutableSet.copyOf(getArgumentsFormattedAsBuildTargets(params.getBuckConfig()));
    if (fullyQualifiedBuildTargets.isEmpty()) {
        params.getBuckEventBus().post(ConsoleEvent.severe("Please specify at least one build target."));
        return 1;
    }
    ImmutableSet<BuildTarget> targets = getArgumentsFormattedAsBuildTargets(params.getBuckConfig()).stream().map(input -> BuildTargetParser.INSTANCE.parse(input, BuildTargetPatternParser.fullyQualified(), params.getCell().getCellPathResolver())).collect(MoreCollectors.toImmutableSet());
    LOG.debug("Getting input for targets: %s", targets);
    TargetGraph graph;
    try (CommandThreadManager pool = new CommandThreadManager("Audit", getConcurrencyLimit(params.getBuckConfig()))) {
        graph = params.getParser().buildTargetGraph(params.getBuckEventBus(), params.getCell(), getEnableParserProfiling(), pool.getExecutor(), targets);
    } catch (BuildFileParseException | BuildTargetException e) {
        params.getBuckEventBus().post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e)));
        return 1;
    }
    if (shouldGenerateJsonOutput()) {
        return printJsonInputs(params, graph);
    }
    return printInputs(params, graph);
}

Example 38 with TargetGraph

use of com.facebook.buck.rules.TargetGraph in project buck by facebook.

the class JavaDepsFinder method findDepsForBuildFiles.

private DepsForBuildFiles findDepsForBuildFiles(final TargetGraph graph, final DependencyInfo dependencyInfo, final Console console) {
    // For the rules that expect to have their deps generated, look through all of their required
    // symbols and try to find the build rule that provides each symbols. Store these build rules in
    // the depsForBuildFiles data structure.
    //
    // Currently, we process each rule with autodeps=True on a single thread. See the class overview
    // for DepsForBuildFiles about what it would take to do this work in a multi-threaded way.
    DepsForBuildFiles depsForBuildFiles = new DepsForBuildFiles();
    for (final TargetNode<?, ?> rule : dependencyInfo.rulesWithAutodeps) {
        final Set<BuildTarget> providedDeps = dependencyInfo.rulesWithAutodepsToProvidedDeps.get(rule);
        final Predicate<TargetNode<?, ?>> isVisibleDepNotAlreadyInProvidedDeps = provider -> provider.isVisibleTo(graph, rule) && !providedDeps.contains(provider.getBuildTarget());
        final boolean isJavaTestRule = rule.getDescription() instanceof JavaTestDescription;
        for (DependencyType type : DependencyType.values()) {
            HashMultimap<TargetNode<?, ?>, String> ruleToSymbolsMap;
            switch(type) {
                case DEPS:
                    ruleToSymbolsMap = dependencyInfo.ruleToRequiredSymbols;
                    break;
                case EXPORTED_DEPS:
                    ruleToSymbolsMap = dependencyInfo.ruleToExportedSymbols;
                    break;
                default:
                    throw new IllegalStateException("Unrecognized type: " + type);
            }
            final DependencyType typeOfDepToAdd;
            if (isJavaTestRule) {
                // java_test rules do not honor exported_deps: add all dependencies to the ordinary deps.
                typeOfDepToAdd = DependencyType.DEPS;
            } else {
                typeOfDepToAdd = type;
            }
            for (String requiredSymbol : ruleToSymbolsMap.get(rule)) {
                BuildTarget provider = findProviderForSymbolFromBuckConfig(requiredSymbol);
                if (provider != null) {
                    depsForBuildFiles.addDep(rule.getBuildTarget(), provider, typeOfDepToAdd);
                    continue;
                }
                Set<TargetNode<?, ?>> providers = dependencyInfo.symbolToProviders.get(requiredSymbol);
                SortedSet<TargetNode<?, ?>> candidateProviders = providers.stream().filter(isVisibleDepNotAlreadyInProvidedDeps).collect(MoreCollectors.toImmutableSortedSet(Comparator.<TargetNode<?, ?>>naturalOrder()));
                int numCandidates = candidateProviders.size();
                if (numCandidates == 1) {
                    depsForBuildFiles.addDep(rule.getBuildTarget(), Iterables.getOnlyElement(candidateProviders).getBuildTarget(), typeOfDepToAdd);
                } else if (numCandidates > 1) {
                    // Warn the user that there is an ambiguity. This could be very common with macros that
                    // generate multiple versions of a java_library() with the same sources.
                    // If numProviders is 0, then hopefully the dep is provided by something the user
                    // hardcoded in the BUCK file.
                    console.printErrorText(String.format("WARNING: Multiple providers for %s: %s. " + "Consider adding entry to .buckconfig to eliminate ambiguity:\n" + "[autodeps]\n" + "java-package-mappings = %s => %s", requiredSymbol, Joiner.on(", ").join(candidateProviders), requiredSymbol, Iterables.getFirst(candidateProviders, null)));
                } else {
                    // If there aren't any candidates, then see if there is a visible rule that can provide
                    // the symbol via its exported_deps. We make this a secondary check because we prefer to
                    // depend on the rule that defines the symbol directly rather than one of possibly many
                    // rules that provides it via its exported_deps.
                    ImmutableSortedSet<TargetNode<?, ?>> newCandidates = providers.stream().flatMap(candidate -> dependencyInfo.ruleToRulesThatExportIt.get(candidate).stream()).filter(ruleThatExportsCandidate -> ruleThatExportsCandidate.isVisibleTo(graph, rule)).collect(MoreCollectors.toImmutableSortedSet(Comparator.<TargetNode<?, ?>>naturalOrder()));
                    int numNewCandidates = newCandidates.size();
                    if (numNewCandidates == 1) {
                        depsForBuildFiles.addDep(rule.getBuildTarget(), Iterables.getOnlyElement(newCandidates).getBuildTarget(), typeOfDepToAdd);
                    } else if (numNewCandidates > 1) {
                        console.printErrorText(String.format("WARNING: No providers found for '%s' for build rule %s, " + "but there are multiple rules that export a rule to provide %s: %s", requiredSymbol, rule.getBuildTarget(), requiredSymbol, Joiner.on(", ").join(newCandidates)));
                    }
                // In the case that numNewCandidates is 0, we assume that the user is taking
                // responsibility for declaring a provider for the symbol by hardcoding it in the deps.
                }
            }
        }
    }
    return depsForBuildFiles;
}

Example 39 with TargetGraph

use of com.facebook.buck.rules.TargetGraph in project buck by facebook.

the class IjModuleGraph method createModules.

/**
   * Create all the modules we are capable of representing in IntelliJ from the supplied graph.
   *
   * @param targetGraph graph whose nodes will be converted to {@link IjModule}s.
   * @return map which for every BuildTarget points to the corresponding IjModule. Multiple
   * BuildTarget can point to one IjModule (many:one mapping), the BuildTargets which
   * can't be prepresented in IntelliJ are missing from this mapping.
   */
private static ImmutableMap<BuildTarget, IjModule> createModules(IjProjectConfig projectConfig, TargetGraph targetGraph, IjModuleFactory moduleFactory, final int minimumPathDepth) {
    final BlockedPathNode blockedPathTree = createAggregationHaltPoints(projectConfig, targetGraph);
    ImmutableListMultimap<Path, TargetNode<?, ?>> baseTargetPathMultimap = targetGraph.getNodes().stream().filter(input -> IjModuleFactory.SUPPORTED_MODULE_DESCRIPTION_CLASSES.contains(input.getDescription().getClass())).collect(MoreCollectors.toImmutableListMultimap(targetNode -> {
        Path path;
        Path basePath = targetNode.getBuildTarget().getBasePath();
        if (targetNode.getConstructorArg() instanceof AndroidResourceDescription.Arg) {
            path = basePath;
        } else {
            path = simplifyPath(basePath, minimumPathDepth, blockedPathTree);
        }
        return path;
    }, targetNode -> targetNode));
    ImmutableMap.Builder<BuildTarget, IjModule> moduleMapBuilder = new ImmutableMap.Builder<>();
    for (Path baseTargetPath : baseTargetPathMultimap.keySet()) {
        ImmutableSet<TargetNode<?, ?>> targets = ImmutableSet.copyOf(baseTargetPathMultimap.get(baseTargetPath));
        IjModule module = moduleFactory.createModule(baseTargetPath, targets);
        for (TargetNode<?, ?> target : targets) {
            moduleMapBuilder.put(target.getBuildTarget(), module);
        }
    }
    return moduleMapBuilder.build();
}

Example 40 with TargetGraph

use of com.facebook.buck.rules.TargetGraph in project buck by facebook.

the class PrebuiltOcamlLibraryDescription method createBuildRule.

@Override
public <A extends Arg> OcamlLibrary createBuildRule(TargetGraph targetGraph, final BuildRuleParams params, BuildRuleResolver resolver, final A args) {
    final BuildTarget target = params.getBuildTarget();
    final boolean bytecodeOnly = args.bytecodeOnly.orElse(false);
    final String libDir = args.libDir.orElse("lib");
    final String libName = args.libName.orElse(target.getShortName());
    final String nativeLib = args.nativeLib.orElse(String.format("%s.cmxa", libName));
    final String bytecodeLib = args.bytecodeLib.orElse(String.format("%s.cma", libName));
    final ImmutableList<String> cLibs = args.cLibs;
    final Path libPath = target.getBasePath().resolve(libDir);
    final Path includeDir = libPath.resolve(args.includeDir.orElse(""));
    final Optional<SourcePath> staticNativeLibraryPath = bytecodeOnly ? Optional.empty() : Optional.of(new PathSourcePath(params.getProjectFilesystem(), libPath.resolve(nativeLib)));
    final SourcePath staticBytecodeLibraryPath = new PathSourcePath(params.getProjectFilesystem(), libPath.resolve(bytecodeLib));
    final ImmutableList<SourcePath> staticCLibraryPaths = cLibs.stream().map(input -> new PathSourcePath(params.getProjectFilesystem(), libPath.resolve(input))).collect(MoreCollectors.toImmutableList());
    final SourcePath bytecodeLibraryPath = new PathSourcePath(params.getProjectFilesystem(), libPath.resolve(bytecodeLib));
    SourcePathRuleFinder ruleFinder = new SourcePathRuleFinder(resolver);
    return new PrebuiltOcamlLibrary(params, ruleFinder, staticNativeLibraryPath, staticBytecodeLibraryPath, staticCLibraryPaths, bytecodeLibraryPath, libPath, includeDir);
}