Examples with BuildFileAST com.google.devtools.build.lib.syntax.BuildFileAST used on opensource projects

Example 6 with BuildFileAST

use of com.google.devtools.build.lib.syntax.BuildFileAST in project bazel by bazelbuild.

the class EvaluationTestCase method parseFile.

protected List<Statement> parseFile(String... input) {
    BuildFileAST ast = BuildFileAST.parseSkylarkString(getEventHandler(), input);
    ast = ast.validate(new ValidationEnvironment(env), getEventHandler());
    return ast.getStatements();
}

Example 7 with BuildFileAST

use of com.google.devtools.build.lib.syntax.BuildFileAST in project bazel by bazelbuild.

the class PackageFunction method loadPackage.

/**
   * Constructs a {@link Package} object for the given package using legacy package loading.
   * Note that the returned package may be in error.
   *
   * <p>May return null if the computation has to be restarted.
   *
   * <p>Exactly one of {@code replacementContents} and {@code buildFileValue} will be
   * non-{@code null}. The former indicates that we have a faux BUILD file with the given contents
   * and the latter indicates that we have a legitimate BUILD file and should actually do
   * preprocessing.
   */
@Nullable
private CacheEntryWithGlobDeps<Package.Builder> loadPackage(String workspaceName, @Nullable String replacementContents, PackageIdentifier packageId, Path buildFilePath, @Nullable FileValue buildFileValue, RuleVisibility defaultVisibility, List<Statement> preludeStatements, Path packageRoot, Environment env) throws InterruptedException, PackageFunctionException {
    CacheEntryWithGlobDeps<Package.Builder> packageFunctionCacheEntry = packageFunctionCache.getIfPresent(packageId);
    if (packageFunctionCacheEntry == null) {
        profiler.startTask(ProfilerTask.CREATE_PACKAGE, packageId.toString());
        if (packageProgress != null) {
            packageProgress.startReadPackage(packageId);
        }
        try {
            CacheEntryWithGlobDeps<AstAfterPreprocessing> astCacheEntry = astCache.getIfPresent(packageId);
            if (astCacheEntry == null) {
                if (showLoadingProgress.get()) {
                    env.getListener().handle(Event.progress("Loading package: " + packageId));
                }
                // We use a LegacyGlobber that doesn't sort the matches for each individual glob pattern,
                // since we want to sort the final result anyway.
                LegacyGlobber legacyGlobber = packageFactory.createLegacyGlobberThatDoesntSort(buildFilePath.getParentDirectory(), packageId, packageLocator);
                SkyframeHybridGlobber skyframeGlobber = new SkyframeHybridGlobber(packageId, packageRoot, env, legacyGlobber);
                Preprocessor.Result preprocessingResult;
                if (replacementContents == null) {
                    Preconditions.checkNotNull(buildFileValue, packageId);
                    byte[] buildFileBytes;
                    try {
                        buildFileBytes = buildFileValue.isSpecialFile() ? FileSystemUtils.readContent(buildFilePath) : FileSystemUtils.readWithKnownFileSize(buildFilePath, buildFileValue.getSize());
                    } catch (IOException e) {
                        // transient.
                        throw new PackageFunctionException(new BuildFileContainsErrorsException(packageId, e.getMessage()), Transience.TRANSIENT);
                    }
                    try {
                        preprocessingResult = packageFactory.preprocess(buildFilePath, packageId, buildFileBytes, skyframeGlobber);
                    } catch (IOException e) {
                        throw new PackageFunctionException(new BuildFileContainsErrorsException(packageId, "preprocessing failed" + e.getMessage(), e), Transience.TRANSIENT);
                    }
                } else {
                    ParserInputSource replacementSource = ParserInputSource.create(replacementContents, buildFilePath.asFragment());
                    preprocessingResult = Preprocessor.Result.noPreprocessing(replacementSource);
                }
                StoredEventHandler astParsingEventHandler = new StoredEventHandler();
                BuildFileAST ast = PackageFactory.parseBuildFile(packageId, preprocessingResult.result, preludeStatements, astParsingEventHandler);
                // If no globs were fetched during preprocessing, then there's no need to reuse the
                // legacy globber instance during BUILD file evaluation since the performance argument
                // below does not apply.
                Set<SkyKey> globDepsRequested = skyframeGlobber.getGlobDepsRequested();
                LegacyGlobber legacyGlobberToStore = globDepsRequested.isEmpty() ? null : legacyGlobber;
                astCacheEntry = new CacheEntryWithGlobDeps<>(new AstAfterPreprocessing(preprocessingResult, ast, astParsingEventHandler), globDepsRequested, legacyGlobberToStore);
                astCache.put(packageId, astCacheEntry);
            }
            AstAfterPreprocessing astAfterPreprocessing = astCacheEntry.value;
            Set<SkyKey> globDepsRequestedDuringPreprocessing = astCacheEntry.globDepKeys;
            SkylarkImportResult importResult;
            try {
                importResult = fetchImportsFromBuildFile(buildFilePath, packageId, astAfterPreprocessing.ast, env, skylarkImportLookupFunctionForInlining);
            } catch (NoSuchPackageException e) {
                throw new PackageFunctionException(e, Transience.PERSISTENT);
            } catch (InterruptedException e) {
                astCache.invalidate(packageId);
                throw e;
            }
            if (importResult == null) {
                return null;
            }
            astCache.invalidate(packageId);
            // If a legacy globber was used to evaluate globs during preprocessing, it's important that
            // we reuse that globber during BUILD file evaluation for performance, in the case that
            // globs were fetched lazily during preprocessing. See Preprocessor.Factory#considersGlobs.
            LegacyGlobber legacyGlobber = astCacheEntry.legacyGlobber != null ? astCacheEntry.legacyGlobber : packageFactory.createLegacyGlobber(buildFilePath.getParentDirectory(), packageId, packageLocator);
            SkyframeHybridGlobber skyframeGlobber = new SkyframeHybridGlobber(packageId, packageRoot, env, legacyGlobber);
            Package.Builder pkgBuilder = packageFactory.createPackageFromPreprocessingAst(workspaceName, packageId, buildFilePath, astAfterPreprocessing, importResult.importMap, importResult.fileDependencies, defaultVisibility, skyframeGlobber);
            Set<SkyKey> globDepsRequested = ImmutableSet.<SkyKey>builder().addAll(globDepsRequestedDuringPreprocessing).addAll(skyframeGlobber.getGlobDepsRequested()).build();
            packageFunctionCacheEntry = new CacheEntryWithGlobDeps<>(pkgBuilder, globDepsRequested, null);
            numPackagesLoaded.incrementAndGet();
            if (packageProgress != null) {
                packageProgress.doneReadPackage(packageId);
            }
            packageFunctionCache.put(packageId, packageFunctionCacheEntry);
        } finally {
            profiler.completeTask(ProfilerTask.CREATE_PACKAGE);
        }
    }
    return packageFunctionCacheEntry;
}

Example 8 with BuildFileAST

use of com.google.devtools.build.lib.syntax.BuildFileAST in project bazel by bazelbuild.

the class WorkspaceASTFunction method splitAST.

/**
   * Cut {@code ast} into a list of AST separated by load statements. We cut right before each load
   * statement series.
   */
private static ImmutableList<BuildFileAST> splitAST(BuildFileAST ast) {
    ImmutableList.Builder<BuildFileAST> asts = ImmutableList.builder();
    int prevIdx = 0;
    // don't cut if the first statement is a load.
    boolean lastIsLoad = true;
    List<Statement> statements = ast.getStatements();
    for (int idx = 0; idx < statements.size(); idx++) {
        Statement st = statements.get(idx);
        if (st instanceof LoadStatement) {
            if (!lastIsLoad) {
                asts.add(ast.subTree(prevIdx, idx));
                prevIdx = idx;
            }
            lastIsLoad = true;
        } else {
            lastIsLoad = false;
        }
    }
    if (!statements.isEmpty()) {
        asts.add(ast.subTree(prevIdx, statements.size()));
    }
    return asts.build();
}

Example 9 with BuildFileAST

use of com.google.devtools.build.lib.syntax.BuildFileAST in project bazel by bazelbuild.

the class WorkspaceFileFunction method compute.

@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws WorkspaceFileFunctionException, InterruptedException {
    WorkspaceFileKey key = (WorkspaceFileKey) skyKey.argument();
    RootedPath workspaceRoot = key.getPath();
    WorkspaceASTValue workspaceASTValue = (WorkspaceASTValue) env.getValue(WorkspaceASTValue.key(workspaceRoot));
    if (workspaceASTValue == null) {
        return null;
    }
    Path repoWorkspace = workspaceRoot.getRoot().getRelative(workspaceRoot.getRelativePath());
    Package.Builder builder = packageFactory.newExternalPackageBuilder(repoWorkspace, ruleClassProvider.getRunfilesPrefix());
    if (workspaceASTValue.getASTs().isEmpty()) {
        return new WorkspaceFileValue(// resulting package
        builder.build(), // list of imports
        ImmutableMap.<String, Extension>of(), // list of symbol bindings
        ImmutableMap.<String, Object>of(), // Workspace root
        workspaceRoot, // first fragment, idx = 0
        0, // last fragment
        false);
    }
    WorkspaceFactory parser;
    try (Mutability mutability = Mutability.create("workspace %s", repoWorkspace)) {
        parser = new WorkspaceFactory(builder, ruleClassProvider, packageFactory.getEnvironmentExtensions(), mutability, key.getIndex() == 0, directories.getEmbeddedBinariesRoot(), directories.getWorkspace());
        if (key.getIndex() > 0) {
            WorkspaceFileValue prevValue = (WorkspaceFileValue) env.getValue(WorkspaceFileValue.key(key.getPath(), key.getIndex() - 1));
            if (prevValue == null) {
                return null;
            }
            if (prevValue.next() == null) {
                return prevValue;
            }
            parser.setParent(prevValue.getPackage(), prevValue.getImportMap(), prevValue.getBindings());
        }
        BuildFileAST ast = workspaceASTValue.getASTs().get(key.getIndex());
        PackageFunction.SkylarkImportResult importResult = PackageFunction.fetchImportsFromBuildFile(repoWorkspace, rootPackage, ast, env, null);
        if (importResult == null) {
            return null;
        }
        parser.execute(ast, importResult.importMap);
    } catch (NoSuchPackageException e) {
        throw new WorkspaceFileFunctionException(e, Transience.PERSISTENT);
    } catch (NameConflictException e) {
        throw new WorkspaceFileFunctionException(e, Transience.PERSISTENT);
    }
    return new WorkspaceFileValue(builder.build(), parser.getImportMap(), parser.getVariableBindings(), workspaceRoot, key.getIndex(), key.getIndex() < workspaceASTValue.getASTs().size() - 1);
}

Example 10 with BuildFileAST

use of com.google.devtools.build.lib.syntax.BuildFileAST in project bazel by bazelbuild.

the class SkylarkImportLookupFunction method computeInternal.

private SkyValue computeInternal(Label fileLabel, boolean inWorkspace, Environment env, @Nullable LinkedHashMap<Label, SkylarkImportLookupValue> alreadyVisited) throws InconsistentFilesystemException, SkylarkImportFailedException, InterruptedException {
    PathFragment filePath = fileLabel.toPathFragment();
    // Load the AST corresponding to this file.
    ASTFileLookupValue astLookupValue;
    try {
        SkyKey astLookupKey = ASTFileLookupValue.key(fileLabel);
        astLookupValue = (ASTFileLookupValue) env.getValueOrThrow(astLookupKey, ErrorReadingSkylarkExtensionException.class, InconsistentFilesystemException.class);
    } catch (ErrorReadingSkylarkExtensionException e) {
        throw SkylarkImportFailedException.errorReadingFile(filePath, e.getMessage());
    }
    if (astLookupValue == null) {
        return null;
    }
    if (!astLookupValue.lookupSuccessful()) {
        // Skylark import files have to exist.
        throw SkylarkImportFailedException.noFile(astLookupValue.getErrorMsg());
    }
    BuildFileAST ast = astLookupValue.getAST();
    if (ast.containsErrors()) {
        throw SkylarkImportFailedException.skylarkErrors(filePath);
    }
    // Process the load statements in the file.
    ImmutableList<SkylarkImport> imports = ast.getImports();
    Map<String, Extension> extensionsForImports = Maps.newHashMapWithExpectedSize(imports.size());
    ImmutableList.Builder<SkylarkFileDependency> fileDependencies = ImmutableList.builder();
    ImmutableMap<String, Label> labelsForImports;
    // Find the labels corresponding to the load statements.
    labelsForImports = findLabelsForLoadStatements(imports, fileLabel, env);
    if (labelsForImports == null) {
        return null;
    }
    // Look up and load the imports.
    ImmutableCollection<Label> importLabels = labelsForImports.values();
    List<SkyKey> importLookupKeys = Lists.newArrayListWithExpectedSize(importLabels.size());
    for (Label importLabel : importLabels) {
        importLookupKeys.add(SkylarkImportLookupValue.key(importLabel, inWorkspace));
    }
    Map<SkyKey, SkyValue> skylarkImportMap;
    boolean valuesMissing = false;
    if (alreadyVisited == null) {
        // Not inlining.
        skylarkImportMap = env.getValues(importLookupKeys);
        valuesMissing = env.valuesMissing();
    } else {
        // Inlining calls to SkylarkImportLookupFunction.
        if (alreadyVisited.containsKey(fileLabel)) {
            ImmutableList<Label> cycle = CycleUtils.splitIntoPathAndChain(Predicates.equalTo(fileLabel), alreadyVisited.keySet()).second;
            throw new SkylarkImportFailedException("Skylark import cycle: " + cycle);
        }
        alreadyVisited.put(fileLabel, null);
        skylarkImportMap = Maps.newHashMapWithExpectedSize(imports.size());
        for (SkyKey importLookupKey : importLookupKeys) {
            SkyValue skyValue = this.computeWithInlineCallsInternal(importLookupKey, env, alreadyVisited);
            if (skyValue == null) {
                Preconditions.checkState(env.valuesMissing(), "no skylark import value for %s", importLookupKey);
                // We continue making inline calls even if some requested values are missing, to maximize
                // the number of dependent (non-inlined) SkyFunctions that are requested, thus avoiding a
                // quadratic number of restarts.
                valuesMissing = true;
            } else {
                skylarkImportMap.put(importLookupKey, skyValue);
            }
        }
        // All imports traversed, this key can no longer be part of a cycle.
        Preconditions.checkState(alreadyVisited.remove(fileLabel) == null, fileLabel);
    }
    if (valuesMissing) {
        // This means some imports are unavailable.
        return null;
    }
    // Process the loaded imports.
    for (Entry<String, Label> importEntry : labelsForImports.entrySet()) {
        String importString = importEntry.getKey();
        Label importLabel = importEntry.getValue();
        SkyKey keyForLabel = SkylarkImportLookupValue.key(importLabel, inWorkspace);
        SkylarkImportLookupValue importLookupValue = (SkylarkImportLookupValue) skylarkImportMap.get(keyForLabel);
        extensionsForImports.put(importString, importLookupValue.getEnvironmentExtension());
        fileDependencies.add(importLookupValue.getDependency());
    }
    // Skylark UserDefinedFunction-s in that file will share this function definition Environment,
    // which will be frozen by the time it is returned by createExtension.
    Extension extension = createExtension(ast, fileLabel, extensionsForImports, env, inWorkspace);
    SkylarkImportLookupValue result = new SkylarkImportLookupValue(extension, new SkylarkFileDependency(fileLabel, fileDependencies.build()));
    if (alreadyVisited != null) {
        alreadyVisited.put(fileLabel, result);
    }
    return result;
}