Examples with FileValue com.google.devtools.build.lib.skyframe.FileValue used on opensource projects

Example 1 with FileValue

use of com.google.devtools.build.lib.skyframe.FileValue in project bazel by bazelbuild.

the class SkylarkRepositoryContext method getPathFromLabel.

// Resolve the label given by value into a file path.
private SkylarkPath getPathFromLabel(Label label) throws EvalException, InterruptedException {
    RootedPath rootedPath = getRootedPathFromLabel(label, env);
    SkyKey fileSkyKey = FileValue.key(rootedPath);
    FileValue fileValue = null;
    try {
        fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class, FileSymlinkException.class, InconsistentFilesystemException.class);
    } catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
        throw new EvalException(Location.BUILTIN, e);
    }
    if (fileValue == null) {
        throw SkylarkRepositoryFunction.restart();
    }
    if (!fileValue.isFile()) {
        throw new EvalException(Location.BUILTIN, "Not a file: " + rootedPath.asPath().getPathString());
    }
    // A label do not contains space so it safe to use as a key.
    markerData.put("FILE:" + label, Integer.toString(fileValue.realFileStateValue().hashCode()));
    return new SkylarkPath(rootedPath.asPath());
}

Example 2 with FileValue

use of com.google.devtools.build.lib.skyframe.FileValue in project bazel by bazelbuild.

the class NewLocalRepositoryFunction method fetch.

@Override
public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, Environment env, Map<String, String> markerData) throws SkyFunctionException, InterruptedException {
    NewRepositoryBuildFileHandler buildFileHandler = new NewRepositoryBuildFileHandler(directories.getWorkspace());
    if (!buildFileHandler.prepareBuildFile(rule, env)) {
        return null;
    }
    PathFragment pathFragment = getTargetPath(rule, directories.getWorkspace());
    FileSystem fs = directories.getOutputBase().getFileSystem();
    Path path = fs.getPath(pathFragment);
    RootedPath dirPath = RootedPath.toRootedPath(fs.getRootDirectory(), path);
    try {
        FileValue dirFileValue = (FileValue) env.getValueOrThrow(FileValue.key(dirPath), IOException.class, FileSymlinkException.class, InconsistentFilesystemException.class);
        if (dirFileValue == null) {
            return null;
        }
        if (!dirFileValue.exists()) {
            throw new RepositoryFunctionException(new IOException("Expected directory at " + dirPath.asPath().getPathString() + " but it does not exist."), Transience.PERSISTENT);
        }
        if (!dirFileValue.isDirectory()) {
            // Someone tried to create a local repository from a file.
            throw new RepositoryFunctionException(new IOException("Expected directory at " + dirPath.asPath().getPathString() + " but it is not a directory."), Transience.PERSISTENT);
        }
    } catch (IOException e) {
        throw new RepositoryFunctionException(e, Transience.PERSISTENT);
    } catch (FileSymlinkException e) {
        throw new RepositoryFunctionException(new IOException(e), Transience.PERSISTENT);
    } catch (InconsistentFilesystemException e) {
        throw new RepositoryFunctionException(new IOException(e), Transience.PERSISTENT);
    }
    // fetch() creates symlinks to each child under 'path' and DiffAwareness handles checking all
    // of these files and directories for changes. However, if a new file/directory is added
    // directly under 'path', Bazel doesn't know that this has to be symlinked in. Thus, this
    // creates a dependency on the contents of the 'path' directory.
    SkyKey dirKey = DirectoryListingValue.key(dirPath);
    DirectoryListingValue directoryValue;
    try {
        directoryValue = (DirectoryListingValue) env.getValueOrThrow(dirKey, InconsistentFilesystemException.class);
    } catch (InconsistentFilesystemException e) {
        throw new RepositoryFunctionException(new IOException(e), Transience.PERSISTENT);
    }
    if (directoryValue == null) {
        return null;
    }
    // Link x/y/z to /some/path/to/y/z.
    if (!symlinkLocalRepositoryContents(outputDirectory, path)) {
        return null;
    }
    buildFileHandler.finishBuildFile(outputDirectory);
    // If someone specified *new*_local_repository, we can assume they didn't want the existing
    // repository info.
    Path workspaceFile = outputDirectory.getRelative("WORKSPACE");
    if (workspaceFile.exists()) {
        try {
            workspaceFile.delete();
        } catch (IOException e) {
            throw new RepositoryFunctionException(e, Transience.TRANSIENT);
        }
    }
    createWorkspaceFile(outputDirectory, rule.getTargetKind(), rule.getName());
    return RepositoryDirectoryValue.builder().setPath(outputDirectory).setSourceDir(directoryValue);
}

Example 3 with FileValue

use of com.google.devtools.build.lib.skyframe.FileValue in project bazel by bazelbuild.

the class NewRepositoryBuildFileHandler method getBuildFileValue.

private FileValue getBuildFileValue(Rule rule, Environment env) throws RepositoryFunctionException, InterruptedException {
    WorkspaceAttributeMapper mapper = WorkspaceAttributeMapper.of(rule);
    String buildFileAttribute;
    try {
        buildFileAttribute = mapper.get("build_file", Type.STRING);
    } catch (EvalException e) {
        throw new RepositoryFunctionException(e, Transience.PERSISTENT);
    }
    RootedPath rootedBuild;
    if (LabelValidator.isAbsolute(buildFileAttribute)) {
        try {
            // Parse a label
            Label label = Label.parseAbsolute(buildFileAttribute);
            SkyKey pkgSkyKey = PackageLookupValue.key(label.getPackageIdentifier());
            PackageLookupValue pkgLookupValue = (PackageLookupValue) env.getValue(pkgSkyKey);
            if (pkgLookupValue == null) {
                return null;
            }
            if (!pkgLookupValue.packageExists()) {
                throw new RepositoryFunctionException(new EvalException(rule.getLocation(), "Unable to load package for " + buildFileAttribute + ": not found."), Transience.PERSISTENT);
            }
            // And now for the file
            Path packageRoot = pkgLookupValue.getRoot();
            rootedBuild = RootedPath.toRootedPath(packageRoot, label.toPathFragment());
        } catch (LabelSyntaxException ex) {
            throw new RepositoryFunctionException(new EvalException(rule.getLocation(), String.format("In %s the 'build_file' attribute does not specify a valid label: %s", rule, ex.getMessage())), Transience.PERSISTENT);
        }
    } else {
        // TODO(dmarting): deprecate using a path for the build_file attribute.
        PathFragment buildFile = new PathFragment(buildFileAttribute);
        Path buildFileTarget = workspacePath.getRelative(buildFile);
        if (!buildFileTarget.exists()) {
            throw new RepositoryFunctionException(new EvalException(rule.getLocation(), String.format("In %s the 'build_file' attribute does not specify an existing file " + "(%s does not exist)", rule, buildFileTarget)), Transience.PERSISTENT);
        }
        if (buildFile.isAbsolute()) {
            rootedBuild = RootedPath.toRootedPath(buildFileTarget.getParentDirectory(), new PathFragment(buildFileTarget.getBaseName()));
        } else {
            rootedBuild = RootedPath.toRootedPath(workspacePath, buildFile);
        }
    }
    SkyKey buildFileKey = FileValue.key(rootedBuild);
    FileValue buildFileValue;
    try {
        // Note that this dependency is, strictly speaking, not necessary: the symlink could simply
        // point to this FileValue and the symlink chasing could be done while loading the package
        // but this results in a nicer error message and it's correct as long as RepositoryFunctions
        // don't write to things in the file system this FileValue depends on. In theory, the latter
        // is possible if the file referenced by build_file is a symlink to somewhere under the
        // external/ directory, but if you do that, you are really asking for trouble.
        buildFileValue = (FileValue) env.getValueOrThrow(buildFileKey, IOException.class, FileSymlinkException.class, InconsistentFilesystemException.class);
        if (buildFileValue == null) {
            return null;
        }
    } catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
        throw new RepositoryFunctionException(new IOException("Cannot lookup " + buildFileAttribute + ": " + e.getMessage()), Transience.TRANSIENT);
    }
    return buildFileValue;
}

Example 4 with FileValue

use of com.google.devtools.build.lib.skyframe.FileValue in project bazel by bazelbuild.

the class RepositoryDelegatorFunction method compute.

@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws SkyFunctionException, InterruptedException {
    RepositoryName repositoryName = (RepositoryName) skyKey.argument();
    Rule rule = RepositoryFunction.getRule(repositoryName, null, env);
    if (rule == null) {
        return null;
    }
    BlazeDirectories directories = PrecomputedValue.BLAZE_DIRECTORIES.get(env);
    if (directories == null) {
        return null;
    }
    RepositoryFunction handler;
    if (rule.getRuleClassObject().isSkylark()) {
        handler = skylarkHandler;
    } else {
        handler = handlers.get(rule.getRuleClass());
    }
    if (handler == null) {
        throw new RepositoryFunctionException(new EvalException(Location.fromFile(directories.getWorkspace().getRelative("WORKSPACE")), "Could not find handler for " + rule), Transience.PERSISTENT);
    }
    handler.setClientEnvironment(clientEnvironment);
    Path repoRoot = RepositoryFunction.getExternalRepositoryDirectory(directories).getRelative(rule.getName());
    byte[] ruleSpecificData = handler.getRuleSpecificMarkerData(rule, env);
    if (ruleSpecificData == null) {
        return null;
    }
    String ruleKey = computeRuleKey(rule, ruleSpecificData);
    Map<String, String> markerData = new TreeMap<>();
    Path markerPath = getMarkerPath(directories, rule);
    if (handler.isLocal(rule)) {
        // Local repositories are always fetched because the operation is generally fast and they do
        // not depend on non-local data, so it does not make much sense to try to cache from across
        // server instances.
        setupRepositoryRoot(repoRoot);
        RepositoryDirectoryValue.Builder localRepo = handler.fetch(rule, repoRoot, directories, env, markerData);
        if (localRepo == null) {
            return null;
        } else {
            // We write the marker file for local repository essentially for getting the digest and
            // injecting it in the RepositoryDirectoryValue.
            byte[] digest = writeMarkerFile(markerPath, markerData, ruleKey);
            return localRepo.setDigest(digest).build();
        }
    }
    // We check the repository root for existence here, but we can't depend on the FileValue,
    // because it's possible that we eventually create that directory in which case the FileValue
    // and the state of the file system would be inconsistent.
    byte[] markerHash = isFilesystemUpToDate(markerPath, rule, ruleKey, handler, env);
    if (env.valuesMissing()) {
        return null;
    }
    if (markerHash != null && repoRoot.exists()) {
        // Now that we know that it exists, we can declare a Skyframe dependency on the repository
        // root.
        RepositoryFunction.getRepositoryDirectory(repoRoot, env);
        if (env.valuesMissing()) {
            return null;
        }
        return RepositoryDirectoryValue.builder().setPath(repoRoot).setDigest(markerHash).build();
    }
    if (isFetch.get()) {
        // Fetching enabled, go ahead.
        setupRepositoryRoot(repoRoot);
        RepositoryDirectoryValue.Builder result = handler.fetch(rule, repoRoot, directories, env, markerData);
        if (env.valuesMissing()) {
            return null;
        }
        // No new Skyframe dependencies must be added between calling the repository implementation
        // and writing the marker file because if they aren't computed, it would cause a Skyframe
        // restart thus calling the possibly very slow (networking, decompression...) fetch()
        // operation again. So we write the marker file here immediately.
        byte[] digest = writeMarkerFile(markerPath, markerData, ruleKey);
        return result.setDigest(digest).build();
    }
    if (!repoRoot.exists()) {
        // The repository isn't on the file system, there is nothing we can do.
        throw new RepositoryFunctionException(new IOException("to fix, run\n\tbazel fetch //...\nExternal repository " + repositoryName + " not found and fetching repositories is disabled."), Transience.TRANSIENT);
    }
    // Declare a Skyframe dependency so that this is re-evaluated when something happens to the
    // directory.
    FileValue repoRootValue = RepositoryFunction.getRepositoryDirectory(repoRoot, env);
    if (env.valuesMissing()) {
        return null;
    }
    // Try to build with whatever is on the file system and emit a warning.
    env.getListener().handle(Event.warn(rule.getLocation(), String.format("External repository '%s' is not up-to-date and fetching is disabled. To update, " + "run the build without the '--nofetch' command line option.", rule.getName())));
    return RepositoryDirectoryValue.builder().setPath(repoRootValue.realRootedPath().asPath()).setFetchingDelayed().build();
}

Example 5 with FileValue

use of com.google.devtools.build.lib.skyframe.FileValue in project bazel by bazelbuild.

the class MavenServerFunction method compute.

@Nullable
@Override
public SkyValue compute(SkyKey skyKey, Environment env) throws InterruptedException, RepositoryFunctionException {
    String repository = (String) skyKey.argument();
    Rule repositoryRule = null;
    try {
        repositoryRule = RepositoryFunction.getRule(repository, env);
    } catch (RepositoryNotFoundException ex) {
    // Ignored. We throw a new one below.
    }
    BlazeDirectories directories = PrecomputedValue.BLAZE_DIRECTORIES.get(env);
    if (env.valuesMissing()) {
        return null;
    }
    String serverName;
    String url;
    Map<String, FileValue> settingsFiles;
    boolean foundRepoRule = repositoryRule != null && repositoryRule.getRuleClass().equals(MavenServerRule.NAME);
    if (!foundRepoRule) {
        if (repository.equals(MavenServerValue.DEFAULT_ID)) {
            settingsFiles = getDefaultSettingsFile(directories, env);
            serverName = MavenServerValue.DEFAULT_ID;
            url = MavenConnector.getMavenCentralRemote().getUrl();
        } else {
            throw new RepositoryFunctionException(new IOException("Could not find maven repository " + repository), Transience.TRANSIENT);
        }
    } else {
        WorkspaceAttributeMapper mapper = WorkspaceAttributeMapper.of(repositoryRule);
        serverName = repositoryRule.getName();
        try {
            url = mapper.get("url", Type.STRING);
            if (!mapper.isAttributeValueExplicitlySpecified("settings_file")) {
                settingsFiles = getDefaultSettingsFile(directories, env);
            } else {
                PathFragment settingsFilePath = new PathFragment(mapper.get("settings_file", Type.STRING));
                RootedPath settingsPath = RootedPath.toRootedPath(directories.getWorkspace().getRelative(settingsFilePath), PathFragment.EMPTY_FRAGMENT);
                FileValue fileValue = (FileValue) env.getValue(FileValue.key(settingsPath));
                if (fileValue == null) {
                    return null;
                }
                if (!fileValue.exists()) {
                    throw new RepositoryFunctionException(new IOException("Could not find settings file " + settingsPath), Transience.TRANSIENT);
                }
                settingsFiles = ImmutableMap.<String, FileValue>builder().put(USER_KEY, fileValue).build();
            }
        } catch (EvalException e) {
            throw new RepositoryFunctionException(e, Transience.PERSISTENT);
        }
    }
    if (settingsFiles == null) {
        return null;
    }
    Fingerprint fingerprint = new Fingerprint();
    try {
        for (Map.Entry<String, FileValue> entry : settingsFiles.entrySet()) {
            fingerprint.addString(entry.getKey());
            Path path = entry.getValue().realRootedPath().asPath();
            if (path.exists()) {
                fingerprint.addBoolean(true);
                fingerprint.addBytes(path.getDigest());
            } else {
                fingerprint.addBoolean(false);
            }
        }
    } catch (IOException e) {
        throw new RepositoryFunctionException(e, Transience.TRANSIENT);
    }
    byte[] fingerprintBytes = fingerprint.digestAndReset();
    if (settingsFiles.isEmpty()) {
        return new MavenServerValue(serverName, url, new Server(), fingerprintBytes);
    }
    DefaultSettingsBuildingRequest request = new DefaultSettingsBuildingRequest();
    if (settingsFiles.containsKey(SYSTEM_KEY)) {
        request.setGlobalSettingsFile(settingsFiles.get(SYSTEM_KEY).realRootedPath().asPath().getPathFile());
    }
    if (settingsFiles.containsKey(USER_KEY)) {
        request.setUserSettingsFile(settingsFiles.get(USER_KEY).realRootedPath().asPath().getPathFile());
    }
    DefaultSettingsBuilder builder = (new DefaultSettingsBuilderFactory()).newInstance();
    SettingsBuildingResult result;
    try {
        result = builder.build(request);
    } catch (SettingsBuildingException e) {
        throw new RepositoryFunctionException(new IOException("Error parsing settings files: " + e.getMessage()), Transience.TRANSIENT);
    }
    if (!result.getProblems().isEmpty()) {
        throw new RepositoryFunctionException(new IOException("Errors interpreting settings file: " + Arrays.toString(result.getProblems().toArray())), Transience.PERSISTENT);
    }
    Settings settings = result.getEffectiveSettings();
    Server server = settings.getServer(serverName);
    server = server == null ? new Server() : server;
    return new MavenServerValue(serverName, url, server, fingerprintBytes);
}