Examples with DirectoryListingValue com.google.devtools.build.lib.skyframe.DirectoryListingValue used on opensource projects

Example 1 with DirectoryListingValue

use of com.google.devtools.build.lib.skyframe.DirectoryListingValue in project bazel by bazelbuild.

the class AndroidNdkRepositoryFunction method fetch.

@Override
public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, Environment env, Map<String, String> markerData) throws InterruptedException, RepositoryFunctionException {
    Map<String, String> environ = declareEnvironmentDependencies(markerData, env, PATH_ENV_VAR_AS_LIST);
    if (environ == null) {
        return null;
    }
    prepareLocalRepositorySymlinkTree(rule, outputDirectory);
    WorkspaceAttributeMapper attributes = WorkspaceAttributeMapper.of(rule);
    PathFragment pathFragment;
    if (attributes.isAttributeValueExplicitlySpecified("path")) {
        pathFragment = getTargetPath(rule, directories.getWorkspace());
    } else if (environ.get(PATH_ENV_VAR) != null) {
        pathFragment = getAndroidNdkHomeEnvironmentVar(directories.getWorkspace(), environ);
    } else {
        throw new RepositoryFunctionException(new EvalException(rule.getLocation(), "Either the path attribute of android_ndk_repository or the ANDROID_NDK_HOME " + " environment variable must be set."), Transience.PERSISTENT);
    }
    Path ndkSymlinkTreeDirectory = outputDirectory.getRelative("ndk");
    try {
        ndkSymlinkTreeDirectory.createDirectory();
    } catch (IOException e) {
        throw new RepositoryFunctionException(e, Transience.TRANSIENT);
    }
    Path ndkHome = directories.getOutputBase().getFileSystem().getPath(pathFragment);
    if (!symlinkLocalRepositoryContents(ndkSymlinkTreeDirectory, ndkHome)) {
        return null;
    }
    String ruleName = rule.getName();
    // We need to fetch the NDK release info from the actual home to avoid cycle in the
    // dependency graph (the path relative to the repository root depends on the
    // repository being fetched).
    NdkRelease ndkRelease = getNdkRelease(ndkHome, env);
    if (env.valuesMissing()) {
        return null;
    }
    String apiLevelString;
    if (attributes.isAttributeValueExplicitlySpecified("api_level")) {
        try {
            apiLevelString = attributes.get("api_level", Type.INTEGER).toString();
        } catch (EvalException e) {
            throw new RepositoryFunctionException(e, Transience.PERSISTENT);
        }
    } else {
        DirectoryListingValue platformsDirectoryValue = AndroidRepositoryUtils.getDirectoryListing(ndkHome, PLATFORMS_DIR, env);
        if (platformsDirectoryValue == null) {
            return null;
        }
        ImmutableSortedSet<Integer> apiLevels = AndroidRepositoryUtils.getApiLevels(platformsDirectoryValue.getDirents());
        if (apiLevels.isEmpty()) {
            // themselves.
            throw new RepositoryFunctionException(new EvalException(rule.getLocation(), "android_ndk_repository requires that at least one Android platform is present in " + "the Android NDK platforms directory. Please ensure that the path attribute " + "or the ANDROID_NDK_HOME environment variable points to a valid NDK."), Transience.PERSISTENT);
        }
        apiLevelString = apiLevels.first().toString();
    }
    // NDK minor revisions should be backwards compatible within a major revision, the crosstools
    // we generate don't care about the minor revision.
    NdkMajorRevision ndkMajorRevision;
    if (!ndkRelease.isValid) {
        String warningMessage = String.format("The revision of the Android NDK referenced by android_ndk_repository rule '%s' " + "could not be determined (the revision string found is '%s'). Defaulting to " + "revision %s.", ruleName, ndkRelease.rawRelease, AndroidNdkCrosstools.LATEST_KNOWN_REVISION.getKey());
        env.getListener().handle(Event.warn(warningMessage));
        ndkMajorRevision = AndroidNdkCrosstools.LATEST_KNOWN_REVISION.getValue();
    } else if (!AndroidNdkCrosstools.isKnownNDKRevision(ndkRelease)) {
        String warningMessage = String.format("The major revision of the Android NDK referenced by android_ndk_repository rule " + "'%s' is %s. The major revisions supported by Bazel are %s. Defaulting to " + "revision %s.", ruleName, ndkRelease.majorRevision, AndroidNdkCrosstools.KNOWN_NDK_MAJOR_REVISIONS.keySet(), AndroidNdkCrosstools.LATEST_KNOWN_REVISION.getKey());
        env.getListener().handle(Event.warn(warningMessage));
        ndkMajorRevision = AndroidNdkCrosstools.LATEST_KNOWN_REVISION.getValue();
    } else {
        ndkMajorRevision = AndroidNdkCrosstools.KNOWN_NDK_MAJOR_REVISIONS.get(ndkRelease.majorRevision);
    }
    ApiLevel apiLevel = ndkMajorRevision.apiLevel(env.getListener(), ruleName, apiLevelString);
    ImmutableList.Builder<CrosstoolStlPair> crosstoolsAndStls = ImmutableList.builder();
    try {
        String hostPlatform = AndroidNdkCrosstools.getHostPlatform(ndkRelease);
        NdkPaths ndkPaths = new NdkPaths(ruleName, hostPlatform, apiLevel);
        for (StlImpl stlImpl : StlImpls.get(ndkPaths)) {
            CrosstoolRelease crosstoolRelease = ndkMajorRevision.crosstoolRelease(ndkPaths, stlImpl, hostPlatform);
            crosstoolsAndStls.add(new CrosstoolStlPair(crosstoolRelease, stlImpl));
        }
    } catch (NdkCrosstoolsException e) {
        throw new RepositoryFunctionException(new IOException(e), Transience.PERSISTENT);
    }
    String buildFile = createBuildFile(ruleName, crosstoolsAndStls.build());
    writeBuildFile(outputDirectory, buildFile);
    return RepositoryDirectoryValue.builder().setPath(outputDirectory);
}

Example 2 with DirectoryListingValue

use of com.google.devtools.build.lib.skyframe.DirectoryListingValue in project bazel by bazelbuild.

the class AndroidSdkRepositoryFunction method getAndroidDeviceSystemImageDirs.

/**
   * Gets PathFragments for /sdk/system-images/*/*/*, which are the directories in the
   * SDK that contain system images needed for android_device.
   *
   * If the sdk/system-images directory does not exist, an empty set is returned.
   */
private static ImmutableSortedSet<PathFragment> getAndroidDeviceSystemImageDirs(Path androidSdkPath, Environment env) throws RepositoryFunctionException, InterruptedException {
    if (!androidSdkPath.getRelative(SYSTEM_IMAGES_DIR).exists()) {
        return ImmutableSortedSet.of();
    }
    DirectoryListingValue systemImagesDirectoryValue = AndroidRepositoryUtils.getDirectoryListing(androidSdkPath, SYSTEM_IMAGES_DIR, env);
    if (systemImagesDirectoryValue == null) {
        return null;
    }
    ImmutableMap<PathFragment, DirectoryListingValue> apiLevelSystemImageDirs = getSubdirectoryListingValues(androidSdkPath, SYSTEM_IMAGES_DIR, systemImagesDirectoryValue, env);
    if (apiLevelSystemImageDirs == null) {
        return null;
    }
    ImmutableSortedSet.Builder<PathFragment> pathFragments = ImmutableSortedSet.naturalOrder();
    for (PathFragment apiLevelDir : apiLevelSystemImageDirs.keySet()) {
        ImmutableMap<PathFragment, DirectoryListingValue> apiTypeSystemImageDirs = getSubdirectoryListingValues(androidSdkPath, apiLevelDir, apiLevelSystemImageDirs.get(apiLevelDir), env);
        if (apiTypeSystemImageDirs == null) {
            return null;
        }
        for (PathFragment apiTypeDir : apiTypeSystemImageDirs.keySet()) {
            for (Dirent architectureSystemImageDir : apiTypeSystemImageDirs.get(apiTypeDir).getDirents()) {
                pathFragments.add(apiTypeDir.getRelative(architectureSystemImageDir.getName()));
            }
        }
    }
    return pathFragments.build();
}

Example 3 with DirectoryListingValue

use of com.google.devtools.build.lib.skyframe.DirectoryListingValue in project bazel by bazelbuild.

the class NewLocalRepositoryFunction method fetch.

@Override
public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, Environment env, Map<String, String> markerData) throws SkyFunctionException, InterruptedException {
    NewRepositoryBuildFileHandler buildFileHandler = new NewRepositoryBuildFileHandler(directories.getWorkspace());
    if (!buildFileHandler.prepareBuildFile(rule, env)) {
        return null;
    }
    PathFragment pathFragment = getTargetPath(rule, directories.getWorkspace());
    FileSystem fs = directories.getOutputBase().getFileSystem();
    Path path = fs.getPath(pathFragment);
    RootedPath dirPath = RootedPath.toRootedPath(fs.getRootDirectory(), path);
    try {
        FileValue dirFileValue = (FileValue) env.getValueOrThrow(FileValue.key(dirPath), IOException.class, FileSymlinkException.class, InconsistentFilesystemException.class);
        if (dirFileValue == null) {
            return null;
        }
        if (!dirFileValue.exists()) {
            throw new RepositoryFunctionException(new IOException("Expected directory at " + dirPath.asPath().getPathString() + " but it does not exist."), Transience.PERSISTENT);
        }
        if (!dirFileValue.isDirectory()) {
            // Someone tried to create a local repository from a file.
            throw new RepositoryFunctionException(new IOException("Expected directory at " + dirPath.asPath().getPathString() + " but it is not a directory."), Transience.PERSISTENT);
        }
    } catch (IOException e) {
        throw new RepositoryFunctionException(e, Transience.PERSISTENT);
    } catch (FileSymlinkException e) {
        throw new RepositoryFunctionException(new IOException(e), Transience.PERSISTENT);
    } catch (InconsistentFilesystemException e) {
        throw new RepositoryFunctionException(new IOException(e), Transience.PERSISTENT);
    }
    // fetch() creates symlinks to each child under 'path' and DiffAwareness handles checking all
    // of these files and directories for changes. However, if a new file/directory is added
    // directly under 'path', Bazel doesn't know that this has to be symlinked in. Thus, this
    // creates a dependency on the contents of the 'path' directory.
    SkyKey dirKey = DirectoryListingValue.key(dirPath);
    DirectoryListingValue directoryValue;
    try {
        directoryValue = (DirectoryListingValue) env.getValueOrThrow(dirKey, InconsistentFilesystemException.class);
    } catch (InconsistentFilesystemException e) {
        throw new RepositoryFunctionException(new IOException(e), Transience.PERSISTENT);
    }
    if (directoryValue == null) {
        return null;
    }
    // Link x/y/z to /some/path/to/y/z.
    if (!symlinkLocalRepositoryContents(outputDirectory, path)) {
        return null;
    }
    buildFileHandler.finishBuildFile(outputDirectory);
    // If someone specified *new*_local_repository, we can assume they didn't want the existing
    // repository info.
    Path workspaceFile = outputDirectory.getRelative("WORKSPACE");
    if (workspaceFile.exists()) {
        try {
            workspaceFile.delete();
        } catch (IOException e) {
            throw new RepositoryFunctionException(e, Transience.TRANSIENT);
        }
    }
    createWorkspaceFile(outputDirectory, rule.getTargetKind(), rule.getName());
    return RepositoryDirectoryValue.builder().setPath(outputDirectory).setSourceDir(directoryValue);
}

Example 4 with DirectoryListingValue

use of com.google.devtools.build.lib.skyframe.DirectoryListingValue in project bazel by bazelbuild.

the class AndroidSdkRepositoryFunction method fetch.

@Override
public RepositoryDirectoryValue.Builder fetch(Rule rule, Path outputDirectory, BlazeDirectories directories, Environment env, Map<String, String> markerData) throws SkyFunctionException, InterruptedException {
    Map<String, String> environ = declareEnvironmentDependencies(markerData, env, PATH_ENV_VAR_AS_LIST);
    if (environ == null) {
        return null;
    }
    prepareLocalRepositorySymlinkTree(rule, outputDirectory);
    WorkspaceAttributeMapper attributes = WorkspaceAttributeMapper.of(rule);
    FileSystem fs = directories.getOutputBase().getFileSystem();
    Path androidSdkPath;
    if (attributes.isAttributeValueExplicitlySpecified("path")) {
        androidSdkPath = fs.getPath(getTargetPath(rule, directories.getWorkspace()));
    } else if (environ.get(PATH_ENV_VAR) != null) {
        androidSdkPath = fs.getPath(getAndroidHomeEnvironmentVar(directories.getWorkspace(), environ));
    } else {
        throw new RepositoryFunctionException(new EvalException(rule.getLocation(), "Either the path attribute of android_sdk_repository or the ANDROID_HOME environment " + " variable must be set."), Transience.PERSISTENT);
    }
    if (!symlinkLocalRepositoryContents(outputDirectory, androidSdkPath)) {
        return null;
    }
    DirectoryListingValue platformsDirectoryValue = AndroidRepositoryUtils.getDirectoryListing(androidSdkPath, PLATFORMS_DIR, env);
    if (platformsDirectoryValue == null) {
        return null;
    }
    ImmutableSortedSet<Integer> apiLevels = AndroidRepositoryUtils.getApiLevels(platformsDirectoryValue.getDirents());
    if (apiLevels.isEmpty()) {
        throw new RepositoryFunctionException(new EvalException(rule.getLocation(), "android_sdk_repository requires that at least one Android SDK Platform is installed " + "in the Android SDK. Please install an Android SDK Platform through the " + "Android SDK manager."), Transience.PERSISTENT);
    }
    Integer defaultApiLevel;
    if (attributes.isAttributeValueExplicitlySpecified("api_level")) {
        try {
            defaultApiLevel = attributes.get("api_level", Type.INTEGER);
        } catch (EvalException e) {
            throw new RepositoryFunctionException(e, Transience.PERSISTENT);
        }
        if (!apiLevels.contains(defaultApiLevel)) {
            throw new RepositoryFunctionException(new EvalException(rule.getLocation(), String.format("Android SDK api level %s was requested but it is not installed in the Android " + "SDK at %s. The api levels found were %s. Please choose an available api " + "level or install api level %s from the Android SDK Manager.", defaultApiLevel, androidSdkPath, apiLevels.toString(), defaultApiLevel)), Transience.PERSISTENT);
        }
    } else {
        // If the api_level attribute is not explicitly set, we select the highest api level that is
        // available in the SDK.
        defaultApiLevel = apiLevels.first();
    }
    String buildToolsDirectory;
    if (attributes.isAttributeValueExplicitlySpecified("build_tools_version")) {
        try {
            buildToolsDirectory = attributes.get("build_tools_version", Type.STRING);
        } catch (EvalException e) {
            throw new RepositoryFunctionException(e, Transience.PERSISTENT);
        }
    } else {
        // If the build_tools_version attribute is not explicitly set, we select the highest version
        // installed in the SDK.
        DirectoryListingValue directoryValue = AndroidRepositoryUtils.getDirectoryListing(androidSdkPath, BUILD_TOOLS_DIR, env);
        if (directoryValue == null) {
            return null;
        }
        buildToolsDirectory = getNewestBuildToolsDirectory(rule, directoryValue.getDirents());
    }
    // android_sdk_repository.build_tools_version is technically actually the name of the
    // directory in $sdk/build-tools. Most of the time this is just the actual build tools
    // version, but for preview build tools, the directory is something like 24.0.0-preview, and
    // the actual version is something like "24 rc3". The android_sdk rule in the template needs
    // the real version.
    String buildToolsVersion;
    if (buildToolsDirectory.contains("-preview")) {
        Properties sourceProperties = getBuildToolsSourceProperties(outputDirectory, buildToolsDirectory, env);
        if (env.valuesMissing()) {
            return null;
        }
        buildToolsVersion = sourceProperties.getProperty("Pkg.Revision");
    } else {
        buildToolsVersion = buildToolsDirectory;
    }
    try {
        assertValidBuildToolsVersion(rule, buildToolsVersion);
    } catch (EvalException e) {
        throw new RepositoryFunctionException(e, Transience.PERSISTENT);
    }
    ImmutableSortedSet<PathFragment> androidDeviceSystemImageDirs = getAndroidDeviceSystemImageDirs(androidSdkPath, env);
    if (androidDeviceSystemImageDirs == null) {
        return null;
    }
    StringBuilder systemImageDirsList = new StringBuilder();
    for (PathFragment systemImageDir : androidDeviceSystemImageDirs) {
        systemImageDirsList.append(String.format("        \"%s\",\n", systemImageDir));
    }
    String template = getStringResource("android_sdk_repository_template.txt");
    String buildFile = template.replace("%repository_name%", rule.getName()).replace("%build_tools_version%", buildToolsVersion).replace("%build_tools_directory%", buildToolsDirectory).replace("%api_levels%", Iterables.toString(apiLevels)).replace("%default_api_level%", String.valueOf(defaultApiLevel)).replace("%system_image_dirs%", systemImageDirsList);
    // All local maven repositories that are shipped in the Android SDK.
    // TODO(ajmichael): Create SkyKeys so that if the SDK changes, this function will get rerun.
    Iterable<Path> localMavenRepositories = ImmutableList.of(outputDirectory.getRelative("extras/android/m2repository"), outputDirectory.getRelative("extras/google/m2repository"));
    try {
        SdkMavenRepository sdkExtrasRepository = SdkMavenRepository.create(Iterables.filter(localMavenRepositories, new Predicate<Path>() {

            @Override
            public boolean apply(@Nullable Path path) {
                return path.isDirectory();
            }
        }));
        sdkExtrasRepository.writeBuildFiles(outputDirectory);
        buildFile = buildFile.replace("%exported_files%", sdkExtrasRepository.getExportsFiles(outputDirectory));
    } catch (IOException e) {
        throw new RepositoryFunctionException(e, Transience.TRANSIENT);
    }
    writeBuildFile(outputDirectory, buildFile);
    return RepositoryDirectoryValue.builder().setPath(outputDirectory);
}

Example 5 with DirectoryListingValue

use of com.google.devtools.build.lib.skyframe.DirectoryListingValue in project bazel by bazelbuild.

the class AndroidSdkRepositoryFunction method getSubdirectoryListingValues.

/** Gets DirectoryListingValues for subdirectories of the directory or returns null. */
private static ImmutableMap<PathFragment, DirectoryListingValue> getSubdirectoryListingValues(final Path root, final PathFragment path, DirectoryListingValue directory, Environment env) throws RepositoryFunctionException, InterruptedException {
    Map<PathFragment, SkyKey> skyKeysForSubdirectoryLookups = Maps.transformEntries(Maps.uniqueIndex(directory.getDirents(), new Function<Dirent, PathFragment>() {

        @Override
        public PathFragment apply(Dirent input) {
            return path.getRelative(input.getName());
        }
    }), new EntryTransformer<PathFragment, Dirent, SkyKey>() {

        @Override
        public SkyKey transformEntry(PathFragment key, Dirent value) {
            return DirectoryListingValue.key(RootedPath.toRootedPath(root, root.getRelative(key)));
        }
    });
    Map<SkyKey, ValueOrException<InconsistentFilesystemException>> values = env.getValuesOrThrow(skyKeysForSubdirectoryLookups.values(), InconsistentFilesystemException.class);
    ImmutableMap.Builder<PathFragment, DirectoryListingValue> directoryListingValues = new ImmutableMap.Builder<>();
    for (PathFragment pathFragment : skyKeysForSubdirectoryLookups.keySet()) {
        try {
            SkyValue skyValue = values.get(skyKeysForSubdirectoryLookups.get(pathFragment)).get();
            if (skyValue == null) {
                return null;
            }
            directoryListingValues.put(pathFragment, (DirectoryListingValue) skyValue);
        } catch (InconsistentFilesystemException e) {
            throw new RepositoryFunctionException(new IOException(e), Transience.PERSISTENT);
        }
    }
    return directoryListingValues.build();
}