Examples with SourcePath com.facebook.buck.rules.SourcePath used on opensource projects

Example 1 with SourcePath

use of com.facebook.buck.rules.SourcePath in project buck by facebook.

the class AndroidAar method getBuildSteps.

@Override
public ImmutableList<Step> getBuildSteps(BuildContext context, BuildableContext buildableContext) {
    ImmutableList.Builder<Step> commands = ImmutableList.builder();
    // Create temp folder to store the files going to be zipped
    commands.add(new MakeCleanDirectoryStep(getProjectFilesystem(), temp));
    // Remove the output .aar file
    commands.add(new RmStep(getProjectFilesystem(), pathToOutputFile));
    // put manifest into tmp folder
    commands.add(CopyStep.forFile(getProjectFilesystem(), context.getSourcePathResolver().getRelativePath(manifest.getSourcePathToOutput()), temp.resolve("AndroidManifest.xml")));
    // put R.txt into tmp folder
    commands.add(CopyStep.forFile(getProjectFilesystem(), context.getSourcePathResolver().getAbsolutePath(Preconditions.checkNotNull(androidResource.getPathToTextSymbolsFile())), temp.resolve("R.txt")));
    // put res/ and assets/ into tmp folder
    commands.add(CopyStep.forDirectory(getProjectFilesystem(), context.getSourcePathResolver().getRelativePath(assembledResourceDirectory), temp.resolve("res"), CopyStep.DirectoryMode.CONTENTS_ONLY));
    commands.add(CopyStep.forDirectory(getProjectFilesystem(), context.getSourcePathResolver().getRelativePath(assembledAssetsDirectory), temp.resolve("assets"), CopyStep.DirectoryMode.CONTENTS_ONLY));
    // Create our Uber-jar, and place it in the tmp folder.
    commands.add(new JarDirectoryStep(getProjectFilesystem(), temp.resolve("classes.jar"), context.getSourcePathResolver().getAllAbsolutePaths(classpathsToIncludeInJar), /* mainClass */
    null, /* manifestFile */
    null));
    // move native libs into tmp folder under jni/
    if (assembledNativeLibs.isPresent()) {
        commands.add(CopyStep.forDirectory(getProjectFilesystem(), assembledNativeLibs.get(), temp.resolve("jni"), CopyStep.DirectoryMode.CONTENTS_ONLY));
    }
    // move native assets into tmp folder under assets/lib/
    for (SourcePath dir : nativeLibAssetsDirectories) {
        CopyNativeLibraries.copyNativeLibrary(getProjectFilesystem(), context.getSourcePathResolver().getAbsolutePath(dir), temp.resolve("assets").resolve("lib"), ImmutableSet.of(), commands);
    }
    // do the zipping
    commands.add(new MkdirStep(getProjectFilesystem(), pathToOutputFile.getParent()));
    commands.add(new ZipStep(getProjectFilesystem(), pathToOutputFile, ImmutableSet.of(), false, ZipCompressionLevel.DEFAULT_COMPRESSION_LEVEL, temp));
    buildableContext.recordArtifact(pathToOutputFile);
    return commands.build();
}

Example 2 with SourcePath

use of com.facebook.buck.rules.SourcePath in project buck by facebook.

the class AndroidBinary method addDexingSteps.

/**
   * Create dex artifacts for all of the individual directories of compiled .class files (or
   * the obfuscated jar files if proguard is used).  If split dex is used, multiple dex artifacts
   * will be produced.
   *  @param classpathEntriesToDex Full set of classpath entries that must make
   *     their way into the final APK structure (but not necessarily into the
   *     primary dex).
   * @param secondaryDexDirectories The contract for updating this builder must match that
   *     of {@link PreDexMerge#getSecondaryDexDirectories()}.
   * @param steps List of steps to add to.
   * @param primaryDexPath Output path for the primary dex file.
   */
@VisibleForTesting
void addDexingSteps(Set<Path> classpathEntriesToDex, Supplier<ImmutableMap<String, HashCode>> classNamesToHashesSupplier, ImmutableSet.Builder<Path> secondaryDexDirectories, ImmutableList.Builder<Step> steps, Path primaryDexPath, Optional<SourcePath> dexReorderToolFile, Optional<SourcePath> dexReorderDataDumpFile, ImmutableMultimap<APKModule, Path> additionalDexStoreToJarPathMap, SourcePathResolver resolver) {
    final Supplier<Set<Path>> primaryInputsToDex;
    final Optional<Path> secondaryDexDir;
    final Optional<Supplier<Multimap<Path, Path>>> secondaryOutputToInputs;
    Path secondaryDexParentDir = getBinPath("__%s_secondary_dex__/");
    Path additionalDexParentDir = getBinPath("__%s_additional_dex__/");
    Path additionalDexAssetsDir = additionalDexParentDir.resolve("assets");
    final Optional<ImmutableSet<Path>> additionalDexDirs;
    if (shouldSplitDex()) {
        Optional<Path> proguardFullConfigFile = Optional.empty();
        Optional<Path> proguardMappingFile = Optional.empty();
        if (packageType.isBuildWithObfuscation()) {
            Path proguardConfigDir = getProguardTextFilesPath();
            proguardFullConfigFile = Optional.of(proguardConfigDir.resolve("configuration.txt"));
            proguardMappingFile = Optional.of(proguardConfigDir.resolve("mapping.txt"));
        }
        // DexLibLoader expects that metadata.txt and secondary jar files are under this dir
        // in assets.
        // Intermediate directory holding the primary split-zip jar.
        Path splitZipDir = getBinPath("__%s_split_zip__");
        steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), splitZipDir));
        Path primaryJarPath = splitZipDir.resolve("primary.jar");
        Path secondaryJarMetaDirParent = splitZipDir.resolve("secondary_meta");
        Path secondaryJarMetaDir = secondaryJarMetaDirParent.resolve(SECONDARY_DEX_SUBDIR);
        steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), secondaryJarMetaDir));
        Path secondaryJarMeta = secondaryJarMetaDir.resolve("metadata.txt");
        // Intermediate directory holding _ONLY_ the secondary split-zip jar files.  This is
        // important because SmartDexingCommand will try to dx every entry in this directory.  It
        // does this because it's impossible to know what outputs split-zip will generate until it
        // runs.
        final Path secondaryZipDir = getBinPath("__%s_secondary_zip__");
        steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), secondaryZipDir));
        // Intermediate directory holding the directories holding _ONLY_ the additional split-zip
        // jar files that are intended for that dex store.
        final Path additionalDexStoresZipDir = getBinPath("__%s_dex_stores_zip__");
        steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), additionalDexStoresZipDir));
        for (APKModule dexStore : additionalDexStoreToJarPathMap.keySet()) {
            steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), additionalDexStoresZipDir.resolve(dexStore.getName())));
            steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), secondaryJarMetaDirParent.resolve("assets").resolve(dexStore.getName())));
        }
        // Run the split-zip command which is responsible for dividing the large set of input
        // classpaths into a more compact set of jar files such that no one jar file when dexed will
        // yield a dex artifact too large for dexopt or the dx method limit to handle.
        Path zipSplitReportDir = getBinPath("__%s_split_zip_report__");
        steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), zipSplitReportDir));
        SplitZipStep splitZipCommand = new SplitZipStep(getProjectFilesystem(), classpathEntriesToDex, secondaryJarMeta, primaryJarPath, secondaryZipDir, "secondary-%d.jar", secondaryJarMetaDirParent, additionalDexStoresZipDir, proguardFullConfigFile, proguardMappingFile, skipProguard, dexSplitMode, dexSplitMode.getPrimaryDexScenarioFile().map(resolver::getAbsolutePath), dexSplitMode.getPrimaryDexClassesFile().map(resolver::getAbsolutePath), dexSplitMode.getSecondaryDexHeadClassesFile().map(resolver::getAbsolutePath), dexSplitMode.getSecondaryDexTailClassesFile().map(resolver::getAbsolutePath), additionalDexStoreToJarPathMap, enhancementResult.getAPKModuleGraph(), zipSplitReportDir);
        steps.add(splitZipCommand);
        // smart dexing command.  Smart dex will handle "cleaning" this directory properly.
        if (reorderClassesIntraDex) {
            secondaryDexDir = Optional.of(secondaryDexParentDir.resolve(SMART_DEX_SECONDARY_DEX_SUBDIR));
            Path intraDexReorderSecondaryDexDir = secondaryDexParentDir.resolve(SECONDARY_DEX_SUBDIR);
            steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), secondaryDexDir.get()));
            steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), intraDexReorderSecondaryDexDir));
        } else {
            secondaryDexDir = Optional.of(secondaryDexParentDir.resolve(SECONDARY_DEX_SUBDIR));
            steps.add(new MkdirStep(getProjectFilesystem(), secondaryDexDir.get()));
        }
        if (additionalDexStoreToJarPathMap.isEmpty()) {
            additionalDexDirs = Optional.empty();
        } else {
            ImmutableSet.Builder<Path> builder = ImmutableSet.builder();
            for (APKModule dexStore : additionalDexStoreToJarPathMap.keySet()) {
                Path dexStorePath = additionalDexAssetsDir.resolve(dexStore.getName());
                builder.add(dexStorePath);
                steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), dexStorePath));
            }
            additionalDexDirs = Optional.of(builder.build());
        }
        if (dexSplitMode.getDexStore() == DexStore.RAW) {
            secondaryDexDirectories.add(secondaryDexDir.get());
        } else {
            secondaryDexDirectories.add(secondaryJarMetaDirParent);
            secondaryDexDirectories.add(secondaryDexParentDir);
        }
        if (additionalDexDirs.isPresent()) {
            secondaryDexDirectories.add(additionalDexParentDir);
        }
        // Adjust smart-dex inputs for the split-zip case.
        primaryInputsToDex = Suppliers.ofInstance(ImmutableSet.of(primaryJarPath));
        Supplier<Multimap<Path, Path>> secondaryOutputToInputsMap = splitZipCommand.getOutputToInputsMapSupplier(secondaryDexDir.get(), additionalDexAssetsDir);
        secondaryOutputToInputs = Optional.of(secondaryOutputToInputsMap);
    } else {
        // Simple case where our inputs are the natural classpath directories and we don't have
        // to worry about secondary jar/dex files.
        primaryInputsToDex = Suppliers.ofInstance(classpathEntriesToDex);
        secondaryDexDir = Optional.empty();
        secondaryOutputToInputs = Optional.empty();
    }
    HashInputJarsToDexStep hashInputJarsToDexStep = new HashInputJarsToDexStep(getProjectFilesystem(), primaryInputsToDex, secondaryOutputToInputs, classNamesToHashesSupplier);
    steps.add(hashInputJarsToDexStep);
    // Stores checksum information from each invocation to intelligently decide when dx needs
    // to be re-run.
    Path successDir = getBinPath("__%s_smart_dex__/.success");
    steps.add(new MkdirStep(getProjectFilesystem(), successDir));
    // Add the smart dexing tool that is capable of avoiding the external dx invocation(s) if
    // it can be shown that the inputs have not changed.  It also parallelizes dx invocations
    // where applicable.
    //
    // Note that by not specifying the number of threads this command will use it will select an
    // optimal default regardless of the value of --num-threads.  This decision was made with the
    // assumption that --num-threads specifies the threading of build rule execution and does not
    // directly apply to the internal threading/parallelization details of various build commands
    // being executed.  For example, aapt is internally threaded by default when preprocessing
    // images.
    EnumSet<DxStep.Option> dxOptions = PackageType.RELEASE.equals(packageType) ? EnumSet.of(DxStep.Option.NO_LOCALS) : EnumSet.of(DxStep.Option.NO_OPTIMIZE);
    Path selectedPrimaryDexPath = primaryDexPath;
    if (reorderClassesIntraDex) {
        String primaryDexFileName = primaryDexPath.getFileName().toString();
        String smartDexPrimaryDexFileName = "smart-dex-" + primaryDexFileName;
        Path smartDexPrimaryDexPath = Paths.get(primaryDexPath.toString().replace(primaryDexFileName, smartDexPrimaryDexFileName));
        selectedPrimaryDexPath = smartDexPrimaryDexPath;
    }
    SmartDexingStep smartDexingCommand = new SmartDexingStep(getProjectFilesystem(), selectedPrimaryDexPath, primaryInputsToDex, secondaryDexDir, secondaryOutputToInputs, hashInputJarsToDexStep, successDir, dxOptions, dxExecutorService, xzCompressionLevel, dxMaxHeapSize);
    steps.add(smartDexingCommand);
    if (reorderClassesIntraDex) {
        IntraDexReorderStep intraDexReorderStep = new IntraDexReorderStep(getProjectFilesystem(), resolver.getAbsolutePath(dexReorderToolFile.get()), resolver.getAbsolutePath(dexReorderDataDumpFile.get()), getBuildTarget(), selectedPrimaryDexPath, primaryDexPath, secondaryOutputToInputs, SMART_DEX_SECONDARY_DEX_SUBDIR, SECONDARY_DEX_SUBDIR);
        steps.add(intraDexReorderStep);
    }
}

Example 3 with SourcePath

use of com.facebook.buck.rules.SourcePath in project buck by facebook.

the class AndroidBinary method getBuildSteps.

@SuppressWarnings("PMD.PrematureDeclaration")
@Override
public ImmutableList<Step> getBuildSteps(BuildContext context, BuildableContext buildableContext) {
    ImmutableList.Builder<Step> steps = ImmutableList.builder();
    // The `HasInstallableApk` interface needs access to the manifest, so make sure we create our
    // own copy of this so that we don't have a runtime dep on the `AaptPackageResources` step.
    Path manifestPath = context.getSourcePathResolver().getRelativePath(getManifestPath());
    steps.add(new MkdirStep(getProjectFilesystem(), manifestPath.getParent()));
    steps.add(CopyStep.forFile(getProjectFilesystem(), context.getSourcePathResolver().getRelativePath(androidManifestPath), manifestPath));
    buildableContext.recordArtifact(manifestPath);
    // Create the .dex files if we aren't doing pre-dexing.
    DexFilesInfo dexFilesInfo = addFinalDxSteps(buildableContext, context.getSourcePathResolver(), steps);
    ////
    // BE VERY CAREFUL adding any code below here.
    // Any inputs to apkbuilder must be reflected in the hash returned by getAbiKeyForDeps.
    ////
    AndroidPackageableCollection packageableCollection = enhancementResult.getPackageableCollection();
    ImmutableSet.Builder<Path> nativeLibraryDirectoriesBuilder = ImmutableSet.builder();
    // Copy the transitive closure of native-libs-as-assets to a single directory, if any.
    ImmutableSet.Builder<Path> nativeLibraryAsAssetDirectories = ImmutableSet.builder();
    for (final APKModule module : enhancementResult.getAPKModuleGraph().getAPKModules()) {
        boolean shouldPackageAssetLibraries = packageAssetLibraries || !module.isRootModule();
        if (!ExopackageMode.enabledForNativeLibraries(exopackageModes) && enhancementResult.getCopyNativeLibraries().isPresent() && enhancementResult.getCopyNativeLibraries().get().containsKey(module)) {
            CopyNativeLibraries copyNativeLibraries = enhancementResult.getCopyNativeLibraries().get().get(module);
            if (shouldPackageAssetLibraries) {
                nativeLibraryDirectoriesBuilder.add(copyNativeLibraries.getPathToNativeLibsDir());
            } else {
                nativeLibraryDirectoriesBuilder.add(copyNativeLibraries.getPathToNativeLibsDir());
                nativeLibraryDirectoriesBuilder.add(copyNativeLibraries.getPathToNativeLibsAssetsDir());
            }
        }
        if ((!packageableCollection.getNativeLibAssetsDirectories().isEmpty()) || (!packageableCollection.getNativeLinkablesAssets().isEmpty() && shouldPackageAssetLibraries)) {
            Path pathForNativeLibsAsAssets = getPathForNativeLibsAsAssets();
            final Path libSubdirectory = pathForNativeLibsAsAssets.resolve("assets").resolve(module.isRootModule() ? "lib" : module.getName());
            ImmutableCollection<SourcePath> nativeLibDirs = packageableCollection.getNativeLibAssetsDirectories().get(module);
            getStepsForNativeAssets(context.getSourcePathResolver(), steps, nativeLibDirs == null ? Optional.empty() : Optional.of(nativeLibDirs), libSubdirectory, module.isRootModule() ? "metadata.txt" : "libs.txt", module);
            nativeLibraryAsAssetDirectories.add(pathForNativeLibsAsAssets);
        }
    }
    // If non-english strings are to be stored as assets, pass them to ApkBuilder.
    ImmutableSet.Builder<Path> zipFiles = ImmutableSet.builder();
    RichStream.from(primaryApkAssetsZips).map(context.getSourcePathResolver()::getRelativePath).forEach(zipFiles::add);
    if (ExopackageMode.enabledForNativeLibraries(exopackageModes)) {
        // We need to include a few dummy native libraries with our application so that Android knows
        // to run it as 32-bit.  Android defaults to 64-bit when no libraries are provided at all,
        // causing us to fail to load our 32-bit exopackage native libraries later.
        String fakeNativeLibraryBundle = System.getProperty("buck.native_exopackage_fake_path");
        if (fakeNativeLibraryBundle == null) {
            throw new RuntimeException("fake native bundle not specified in properties");
        }
        zipFiles.add(Paths.get(fakeNativeLibraryBundle));
    }
    ImmutableSet<Path> allAssetDirectories = ImmutableSet.<Path>builder().addAll(nativeLibraryAsAssetDirectories.build()).addAll(dexFilesInfo.secondaryDexDirs).build();
    SourcePathResolver resolver = context.getSourcePathResolver();
    Path signedApkPath = getSignedApkPath();
    final Path pathToKeystore = resolver.getAbsolutePath(keystorePath);
    Supplier<KeystoreProperties> keystoreProperties = Suppliers.memoize(() -> {
        try {
            return KeystoreProperties.createFromPropertiesFile(pathToKeystore, resolver.getAbsolutePath(keystorePropertiesPath), getProjectFilesystem());
        } catch (IOException e) {
            throw new RuntimeException();
        }
    });
    ApkBuilderStep apkBuilderCommand = new ApkBuilderStep(getProjectFilesystem(), context.getSourcePathResolver().getAbsolutePath(resourcesApkPath), getSignedApkPath(), dexFilesInfo.primaryDexPath, allAssetDirectories, nativeLibraryDirectoriesBuilder.build(), zipFiles.build(), packageableCollection.getPathsToThirdPartyJars().stream().map(resolver::getAbsolutePath).collect(MoreCollectors.toImmutableSet()), pathToKeystore, keystoreProperties, /* debugMode */
    false, javaRuntimeLauncher);
    steps.add(apkBuilderCommand);
    // The `ApkBuilderStep` delegates to android tools to build a ZIP with timestamps in it, making
    // the output non-deterministic.  So use an additional scrubbing step to zero these out.
    steps.add(new ZipScrubberStep(getProjectFilesystem(), signedApkPath));
    Path apkToRedexAndAlign;
    // Optionally, compress the resources file in the .apk.
    if (this.isCompressResources()) {
        Path compressedApkPath = getCompressedResourcesApkPath();
        apkToRedexAndAlign = compressedApkPath;
        RepackZipEntriesStep arscComp = new RepackZipEntriesStep(getProjectFilesystem(), signedApkPath, compressedApkPath, ImmutableSet.of("resources.arsc"));
        steps.add(arscComp);
    } else {
        apkToRedexAndAlign = signedApkPath;
    }
    boolean applyRedex = redexOptions.isPresent();
    Path apkPath = context.getSourcePathResolver().getRelativePath(getSourcePathToOutput());
    Path apkToAlign = apkToRedexAndAlign;
    // redex
    if (applyRedex) {
        Path proguardConfigDir = getProguardTextFilesPath();
        Path redexedApk = apkPath.getParent().resolve(apkPath.getFileName().toString() + ".redex");
        apkToAlign = redexedApk;
        ImmutableList<Step> redexSteps = ReDexStep.createSteps(getProjectFilesystem(), resolver, redexOptions.get(), apkToRedexAndAlign, redexedApk, keystoreProperties, proguardConfigDir, context.getSourcePathResolver());
        steps.addAll(redexSteps);
    }
    steps.add(new ZipalignStep(getProjectFilesystem().getRootPath(), apkToAlign, apkPath));
    buildableContext.recordArtifact(apkPath);
    return steps.build();
}

Example 4 with SourcePath

use of com.facebook.buck.rules.SourcePath in project buck by facebook.

the class AndroidResourceDescription method collectInputFiles.

@VisibleForTesting
ImmutableSortedMap<Path, SourcePath> collectInputFiles(final ProjectFilesystem filesystem, Path inputDir) {
    final ImmutableSortedMap.Builder<Path, SourcePath> paths = ImmutableSortedMap.naturalOrder();
    // We ignore the same files that mini-aapt and aapt ignore.
    FileVisitor<Path> fileVisitor = new SimpleFileVisitor<Path>() {

        @Override
        public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attr) throws IOException {
            String dirName = dir.getFileName().toString();
            // Special case: directory starting with '_' as per aapt.
            if (dirName.charAt(0) == '_' || !isPossibleResourceName(dirName)) {
                return FileVisitResult.SKIP_SUBTREE;
            }
            return FileVisitResult.CONTINUE;
        }

        @Override
        public FileVisitResult visitFile(Path file, BasicFileAttributes attr) throws IOException {
            String filename = file.getFileName().toString();
            if (isPossibleResourceName(filename)) {
                paths.put(MorePaths.relativize(inputDir, file), new PathSourcePath(filesystem, file));
            }
            return FileVisitResult.CONTINUE;
        }
    };
    try {
        filesystem.walkRelativeFileTree(inputDir, fileVisitor);
    } catch (IOException e) {
        throw new HumanReadableException(e, "Error while searching for android resources in directory %s.", inputDir);
    }
    return paths.build();
}

Example 5 with SourcePath

use of com.facebook.buck.rules.SourcePath in project buck by facebook.

the class AndroidResourceDescription method createSymlinkTree.

private SymlinkTree createSymlinkTree(SourcePathRuleFinder ruleFinder, BuildRuleParams params, Optional<Either<SourcePath, ImmutableSortedMap<String, SourcePath>>> symlinkAttribute, String outputDirName) {
    ImmutableMap<Path, SourcePath> links = ImmutableMap.of();
    if (symlinkAttribute.isPresent()) {
        if (symlinkAttribute.get().isLeft()) {
            // If our resources are coming from a `PathSourcePath`, we collect only the inputs we care
            // about and pass those in separately, so that that `AndroidResource` rule knows to only
            // hash these into it's rule key.
            // TODO(k21): This is deprecated and should be disabled or removed.
            // Accessing the filesystem during rule creation is problematic because the accesses are
            // not cached or tracked in any way.
            Preconditions.checkArgument(symlinkAttribute.get().getLeft() instanceof PathSourcePath, "Resource or asset symlink tree can only be built for a PathSourcePath");
            PathSourcePath path = (PathSourcePath) symlinkAttribute.get().getLeft();
            links = collectInputFiles(path.getFilesystem(), path.getRelativePath());
        } else {
            links = RichStream.from(symlinkAttribute.get().getRight().entrySet()).map(e -> new AbstractMap.SimpleEntry<>(Paths.get(e.getKey()), e.getValue())).filter(e -> isPossibleResourcePath(e.getKey())).collect(MoreCollectors.toImmutableMap(e -> e.getKey(), e -> e.getValue()));
        }
    }
    Path symlinkTreeRoot = BuildTargets.getGenPath(params.getProjectFilesystem(), params.getBuildTarget(), "%s").resolve(outputDirName);
    return new SymlinkTree(params.getBuildTarget(), params.getProjectFilesystem(), symlinkTreeRoot, links, ruleFinder);
}