Examples with ZipScrubberStep com.facebook.buck.zip.ZipScrubberStep used on opensource projects

Example 1 with ZipScrubberStep

use of com.facebook.buck.zip.ZipScrubberStep in project buck by facebook.

the class AndroidBinary method getBuildSteps.

@SuppressWarnings("PMD.PrematureDeclaration")
@Override
public ImmutableList<Step> getBuildSteps(BuildContext context, BuildableContext buildableContext) {
    ImmutableList.Builder<Step> steps = ImmutableList.builder();
    // The `HasInstallableApk` interface needs access to the manifest, so make sure we create our
    // own copy of this so that we don't have a runtime dep on the `AaptPackageResources` step.
    Path manifestPath = context.getSourcePathResolver().getRelativePath(getManifestPath());
    steps.add(new MkdirStep(getProjectFilesystem(), manifestPath.getParent()));
    steps.add(CopyStep.forFile(getProjectFilesystem(), context.getSourcePathResolver().getRelativePath(androidManifestPath), manifestPath));
    buildableContext.recordArtifact(manifestPath);
    // Create the .dex files if we aren't doing pre-dexing.
    DexFilesInfo dexFilesInfo = addFinalDxSteps(buildableContext, context.getSourcePathResolver(), steps);
    ////
    // BE VERY CAREFUL adding any code below here.
    // Any inputs to apkbuilder must be reflected in the hash returned by getAbiKeyForDeps.
    ////
    AndroidPackageableCollection packageableCollection = enhancementResult.getPackageableCollection();
    ImmutableSet.Builder<Path> nativeLibraryDirectoriesBuilder = ImmutableSet.builder();
    // Copy the transitive closure of native-libs-as-assets to a single directory, if any.
    ImmutableSet.Builder<Path> nativeLibraryAsAssetDirectories = ImmutableSet.builder();
    for (final APKModule module : enhancementResult.getAPKModuleGraph().getAPKModules()) {
        boolean shouldPackageAssetLibraries = packageAssetLibraries || !module.isRootModule();
        if (!ExopackageMode.enabledForNativeLibraries(exopackageModes) && enhancementResult.getCopyNativeLibraries().isPresent() && enhancementResult.getCopyNativeLibraries().get().containsKey(module)) {
            CopyNativeLibraries copyNativeLibraries = enhancementResult.getCopyNativeLibraries().get().get(module);
            if (shouldPackageAssetLibraries) {
                nativeLibraryDirectoriesBuilder.add(copyNativeLibraries.getPathToNativeLibsDir());
            } else {
                nativeLibraryDirectoriesBuilder.add(copyNativeLibraries.getPathToNativeLibsDir());
                nativeLibraryDirectoriesBuilder.add(copyNativeLibraries.getPathToNativeLibsAssetsDir());
            }
        }
        if ((!packageableCollection.getNativeLibAssetsDirectories().isEmpty()) || (!packageableCollection.getNativeLinkablesAssets().isEmpty() && shouldPackageAssetLibraries)) {
            Path pathForNativeLibsAsAssets = getPathForNativeLibsAsAssets();
            final Path libSubdirectory = pathForNativeLibsAsAssets.resolve("assets").resolve(module.isRootModule() ? "lib" : module.getName());
            ImmutableCollection<SourcePath> nativeLibDirs = packageableCollection.getNativeLibAssetsDirectories().get(module);
            getStepsForNativeAssets(context.getSourcePathResolver(), steps, nativeLibDirs == null ? Optional.empty() : Optional.of(nativeLibDirs), libSubdirectory, module.isRootModule() ? "metadata.txt" : "libs.txt", module);
            nativeLibraryAsAssetDirectories.add(pathForNativeLibsAsAssets);
        }
    }
    // If non-english strings are to be stored as assets, pass them to ApkBuilder.
    ImmutableSet.Builder<Path> zipFiles = ImmutableSet.builder();
    RichStream.from(primaryApkAssetsZips).map(context.getSourcePathResolver()::getRelativePath).forEach(zipFiles::add);
    if (ExopackageMode.enabledForNativeLibraries(exopackageModes)) {
        // We need to include a few dummy native libraries with our application so that Android knows
        // to run it as 32-bit.  Android defaults to 64-bit when no libraries are provided at all,
        // causing us to fail to load our 32-bit exopackage native libraries later.
        String fakeNativeLibraryBundle = System.getProperty("buck.native_exopackage_fake_path");
        if (fakeNativeLibraryBundle == null) {
            throw new RuntimeException("fake native bundle not specified in properties");
        }
        zipFiles.add(Paths.get(fakeNativeLibraryBundle));
    }
    ImmutableSet<Path> allAssetDirectories = ImmutableSet.<Path>builder().addAll(nativeLibraryAsAssetDirectories.build()).addAll(dexFilesInfo.secondaryDexDirs).build();
    SourcePathResolver resolver = context.getSourcePathResolver();
    Path signedApkPath = getSignedApkPath();
    final Path pathToKeystore = resolver.getAbsolutePath(keystorePath);
    Supplier<KeystoreProperties> keystoreProperties = Suppliers.memoize(() -> {
        try {
            return KeystoreProperties.createFromPropertiesFile(pathToKeystore, resolver.getAbsolutePath(keystorePropertiesPath), getProjectFilesystem());
        } catch (IOException e) {
            throw new RuntimeException();
        }
    });
    ApkBuilderStep apkBuilderCommand = new ApkBuilderStep(getProjectFilesystem(), context.getSourcePathResolver().getAbsolutePath(resourcesApkPath), getSignedApkPath(), dexFilesInfo.primaryDexPath, allAssetDirectories, nativeLibraryDirectoriesBuilder.build(), zipFiles.build(), packageableCollection.getPathsToThirdPartyJars().stream().map(resolver::getAbsolutePath).collect(MoreCollectors.toImmutableSet()), pathToKeystore, keystoreProperties, /* debugMode */
    false, javaRuntimeLauncher);
    steps.add(apkBuilderCommand);
    // The `ApkBuilderStep` delegates to android tools to build a ZIP with timestamps in it, making
    // the output non-deterministic.  So use an additional scrubbing step to zero these out.
    steps.add(new ZipScrubberStep(getProjectFilesystem(), signedApkPath));
    Path apkToRedexAndAlign;
    // Optionally, compress the resources file in the .apk.
    if (this.isCompressResources()) {
        Path compressedApkPath = getCompressedResourcesApkPath();
        apkToRedexAndAlign = compressedApkPath;
        RepackZipEntriesStep arscComp = new RepackZipEntriesStep(getProjectFilesystem(), signedApkPath, compressedApkPath, ImmutableSet.of("resources.arsc"));
        steps.add(arscComp);
    } else {
        apkToRedexAndAlign = signedApkPath;
    }
    boolean applyRedex = redexOptions.isPresent();
    Path apkPath = context.getSourcePathResolver().getRelativePath(getSourcePathToOutput());
    Path apkToAlign = apkToRedexAndAlign;
    // redex
    if (applyRedex) {
        Path proguardConfigDir = getProguardTextFilesPath();
        Path redexedApk = apkPath.getParent().resolve(apkPath.getFileName().toString() + ".redex");
        apkToAlign = redexedApk;
        ImmutableList<Step> redexSteps = ReDexStep.createSteps(getProjectFilesystem(), resolver, redexOptions.get(), apkToRedexAndAlign, redexedApk, keystoreProperties, proguardConfigDir, context.getSourcePathResolver());
        steps.addAll(redexSteps);
    }
    steps.add(new ZipalignStep(getProjectFilesystem().getRootPath(), apkToAlign, apkPath));
    buildableContext.recordArtifact(apkPath);
    return steps.build();
}

Example 2 with ZipScrubberStep

use of com.facebook.buck.zip.ZipScrubberStep in project buck by facebook.

the class Genrule method getBuildSteps.

@Override
@VisibleForTesting
public ImmutableList<Step> getBuildSteps(BuildContext context, BuildableContext buildableContext) {
    ImmutableList.Builder<Step> commands = ImmutableList.builder();
    // Make sure that the directory to contain the output file exists, deleting any pre-existing
    // ones. Rules get output to a directory named after the base path, so we don't want to nuke
    // the entire directory.
    commands.add(new MakeCleanDirectoryStep(getProjectFilesystem(), pathToOutDirectory));
    // Delete the old temp directory
    commands.add(new MakeCleanDirectoryStep(getProjectFilesystem(), pathToTmpDirectory));
    // Create a directory to hold all the source files.
    commands.add(new MakeCleanDirectoryStep(getProjectFilesystem(), pathToSrcDirectory));
    addSymlinkCommands(context, commands);
    // Create a shell command that corresponds to this.cmd.
    if (this.isWorkerGenrule) {
        commands.add(createWorkerShellStep(context));
    } else {
        commands.add(createGenruleStep(context));
    }
    if (MorePaths.getFileExtension(pathToOutFile).equals("zip")) {
        commands.add(new ZipScrubberStep(getProjectFilesystem(), pathToOutFile));
    }
    buildableContext.recordArtifact(pathToOutFile);
    return commands.build();
}

Example 3 with ZipScrubberStep

use of com.facebook.buck.zip.ZipScrubberStep in project buck by facebook.

the class AaptPackageResources method getBuildSteps.

@Override
public ImmutableList<Step> getBuildSteps(BuildContext context, final BuildableContext buildableContext) {
    ImmutableList.Builder<Step> steps = ImmutableList.builder();
    // Copy manifest to a path named AndroidManifest.xml after replacing the manifest placeholders
    // if needed. Do this before running any other commands to ensure that it is available at the
    // desired path.
    steps.add(new MkdirStep(getProjectFilesystem(), getAndroidManifestXml().getParent()));
    Optional<ImmutableMap<String, String>> placeholders = manifestEntries.getPlaceholders();
    if (placeholders.isPresent() && !placeholders.get().isEmpty()) {
        steps.add(new ReplaceManifestPlaceholdersStep(getProjectFilesystem(), context.getSourcePathResolver().getAbsolutePath(manifest), getAndroidManifestXml(), placeholders.get()));
    } else {
        steps.add(CopyStep.forFile(getProjectFilesystem(), context.getSourcePathResolver().getAbsolutePath(manifest), getAndroidManifestXml()));
    }
    steps.add(new MkdirStep(getProjectFilesystem(), getResourceApkPath().getParent()));
    Path rDotTxtDir = getPathToRDotTxtDir();
    steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), rDotTxtDir));
    Path pathToGeneratedProguardConfig = getPathToGeneratedProguardConfigFile();
    steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), pathToGeneratedProguardConfig.getParent()));
    buildableContext.recordArtifact(pathToGeneratedProguardConfig);
    steps.add(new AaptStep(getProjectFilesystem().getRootPath(), getAndroidManifestXml(), filteredResourcesProvider.getResDirectories(), context.getSourcePathResolver().getAllAbsolutePaths(assetsDirectories), getResourceApkPath(), rDotTxtDir, pathToGeneratedProguardConfig, /*
             * In practice, it appears that if --no-crunch is used, resources will occasionally
             * appear distorted in the APK produced by this command (and what's worse, a clean
             * reinstall does not make the problem go away). This is not reliably reproducible, so
             * for now, we categorically outlaw the use of --no-crunch so that developers do not get
             * stuck in the distorted image state. One would expect the use of --no-crunch to allow
             * for faster build times, so it would be nice to figure out a way to leverage it in
             * debug mode that never results in distorted images.
             */
    !skipCrunchPngs, /* && packageType.isCrunchPngFiles() */
    includesVectorDrawables, manifestEntries), new ZipScrubberStep(getProjectFilesystem(), getResourceApkPath()));
    // If we had an empty res directory, we won't generate an R.txt file.  This ensures that it
    // always exists.
    steps.add(new TouchStep(getProjectFilesystem(), getPathToRDotTxtFile()));
    if (hasRDotJava()) {
        generateRDotJavaFiles(steps, buildableContext, context);
    }
    // Record the filtered resources dirs, since when we initialize ourselves from disk, we'll
    // need to test whether this is empty or not without requiring the `ResourcesFilter` rule to
    // be available.
    buildableContext.addMetadata(FILTERED_RESOURCE_DIRS_KEY, FluentIterable.from(filteredResourcesProvider.getResDirectories()).transform(Object::toString).toSortedList(Ordering.natural()));
    buildableContext.recordArtifact(getAndroidManifestXml());
    buildableContext.recordArtifact(getResourceApkPath());
    steps.add(new RecordFileSha1Step(getProjectFilesystem(), getResourceApkPath(), RESOURCE_PACKAGE_HASH_KEY, buildableContext));
    return steps.build();
}

Example 4 with ZipScrubberStep

use of com.facebook.buck.zip.ZipScrubberStep in project buck by facebook.

the class DexProducedFromJavaLibrary method getBuildSteps.

@Override
public ImmutableList<Step> getBuildSteps(BuildContext context, final BuildableContext buildableContext) {
    ImmutableList.Builder<Step> steps = ImmutableList.builder();
    steps.add(new RmStep(getProjectFilesystem(), getPathToDex()));
    // Make sure that the buck-out/gen/ directory exists for this.buildTarget.
    steps.add(new MkdirStep(getProjectFilesystem(), getPathToDex().getParent()));
    // If there are classes, run dx.
    final ImmutableSortedMap<String, HashCode> classNamesToHashes = javaLibrary.getClassNamesToHashes();
    final boolean hasClassesToDx = !classNamesToHashes.isEmpty();
    final Supplier<Integer> weightEstimate;
    @Nullable final DxStep dx;
    if (hasClassesToDx) {
        Path pathToOutputFile = context.getSourcePathResolver().getAbsolutePath(javaLibrarySourcePath);
        EstimateDexWeightStep estimate = new EstimateDexWeightStep(getProjectFilesystem(), pathToOutputFile);
        steps.add(estimate);
        weightEstimate = estimate;
        // To be conservative, use --force-jumbo for these intermediate .dex files so that they can be
        // merged into a final classes.dex that uses jumbo instructions.
        dx = new DxStep(getProjectFilesystem(), getPathToDex(), Collections.singleton(pathToOutputFile), EnumSet.of(DxStep.Option.USE_CUSTOM_DX_IF_AVAILABLE, DxStep.Option.RUN_IN_PROCESS, DxStep.Option.NO_OPTIMIZE, DxStep.Option.FORCE_JUMBO));
        steps.add(dx);
        // The `DxStep` delegates to android tools to build a ZIP with timestamps in it, making
        // the output non-deterministic.  So use an additional scrubbing step to zero these out.
        steps.add(new ZipScrubberStep(getProjectFilesystem(), getPathToDex()));
    } else {
        dx = null;
        weightEstimate = Suppliers.ofInstance(0);
    }
    // Run a step to record artifacts and metadata. The values recorded depend upon whether dx was
    // run.
    String stepName = hasClassesToDx ? "record_dx_success" : "record_empty_dx";
    AbstractExecutionStep recordArtifactAndMetadataStep = new AbstractExecutionStep(stepName) {

        @Override
        public StepExecutionResult execute(ExecutionContext context) throws IOException {
            if (hasClassesToDx) {
                buildableContext.recordArtifact(getPathToDex());
                @Nullable Collection<String> referencedResources = dx.getResourcesReferencedInCode();
                if (referencedResources != null) {
                    buildableContext.addMetadata(REFERENCED_RESOURCES, Ordering.natural().immutableSortedCopy(referencedResources));
                }
            }
            buildableContext.addMetadata(WEIGHT_ESTIMATE, String.valueOf(weightEstimate.get()));
            // Record the classnames to hashes map.
            buildableContext.addMetadata(CLASSNAMES_TO_HASHES, context.getObjectMapper().writeValueAsString(Maps.transformValues(classNamesToHashes, Object::toString)));
            return StepExecutionResult.SUCCESS;
        }
    };
    steps.add(recordArtifactAndMetadataStep);
    return steps.build();
}

Example 5 with ZipScrubberStep

use of com.facebook.buck.zip.ZipScrubberStep in project buck by facebook.

the class TrimUberRDotJava method getBuildSteps.

@Override
public ImmutableList<Step> getBuildSteps(BuildContext context, BuildableContext buildableContext) {
    Path output = context.getSourcePathResolver().getRelativePath(getSourcePathToOutput());
    buildableContext.recordArtifact(output);
    return ImmutableList.of(new MakeCleanDirectoryStep(getProjectFilesystem(), output.getParent()), new PerformTrimStep(context.getSourcePathResolver().getAbsolutePath(getSourcePathToOutput())), new ZipScrubberStep(getProjectFilesystem(), output));
}