Examples with XzStep com.facebook.buck.step.fs.XzStep used on opensource projects

Example 1 with XzStep

use of com.facebook.buck.step.fs.XzStep in project buck by facebook.

the class AndroidBinary method getStepsForNativeAssets.

private void getStepsForNativeAssets(SourcePathResolver resolver, ImmutableList.Builder<Step> steps, Optional<ImmutableCollection<SourcePath>> nativeLibDirs, final Path libSubdirectory, final String metadataFilename, final APKModule module) {
    steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), libSubdirectory));
    // Filter, rename and copy the ndk libraries marked as assets.
    if (nativeLibDirs.isPresent()) {
        for (SourcePath nativeLibDir : nativeLibDirs.get()) {
            CopyNativeLibraries.copyNativeLibrary(getProjectFilesystem(), resolver.getAbsolutePath(nativeLibDir), libSubdirectory, cpuFilters, steps);
        }
    }
    // Input asset libraries are sorted in descending filesize order.
    final ImmutableSortedSet.Builder<Path> inputAssetLibrariesBuilder = ImmutableSortedSet.orderedBy((libPath1, libPath2) -> {
        try {
            ProjectFilesystem filesystem = getProjectFilesystem();
            int filesizeResult = -Long.compare(filesystem.getFileSize(libPath1), filesystem.getFileSize(libPath2));
            int pathnameResult = libPath1.compareTo(libPath2);
            return filesizeResult != 0 ? filesizeResult : pathnameResult;
        } catch (IOException e) {
            return 0;
        }
    });
    if (packageAssetLibraries || !module.isRootModule()) {
        if (enhancementResult.getCopyNativeLibraries().isPresent() && enhancementResult.getCopyNativeLibraries().get().containsKey(module)) {
            // Copy in cxx libraries marked as assets. Filtering and renaming was already done
            // in CopyNativeLibraries.getBuildSteps().
            Path cxxNativeLibsSrc = enhancementResult.getCopyNativeLibraries().get().get(module).getPathToNativeLibsAssetsDir();
            steps.add(CopyStep.forDirectory(getProjectFilesystem(), cxxNativeLibsSrc, libSubdirectory, CopyStep.DirectoryMode.CONTENTS_ONLY));
        }
        steps.add(// Step that populates a list of libraries and writes a metadata.txt to decompress.
        new AbstractExecutionStep("write_metadata_for_asset_libraries_" + module.getName()) {

            @Override
            public StepExecutionResult execute(ExecutionContext context) {
                ProjectFilesystem filesystem = getProjectFilesystem();
                try {
                    // Walk file tree to find libraries
                    filesystem.walkRelativeFileTree(libSubdirectory, new SimpleFileVisitor<Path>() {

                        @Override
                        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                            if (!file.toString().endsWith(".so")) {
                                throw new IOException("unexpected file in lib directory");
                            }
                            inputAssetLibrariesBuilder.add(file);
                            return FileVisitResult.CONTINUE;
                        }
                    });
                    // Write a metadata
                    ImmutableList.Builder<String> metadataLines = ImmutableList.builder();
                    Path metadataOutput = libSubdirectory.resolve(metadataFilename);
                    for (Path libPath : inputAssetLibrariesBuilder.build()) {
                        // Should return something like x86/libfoo.so
                        Path relativeLibPath = libSubdirectory.relativize(libPath);
                        long filesize = filesystem.getFileSize(libPath);
                        String desiredOutput = relativeLibPath.toString();
                        String checksum = filesystem.computeSha256(libPath);
                        metadataLines.add(desiredOutput + ' ' + filesize + ' ' + checksum);
                    }
                    ImmutableList<String> metadata = metadataLines.build();
                    if (!metadata.isEmpty()) {
                        filesystem.writeLinesToPath(metadata, metadataOutput);
                    }
                } catch (IOException e) {
                    context.logError(e, "Writing metadata for asset libraries failed.");
                    return StepExecutionResult.ERROR;
                }
                return StepExecutionResult.SUCCESS;
            }
        });
    }
    if (compressAssetLibraries || !module.isRootModule()) {
        final ImmutableList.Builder<Path> outputAssetLibrariesBuilder = ImmutableList.builder();
        steps.add(new AbstractExecutionStep("rename_asset_libraries_as_temp_files_" + module.getName()) {

            @Override
            public StepExecutionResult execute(ExecutionContext context) {
                try {
                    ProjectFilesystem filesystem = getProjectFilesystem();
                    for (Path libPath : inputAssetLibrariesBuilder.build()) {
                        Path tempPath = libPath.resolveSibling(libPath.getFileName() + "~");
                        filesystem.move(libPath, tempPath);
                        outputAssetLibrariesBuilder.add(tempPath);
                    }
                    return StepExecutionResult.SUCCESS;
                } catch (IOException e) {
                    context.logError(e, "Renaming asset libraries failed");
                    return StepExecutionResult.ERROR;
                }
            }
        });
        // Concat and xz compress.
        Path libOutputBlob = libSubdirectory.resolve("libraries.blob");
        steps.add(new ConcatStep(getProjectFilesystem(), outputAssetLibrariesBuilder, libOutputBlob));
        int compressionLevel = xzCompressionLevel.orElse(XzStep.DEFAULT_COMPRESSION_LEVEL).intValue();
        steps.add(new XzStep(getProjectFilesystem(), libOutputBlob, libSubdirectory.resolve(SOLID_COMPRESSED_ASSET_LIBRARY_FILENAME), compressionLevel));
    }
}

Example 2 with XzStep

use of com.facebook.buck.step.fs.XzStep in project buck by facebook.

the class SmartDexingStep method execute.

@Override
public StepExecutionResult execute(ExecutionContext context) throws InterruptedException {
    try {
        Multimap<Path, Path> outputToInputs = outputToInputsSupplier.get();
        runDxCommands(context, outputToInputs);
        if (secondaryOutputDir.isPresent()) {
            removeExtraneousSecondaryArtifacts(secondaryOutputDir.get(), outputToInputs.keySet(), filesystem);
            // Concatenate if solid compression is specified.
            // create a mapping of the xzs file target and the dex.jar files that go into it
            ImmutableMultimap.Builder<Path, Path> secondaryDexJarsMultimapBuilder = ImmutableMultimap.builder();
            for (Path p : outputToInputs.keySet()) {
                if (DexStore.XZS.matchesPath(p)) {
                    String[] matches = p.getFileName().toString().split("-");
                    Path output = p.getParent().resolve(matches[0].concat(SECONDARY_SOLID_DEX_EXTENSION));
                    secondaryDexJarsMultimapBuilder.put(output, p);
                }
            }
            ImmutableMultimap<Path, Path> secondaryDexJarsMultimap = secondaryDexJarsMultimapBuilder.build();
            if (!secondaryDexJarsMultimap.isEmpty()) {
                for (Map.Entry<Path, Collection<Path>> entry : secondaryDexJarsMultimap.asMap().entrySet()) {
                    Path store = entry.getKey();
                    Collection<Path> secondaryDexJars = entry.getValue();
                    // Construct the output path for our solid blob and its compressed form.
                    Path secondaryBlobOutput = store.getParent().resolve("uncompressed.dex.blob");
                    Path secondaryCompressedBlobOutput = store;
                    // Concatenate the jars into a blob and compress it.
                    StepRunner stepRunner = new DefaultStepRunner();
                    Step concatStep = new ConcatStep(filesystem, ImmutableList.copyOf(secondaryDexJars), secondaryBlobOutput);
                    Step xzStep;
                    if (xzCompressionLevel.isPresent()) {
                        xzStep = new XzStep(filesystem, secondaryBlobOutput, secondaryCompressedBlobOutput, xzCompressionLevel.get().intValue());
                    } else {
                        xzStep = new XzStep(filesystem, secondaryBlobOutput, secondaryCompressedBlobOutput);
                    }
                    stepRunner.runStepForBuildTarget(context, concatStep, Optional.empty());
                    stepRunner.runStepForBuildTarget(context, xzStep, Optional.empty());
                }
            }
        }
    } catch (StepFailedException | IOException e) {
        context.logError(e, "There was an error in smart dexing step.");
        return StepExecutionResult.ERROR;
    }
    return StepExecutionResult.SUCCESS;
}

Example 3 with XzStep

use of com.facebook.buck.step.fs.XzStep in project buck by facebook.

the class SmartDexingStep method createDxStepForDxPseudoRule.

/**
   * The step to produce the .dex file will be determined by the file extension of outputPath, much
   * as {@code dx} itself chooses whether to embed the dex inside a jar/zip based on the destination
   * file passed to it.  We also create a ".meta" file that contains information about the
   * compressed and uncompressed size of the dex; this information is useful later, in applications,
   * when unpacking.
   */
static Step createDxStepForDxPseudoRule(ProjectFilesystem filesystem, Collection<Path> filesToDex, Path outputPath, EnumSet<Option> dxOptions, Optional<Integer> xzCompressionLevel, Optional<String> dxMaxHeapSize) {
    String output = outputPath.toString();
    List<Step> steps = Lists.newArrayList();
    if (DexStore.XZ.matchesPath(outputPath)) {
        Path tempDexJarOutput = Paths.get(output.replaceAll("\\.jar\\.xz$", ".tmp.jar"));
        steps.add(new DxStep(filesystem, tempDexJarOutput, filesToDex, dxOptions, dxMaxHeapSize));
        // We need to make sure classes.dex is STOREd in the .dex.jar file, otherwise .XZ
        // compression won't be effective.
        Path repackedJar = Paths.get(output.replaceAll("\\.xz$", ""));
        steps.add(new RepackZipEntriesStep(filesystem, tempDexJarOutput, repackedJar, ImmutableSet.of("classes.dex"), ZipCompressionLevel.MIN_COMPRESSION_LEVEL));
        steps.add(new RmStep(filesystem, tempDexJarOutput));
        steps.add(new DexJarAnalysisStep(filesystem, repackedJar, repackedJar.resolveSibling(repackedJar.getFileName() + ".meta")));
        if (xzCompressionLevel.isPresent()) {
            steps.add(new XzStep(filesystem, repackedJar, xzCompressionLevel.get().intValue()));
        } else {
            steps.add(new XzStep(filesystem, repackedJar));
        }
    } else if (DexStore.XZS.matchesPath(outputPath)) {
        // Essentially the same logic as the XZ case above, except we compress later.
        // The differences in output file names make it worth separating into a different case.
        // Ensure classes.dex is stored.
        Path tempDexJarOutput = Paths.get(output.replaceAll("\\.jar\\.xzs\\.tmp~$", ".tmp.jar"));
        steps.add(new DxStep(filesystem, tempDexJarOutput, filesToDex, dxOptions, dxMaxHeapSize));
        steps.add(new RepackZipEntriesStep(filesystem, tempDexJarOutput, outputPath, ImmutableSet.of("classes.dex"), ZipCompressionLevel.MIN_COMPRESSION_LEVEL));
        steps.add(new RmStep(filesystem, tempDexJarOutput));
        // Write a .meta file.
        steps.add(new DexJarAnalysisStep(filesystem, outputPath, outputPath.resolveSibling(outputPath.getFileName() + ".meta")));
    } else if (DexStore.JAR.matchesPath(outputPath) || DexStore.RAW.matchesPath(outputPath) || output.endsWith("classes.dex")) {
        steps.add(new DxStep(filesystem, outputPath, filesToDex, dxOptions, dxMaxHeapSize));
        if (DexStore.JAR.matchesPath(outputPath)) {
            steps.add(new DexJarAnalysisStep(filesystem, outputPath, outputPath.resolveSibling(outputPath.getFileName() + ".meta")));
        }
    } else {
        throw new IllegalArgumentException(String.format("Suffix of %s does not have a corresponding DexStore type.", outputPath));
    }
    return new CompositeStep(steps);
}