Examples with FileCache com.android.builder.utils.FileCache used on opensource projects

Example 1 with FileCache

use of com.android.builder.utils.FileCache in project atlas by alibaba.

the class AtlasDesugarTransform method processSingle.

private void processSingle(@NonNull Path input, @NonNull Path output, @NonNull Set<? super QualifiedContent.Scope> scopes) throws Exception {
    waitableExecutor.execute(() -> {
        if (output.toString().endsWith(SdkConstants.DOT_JAR)) {
            Files.createDirectories(output.getParent());
        } else {
            Files.createDirectories(output);
        }
        FileCache cacheToUse;
        if (Files.isRegularFile(input) && Objects.equals(scopes, Collections.singleton(QualifiedContent.Scope.EXTERNAL_LIBRARIES))) {
            cacheToUse = userCache;
        } else {
            cacheToUse = null;
        }
        processUsingCache(input, output, cacheToUse);
        return null;
    });
}

Example 2 with FileCache

use of com.android.builder.utils.FileCache in project atlas by alibaba.

the class AtlasDexArchiveBuilderCacheHander method populateCache.

void populateCache(Multimap<QualifiedContent, File> cacheableItems) throws Exception {
    for (QualifiedContent input : cacheableItems.keys()) {
        FileCache cache = getBuildCache(input.getFile(), isExternalLib(input), userLevelCache);
        if (cache != null) {
            FileCache.Inputs buildCacheInputs = AtlasDexArchiveBuilderCacheHander.getBuildCacheInputs(input.getFile(), dexOptions, dexer, minSdkVersion, isDebuggable);
            FileCache.QueryResult result = cache.createFileInCacheIfAbsent(buildCacheInputs, in -> {
                Collection<File> dexArchives = cacheableItems.get(input);
                logger.verbose("Merging %1$s into %2$s", Joiner.on(',').join(dexArchives), in.getAbsolutePath());
                mergeJars(in, cacheableItems.get(input));
            });
            if (result.getQueryEvent().equals(FileCache.QueryEvent.CORRUPTED)) {
                Verify.verifyNotNull(result.getCauseOfCorruption());
                logger.info("The build cache at '%1$s' contained an invalid cache entry.\n" + "Cause: %2$s\n" + "We have recreated the cache entry.\n" + "%3$s", cache.getCacheDirectory().getAbsolutePath(), Throwables.getStackTraceAsString(result.getCauseOfCorruption()), BuildCacheUtils.BUILD_CACHE_TROUBLESHOOTING_MESSAGE);
            }
        }
    }
}

Example 3 with FileCache

use of com.android.builder.utils.FileCache in project atlas by alibaba.

the class AtlasDexArchiveBuilderCacheHander method getCachedVersionIfPresent.

@Nullable
public File getCachedVersionIfPresent(JarInput input) throws Exception {
    FileCache cache = getBuildCache(input.getFile(), isExternalLib(input), userLevelCache);
    if (cache == null) {
        return null;
    }
    FileCache.Inputs buildCacheInputs = getBuildCacheInputs(input.getFile(), dexOptions, dexer, minSdkVersion, isDebuggable);
    return cache.cacheEntryExists(buildCacheInputs) ? cache.getFileInCache(buildCacheInputs) : null;
}

Example 4 with FileCache

use of com.android.builder.utils.FileCache in project atlas by alibaba.

the class AtlasMainDexMerger method merge.

public void merge(List<File> mainDexFiles, File outputDir, File[][] mergeDexs) {
    sort(mainDexFiles);
    FileCache.Inputs buildCacheInputs = getBuildCacheInputs(mainDexFiles, dexingType, dexMerger, mainDexListFile == null ? null : mainDexListFile.getSingleFile(), minSdkVersion, isDebuggable, "maindex", ID);
    ProcessOutput output = outputHandler.createOutput();
    FileCache fileCache = BuildCacheUtils.createBuildCacheIfEnabled(variantOutputContext.getVariantContext().getProject(), variantOutputContext.getScope().getGlobalScope().getProjectOptions());
    if (fileCache == null) {
        try {
            fileCache = FileCache.getInstanceWithMultiProcessLocking(new File(AndroidLocation.getFolder(), "atlas-buildCache"));
        } catch (AndroidLocation.AndroidLocationException e) {
            e.printStackTrace();
        }
    }
    if (variantOutputContext.getVariantContext().getAtlasExtension().getTBuildConfig().getMergeBundlesDex()) {
        allDexsArchives.addAll(Arrays.asList(mergeDexs[0]));
        return;
    }
    try {
        FileCache.QueryResult result = fileCache.createFileInCacheIfAbsent(buildCacheInputs, in -> {
            List<ForkJoinTask<Void>> mergeTasks = new ArrayList();
            if (dexingType == DexingType.NATIVE_MULTIDEX) {
                mergeTasks.addAll(handleNativeMultiDex(Arrays.asList(mergeDexs[0]), output, outputDir, mainDexListFile == null ? null : mainDexListFile.getSingleFile()));
            } else {
                mergeTasks.addAll(handleLegacyAndMonoDex(Arrays.asList(mergeDexs[0]), output, outputDir, mainDexListFile == null ? null : mainDexListFile.getSingleFile()));
            }
            mergeTasks.forEach(voidForkJoinTask -> voidForkJoinTask.join());
            cacheHandler.handleMissActionResult(outputDir, in);
            if (output != null) {
                try {
                    outputHandler.handleOutput(output);
                } catch (ProcessException e) {
                // ignore this one
                }
            }
        });
        cacheHandler.handleQueryResult(result, outputDir, "maindex");
    } catch (Exception e) {
        e.printStackTrace();
    }
}

Example 5 with FileCache

use of com.android.builder.utils.FileCache in project atlas by alibaba.

the class DexByteCodeConverterHook method convertByteCode.

// @Override
// public void runDexer(DexProcessBuilder builder, DexOptions dexOptions, ProcessOutputHandler processOutputHandler) throws ProcessException, IOException, InterruptedException {
// builder.addInputs(inputFile);
// super.runDexer(builder,dexOptions,processOutputHandler);
// 
// 
// }
@Override
public void convertByteCode(Collection<File> inputs, File outDexFolder, boolean multidex, final File mainDexList, DexOptions dexOptions, ProcessOutputHandler processOutputHandler, int minSdkVersion) throws IOException, InterruptedException, ProcessException {
    logger.warning("outDexFolder:" + outDexFolder.getAbsolutePath());
    FileUtils.forceMkdir(outDexFolder);
    // outDexFolder.mkdirs();
    AtlasDependencyTree atlasDependencyTree = AtlasBuildContext.androidDependencyTrees.get(variantContext.getVariantName());
    if (null != atlasDependencyTree) {
        ProcessOutputHandler outputHandler = new ParsingProcessOutputHandler(new ToolOutputParser(new DexParser(), Message.Kind.ERROR, LoggerWrapper.getLogger(DexByteCodeConverterHook.class)), new ToolOutputParser(new DexParser(), LoggerWrapper.getLogger(DexByteCodeConverterHook.class)), AtlasBuildContext.androidBuilderMap.get(variantContext.getProject()).getErrorReporter());
        for (final AwbBundle awbBundle : atlasDependencyTree.getAwbBundles()) {
            waitableExecutor.execute((Callable<Void>) () -> {
                try {
                    long start = System.currentTimeMillis();
                    // create dex
                    File dexOutputFile = ((AppVariantContext) variantContext).getAwbDexOutput(awbBundle.getName());
                    if (dexOutputFile.exists()) {
                        FileUtils.cleanDirectory(dexOutputFile);
                    } else {
                        FileUtils.forceMkdir(dexOutputFile);
                    }
                    // if some of our .jar input files exist, just reset the inputDir to null
                    AwbTransform awbTransform = variantOutputContext.getAwbTransformMap().get(awbBundle.getName());
                    List<File> inputFiles = new ArrayList<File>();
                    inputFiles.addAll(awbTransform.getInputFiles());
                    inputFiles.addAll(awbTransform.getInputLibraries());
                    if (null != awbTransform.getInputDirs()) {
                        inputFiles.addAll(awbTransform.getInputDirs());
                    }
                    if (variantContext.getScope().getDexer() == DexerTool.DX) {
                        AtlasBuildContext.androidBuilderMap.get(variantContext.getProject()).convertByteCode(inputFiles, dexOutputFile, false, null, dexOptions, outputHandler, true);
                    } else if (variantContext.getScope().getDexer() == DexerTool.D8) {
                        new AtlasD8Creator(inputFiles, ((AppVariantContext) variantContext).getAwbDexAchiveOutput(awbBundle), multidex, mainDexList, dexOptions, minSdkVersion, fileCache, processOutputHandler, variantContext, variantOutputContext).create(awbBundle);
                    }
                    if (awbBundle.isMBundle) {
                        mBundleSets.add(awbBundle);
                    }
                } catch (Exception e) {
                    throw new ProcessException(awbBundle.getName(), e);
                }
                return null;
            });
        }
    }
    File tempDexFolder = null;
    inputFile.addAll(AtlasBuildContext.atlasMainDexHelperMap.get(variantContext.getVariantName()).getAllMainDexJars());
    inputFile.addAll(AtlasBuildContext.atlasMainDexHelperMap.get(variantContext.getVariantName()).getInputDirs());
    logger.warning("maindex inputFile size :" + inputFile.size());
    if (variantContext.getScope().getDexer() == DexerTool.D8) {
        AtlasD8Creator atlasD8Creator = new AtlasD8Creator(inputs, ((AppVariantContext) variantContext).getMainDexAchive(), multidex, mainDexList, dexOptions, minSdkVersion, fileCache, processOutputHandler, variantContext, variantOutputContext);
        atlasD8Creator.setMainDexOut(outDexFolder);
        atlasD8Creator.create(new AwbBundle());
        return;
    }
    initDexExecutorService(dexOptions);
    if (!multidex) {
        if (fileCache != null && globalCacheBuilder == null) {
            globalCacheBuilder = new FileCache.Inputs.Builder(FileCache.Command.PREDEX_LIBRARY).putBoolean("multidex", false).putLong("minisdk", minSdkVersion).putString("dexoptions", dexOptions.getAdditionalParameters().toString()).putBoolean("jumbomode", dexOptions.getJumboMode()).putString("type", type);
            inputFile = new ArrayList<>(inputFile);
            Collections.sort((List<File>) inputFile);
            FileCache.Inputs inputsKey = globalCacheBuilder.putString("md5", MD5Util.getFileMd5(inputFile)).build();
            try {
                fileCache.createFile(outDexFolder, inputsKey, () -> {
                    logger.warning("dex inputFile missCache: " + inputFile.toString());
                    outDexFolder.mkdirs();
                    DexByteCodeConverterHook.super.convertByteCode(inputFile, outDexFolder, multidex, mainDexList, dexOptions, processOutputHandler, minSdkVersion);
                });
            } catch (ExecutionException e) {
                e.printStackTrace();
                failures.add(e);
            }
        } else {
            DexByteCodeConverterHook.super.convertByteCode(inputFile, outDexFolder, multidex, mainDexList, dexOptions, processOutputHandler, minSdkVersion);
        }
    // try {
    // for (Future future : futureList) {
    // future.get();
    // }
    // } catch (Exception e) {
    // throw new ProcessException(e);
    // }
    } else {
        if (mainDexList != null && !mainDexList.exists()) {
            generateMainDexList(mainDexList);
        }
        tempDexFolder = variantOutputContext.getMainDexOutDir();
        if (tempDexFolder.exists()) {
            FileUtils.cleanDirectory(tempDexFolder);
        }
        File finalTempDexFolder = tempDexFolder;
        if (fileCache != null && globalCacheBuilder == null) {
            if (mainDexList != null) {
                globalCacheBuilder = new FileCache.Inputs.Builder(FileCache.Command.PREDEX_LIBRARY).putBoolean("multidex", true).putFile("multidexlist", mainDexList, FileCache.FileProperties.HASH).putLong("minisdk", minSdkVersion).putString("dexoptions", dexOptions.getAdditionalParameters().toString()).putBoolean("jumbomode", dexOptions.getJumboMode()).putString("type", type);
            } else {
                globalCacheBuilder = new FileCache.Inputs.Builder(FileCache.Command.PREDEX_LIBRARY).putBoolean("multidex", true).putLong("minisdk", minSdkVersion).putString("dexoptions", dexOptions.getAdditionalParameters().toString()).putBoolean("jumbomode", dexOptions.getJumboMode()).putString("type", type);
            }
        }
        if (inputFile.size() == 1) {
            splitFile();
        }
        inputFile.parallelStream().forEach(file -> {
            File outPutFolder = new File(finalTempDexFolder, FilenameUtils.getBaseName(file.getName()));
            if (globalCacheBuilder != null && file.isFile()) {
                FileCache.Inputs.Builder builder = copyOf(globalCacheBuilder);
                FileCache.Inputs cacheInputs = null;
                if (file.isFile()) {
                    cacheInputs = builder.putFile("hash", file, FileCache.FileProperties.HASH).build();
                } else {
                    Collection<File> files = FileUtils.listFiles(file, new String[] { "class" }, true);
                    Collections.sort((List<File>) files);
                    cacheInputs = builder.putString("hash", MD5Util.getFileMd5(files)).build();
                }
                try {
                    fileCache.createFile(outPutFolder, cacheInputs, () -> {
                        logger.warning("maindex inputFile missCache:" + file.getAbsolutePath());
                        outPutFolder.mkdirs();
                        DexByteCodeConverterHook.super.convertByteCode(Arrays.asList(file), outPutFolder, true, mainDexList, dexOptions, processOutputHandler, minSdkVersion);
                    });
                } catch (Exception e) {
                    failures.add(e);
                    e.printStackTrace();
                }
            } else {
                logger.warning("maindex inputFile:" + file.getAbsolutePath());
                outPutFolder.mkdirs();
                try {
                    DexByteCodeConverterHook.super.convertByteCode(Arrays.asList(file), outPutFolder, true, mainDexList, dexOptions, processOutputHandler, minSdkVersion);
                } catch (Exception e) {
                    e.printStackTrace();
                    failures.add(e);
                }
            }
        });
        if (failures.size() > 0) {
            throw new ProcessException(failures.get(0));
        }
        // for (Future future : futureList) {
        // try {
        // future.get();
        // } catch (ExecutionException e) {
        // throw new ProcessException(e);
        // }
        // }
        // 
        // inputFile.stream().parallel().forEach(new Consumer<File>() {
        // @Override
        // public void accept(File file) {
        // fileCache.createFile()fileCacheMap.get(file)
        // }
        // });
        Collection<File> dexFiles = FileUtils.listFiles(tempDexFolder, new String[] { "dex" }, true);
        if (dexFiles != null) {
            logger.warning("maindex outDexFiles size:" + dexFiles.size());
            dexPaths = dexFiles.stream().map(file -> file.toPath()).collect(Collectors.toList());
        }
        mainforkJoinPool = new ForkJoinPool();
        atlasDexArchiveMerger = new AtlasDexArchiveMerger(mainforkJoinPool);
        if (!variantContext.getAtlasExtension().getTBuildConfig().getMergeBundlesDex()) {
            try {
                atlasDexArchiveMerger.mergeDexArchives(dexPaths, outDexFolder.toPath(), mainDexList == null ? null : mainDexList.toPath(), DexingType.LEGACY_MULTIDEX);
            } catch (DexArchiveMergerException e) {
                throw new ProcessException(e);
            }
        }
    }
    waitableExecutor.waitForTasksWithQuickFail(true);
    atomicInteger.set(FileUtils.listFiles(outDexFolder, new String[] { "dex" }, true).size());
    logger.warning("maindex final dexs size:" + atomicInteger.get());
    for (AwbBundle bundle : mBundleSets) {
        File awbDex = new File(((AppVariantContext) variantContext).getAwbDexOutput(bundle.getName()), "classes.dex");
        if (awbDex.exists() && !variantContext.getAtlasExtension().getTBuildConfig().getMergeBundlesDex()) {
            FileUtils.moveFile(awbDex, new File(outDexFolder, "classes" + atomicInteger.incrementAndGet() + ".dex"));
        } else if (awbDex.exists() && variantContext.getAtlasExtension().getTBuildConfig().getMergeBundlesDex()) {
            dexPaths.add(awbDex.toPath());
        } else {
            logger.warning(awbDex.getAbsoluteFile() + " is not exist!");
        }
    }
    if (variantContext.getAtlasExtension().getTBuildConfig().getMergeBundlesDex()) {
        try {
            atlasDexArchiveMerger.mergeDexArchives(dexPaths, outDexFolder.toPath(), null, DexingType.LEGACY_MULTIDEX);
        } catch (DexArchiveMergerException e) {
            e.printStackTrace();
        } finally {
        }
    }
    if (tempDexFolder != null && tempDexFolder.exists()) {
        FileUtils.deleteDirectory(tempDexFolder);
    }
}