Examples with ImmutableMultimap org.apache.beam.vendor.calcite.v1_28_0.com.google.common.collect.ImmutableMultimap used on opensource projects

Example 1 with ImmutableMultimap

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.collect.ImmutableMultimap in project buck by facebook.

the class AndroidBinary method addDexingSteps.

/**
   * Create dex artifacts for all of the individual directories of compiled .class files (or
   * the obfuscated jar files if proguard is used).  If split dex is used, multiple dex artifacts
   * will be produced.
   *  @param classpathEntriesToDex Full set of classpath entries that must make
   *     their way into the final APK structure (but not necessarily into the
   *     primary dex).
   * @param secondaryDexDirectories The contract for updating this builder must match that
   *     of {@link PreDexMerge#getSecondaryDexDirectories()}.
   * @param steps List of steps to add to.
   * @param primaryDexPath Output path for the primary dex file.
   */
@VisibleForTesting
void addDexingSteps(Set<Path> classpathEntriesToDex, Supplier<ImmutableMap<String, HashCode>> classNamesToHashesSupplier, ImmutableSet.Builder<Path> secondaryDexDirectories, ImmutableList.Builder<Step> steps, Path primaryDexPath, Optional<SourcePath> dexReorderToolFile, Optional<SourcePath> dexReorderDataDumpFile, ImmutableMultimap<APKModule, Path> additionalDexStoreToJarPathMap, SourcePathResolver resolver) {
    final Supplier<Set<Path>> primaryInputsToDex;
    final Optional<Path> secondaryDexDir;
    final Optional<Supplier<Multimap<Path, Path>>> secondaryOutputToInputs;
    Path secondaryDexParentDir = getBinPath("__%s_secondary_dex__/");
    Path additionalDexParentDir = getBinPath("__%s_additional_dex__/");
    Path additionalDexAssetsDir = additionalDexParentDir.resolve("assets");
    final Optional<ImmutableSet<Path>> additionalDexDirs;
    if (shouldSplitDex()) {
        Optional<Path> proguardFullConfigFile = Optional.empty();
        Optional<Path> proguardMappingFile = Optional.empty();
        if (packageType.isBuildWithObfuscation()) {
            Path proguardConfigDir = getProguardTextFilesPath();
            proguardFullConfigFile = Optional.of(proguardConfigDir.resolve("configuration.txt"));
            proguardMappingFile = Optional.of(proguardConfigDir.resolve("mapping.txt"));
        }
        // DexLibLoader expects that metadata.txt and secondary jar files are under this dir
        // in assets.
        // Intermediate directory holding the primary split-zip jar.
        Path splitZipDir = getBinPath("__%s_split_zip__");
        steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), splitZipDir));
        Path primaryJarPath = splitZipDir.resolve("primary.jar");
        Path secondaryJarMetaDirParent = splitZipDir.resolve("secondary_meta");
        Path secondaryJarMetaDir = secondaryJarMetaDirParent.resolve(SECONDARY_DEX_SUBDIR);
        steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), secondaryJarMetaDir));
        Path secondaryJarMeta = secondaryJarMetaDir.resolve("metadata.txt");
        // Intermediate directory holding _ONLY_ the secondary split-zip jar files.  This is
        // important because SmartDexingCommand will try to dx every entry in this directory.  It
        // does this because it's impossible to know what outputs split-zip will generate until it
        // runs.
        final Path secondaryZipDir = getBinPath("__%s_secondary_zip__");
        steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), secondaryZipDir));
        // Intermediate directory holding the directories holding _ONLY_ the additional split-zip
        // jar files that are intended for that dex store.
        final Path additionalDexStoresZipDir = getBinPath("__%s_dex_stores_zip__");
        steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), additionalDexStoresZipDir));
        for (APKModule dexStore : additionalDexStoreToJarPathMap.keySet()) {
            steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), additionalDexStoresZipDir.resolve(dexStore.getName())));
            steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), secondaryJarMetaDirParent.resolve("assets").resolve(dexStore.getName())));
        }
        // Run the split-zip command which is responsible for dividing the large set of input
        // classpaths into a more compact set of jar files such that no one jar file when dexed will
        // yield a dex artifact too large for dexopt or the dx method limit to handle.
        Path zipSplitReportDir = getBinPath("__%s_split_zip_report__");
        steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), zipSplitReportDir));
        SplitZipStep splitZipCommand = new SplitZipStep(getProjectFilesystem(), classpathEntriesToDex, secondaryJarMeta, primaryJarPath, secondaryZipDir, "secondary-%d.jar", secondaryJarMetaDirParent, additionalDexStoresZipDir, proguardFullConfigFile, proguardMappingFile, skipProguard, dexSplitMode, dexSplitMode.getPrimaryDexScenarioFile().map(resolver::getAbsolutePath), dexSplitMode.getPrimaryDexClassesFile().map(resolver::getAbsolutePath), dexSplitMode.getSecondaryDexHeadClassesFile().map(resolver::getAbsolutePath), dexSplitMode.getSecondaryDexTailClassesFile().map(resolver::getAbsolutePath), additionalDexStoreToJarPathMap, enhancementResult.getAPKModuleGraph(), zipSplitReportDir);
        steps.add(splitZipCommand);
        // smart dexing command.  Smart dex will handle "cleaning" this directory properly.
        if (reorderClassesIntraDex) {
            secondaryDexDir = Optional.of(secondaryDexParentDir.resolve(SMART_DEX_SECONDARY_DEX_SUBDIR));
            Path intraDexReorderSecondaryDexDir = secondaryDexParentDir.resolve(SECONDARY_DEX_SUBDIR);
            steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), secondaryDexDir.get()));
            steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), intraDexReorderSecondaryDexDir));
        } else {
            secondaryDexDir = Optional.of(secondaryDexParentDir.resolve(SECONDARY_DEX_SUBDIR));
            steps.add(new MkdirStep(getProjectFilesystem(), secondaryDexDir.get()));
        }
        if (additionalDexStoreToJarPathMap.isEmpty()) {
            additionalDexDirs = Optional.empty();
        } else {
            ImmutableSet.Builder<Path> builder = ImmutableSet.builder();
            for (APKModule dexStore : additionalDexStoreToJarPathMap.keySet()) {
                Path dexStorePath = additionalDexAssetsDir.resolve(dexStore.getName());
                builder.add(dexStorePath);
                steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), dexStorePath));
            }
            additionalDexDirs = Optional.of(builder.build());
        }
        if (dexSplitMode.getDexStore() == DexStore.RAW) {
            secondaryDexDirectories.add(secondaryDexDir.get());
        } else {
            secondaryDexDirectories.add(secondaryJarMetaDirParent);
            secondaryDexDirectories.add(secondaryDexParentDir);
        }
        if (additionalDexDirs.isPresent()) {
            secondaryDexDirectories.add(additionalDexParentDir);
        }
        // Adjust smart-dex inputs for the split-zip case.
        primaryInputsToDex = Suppliers.ofInstance(ImmutableSet.of(primaryJarPath));
        Supplier<Multimap<Path, Path>> secondaryOutputToInputsMap = splitZipCommand.getOutputToInputsMapSupplier(secondaryDexDir.get(), additionalDexAssetsDir);
        secondaryOutputToInputs = Optional.of(secondaryOutputToInputsMap);
    } else {
        // Simple case where our inputs are the natural classpath directories and we don't have
        // to worry about secondary jar/dex files.
        primaryInputsToDex = Suppliers.ofInstance(classpathEntriesToDex);
        secondaryDexDir = Optional.empty();
        secondaryOutputToInputs = Optional.empty();
    }
    HashInputJarsToDexStep hashInputJarsToDexStep = new HashInputJarsToDexStep(getProjectFilesystem(), primaryInputsToDex, secondaryOutputToInputs, classNamesToHashesSupplier);
    steps.add(hashInputJarsToDexStep);
    // Stores checksum information from each invocation to intelligently decide when dx needs
    // to be re-run.
    Path successDir = getBinPath("__%s_smart_dex__/.success");
    steps.add(new MkdirStep(getProjectFilesystem(), successDir));
    // Add the smart dexing tool that is capable of avoiding the external dx invocation(s) if
    // it can be shown that the inputs have not changed.  It also parallelizes dx invocations
    // where applicable.
    //
    // Note that by not specifying the number of threads this command will use it will select an
    // optimal default regardless of the value of --num-threads.  This decision was made with the
    // assumption that --num-threads specifies the threading of build rule execution and does not
    // directly apply to the internal threading/parallelization details of various build commands
    // being executed.  For example, aapt is internally threaded by default when preprocessing
    // images.
    EnumSet<DxStep.Option> dxOptions = PackageType.RELEASE.equals(packageType) ? EnumSet.of(DxStep.Option.NO_LOCALS) : EnumSet.of(DxStep.Option.NO_OPTIMIZE);
    Path selectedPrimaryDexPath = primaryDexPath;
    if (reorderClassesIntraDex) {
        String primaryDexFileName = primaryDexPath.getFileName().toString();
        String smartDexPrimaryDexFileName = "smart-dex-" + primaryDexFileName;
        Path smartDexPrimaryDexPath = Paths.get(primaryDexPath.toString().replace(primaryDexFileName, smartDexPrimaryDexFileName));
        selectedPrimaryDexPath = smartDexPrimaryDexPath;
    }
    SmartDexingStep smartDexingCommand = new SmartDexingStep(getProjectFilesystem(), selectedPrimaryDexPath, primaryInputsToDex, secondaryDexDir, secondaryOutputToInputs, hashInputJarsToDexStep, successDir, dxOptions, dxExecutorService, xzCompressionLevel, dxMaxHeapSize);
    steps.add(smartDexingCommand);
    if (reorderClassesIntraDex) {
        IntraDexReorderStep intraDexReorderStep = new IntraDexReorderStep(getProjectFilesystem(), resolver.getAbsolutePath(dexReorderToolFile.get()), resolver.getAbsolutePath(dexReorderDataDumpFile.get()), getBuildTarget(), selectedPrimaryDexPath, primaryDexPath, secondaryOutputToInputs, SMART_DEX_SECONDARY_DEX_SUBDIR, SECONDARY_DEX_SUBDIR);
        steps.add(intraDexReorderStep);
    }
}

Example 2 with ImmutableMultimap

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.collect.ImmutableMultimap in project buck by facebook.

the class AndroidBinaryGraphEnhancer method createPreDexRulesForLibraries.

@VisibleForTesting
ImmutableMultimap<APKModule, DexProducedFromJavaLibrary> createPreDexRulesForLibraries(Iterable<BuildRule> additionalJavaLibrariesToDex, AndroidPackageableCollection packageableCollection) {
    Iterable<BuildTarget> additionalJavaLibraryTargets = FluentIterable.from(additionalJavaLibrariesToDex).transform(BuildRule::getBuildTarget);
    ImmutableMultimap.Builder<APKModule, DexProducedFromJavaLibrary> preDexDeps = ImmutableMultimap.builder();
    for (BuildTarget buildTarget : Iterables.concat(packageableCollection.getJavaLibrariesToDex(), additionalJavaLibraryTargets)) {
        Preconditions.checkState(!buildTargetsToExcludeFromDex.contains(buildTarget), "JavaLibrary should have been excluded from target to dex: %s", buildTarget);
        BuildRule libraryRule = ruleResolver.getRule(buildTarget);
        Preconditions.checkState(libraryRule instanceof JavaLibrary);
        JavaLibrary javaLibrary = (JavaLibrary) libraryRule;
        // resources, but export_deps is true), then there will not be anything to dx.
        if (javaLibrary.getSourcePathToOutput() == null) {
            continue;
        }
        // See whether the corresponding IntermediateDexRule has already been added to the
        // ruleResolver.
        BuildTarget originalTarget = javaLibrary.getBuildTarget();
        BuildTarget preDexTarget = BuildTarget.builder(originalTarget).addFlavors(DEX_FLAVOR).build();
        Optional<BuildRule> preDexRule = ruleResolver.getRuleOptional(preDexTarget);
        if (preDexRule.isPresent()) {
            preDexDeps.put(apkModuleGraph.findModuleForTarget(buildTarget), (DexProducedFromJavaLibrary) preDexRule.get());
            continue;
        }
        // Create the IntermediateDexRule and add it to both the ruleResolver and preDexDeps.
        BuildRuleParams paramsForPreDex = buildRuleParams.withBuildTarget(preDexTarget).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(ImmutableSortedSet.of(ruleResolver.getRule(javaLibrary.getBuildTarget()))), Suppliers.ofInstance(ImmutableSortedSet.of()));
        DexProducedFromJavaLibrary preDex = new DexProducedFromJavaLibrary(paramsForPreDex, javaLibrary);
        ruleResolver.addToIndex(preDex);
        preDexDeps.put(apkModuleGraph.findModuleForTarget(buildTarget), preDex);
    }
    return preDexDeps.build();
}

Example 3 with ImmutableMultimap

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.collect.ImmutableMultimap in project buck by facebook.

the class CompileStringsStep method groupFilesByLocale.

/**
   * Groups a list of strings.xml files by locale.
   * String files with no resource qualifier (eg. values/strings.xml) are mapped to the "en" locale
   *
   * eg. given the following list:
   *
   * ImmutableList.of(
   *   Paths.get("one/res/values-es/strings.xml"),
   *   Paths.get("two/res/values-es/strings.xml"),
   *   Paths.get("three/res/values-pt-rBR/strings.xml"),
   *   Paths.get("four/res/values-pt-rPT/strings.xml"),
   *   Paths.get("five/res/values/strings.xml"));
   *
   * returns:
   *
   * ImmutableMap.of(
   *   "es", ImmutableList.of(Paths.get("one/res/values-es/strings.xml"),
   *        Paths.get("two/res/values-es/strings.xml")),
   *   "pt_BR", ImmutableList.of(Paths.get("three/res/values-pt-rBR/strings.xml'),
   *   "pt_PT", ImmutableList.of(Paths.get("four/res/values-pt-rPT/strings.xml"),
   *   "en", ImmutableList.of(Paths.get("five/res/values/strings.xml")));
   */
@VisibleForTesting
ImmutableMultimap<String, Path> groupFilesByLocale(ImmutableList<Path> files) {
    ImmutableMultimap.Builder<String, Path> localeToFiles = ImmutableMultimap.builder();
    for (Path filepath : files) {
        String path = MorePaths.pathWithUnixSeparators(filepath);
        Matcher matcher = NON_ENGLISH_STRING_FILE_PATTERN.matcher(path);
        if (matcher.matches()) {
            String baseLocale = matcher.group(1);
            String country = matcher.group(2);
            String locale = country == null ? baseLocale : baseLocale + "_" + country;
            if (country != null && !regionSpecificToBaseLocaleMap.containsKey(locale)) {
                regionSpecificToBaseLocaleMap.put(locale, baseLocale);
            }
            localeToFiles.put(locale, filepath);
        } else {
            Preconditions.checkState(path.endsWith(ENGLISH_STRING_PATH_SUFFIX), "Invalid path passed to compile strings: " + path);
            localeToFiles.put(ENGLISH_LOCALE, filepath);
        }
    }
    return localeToFiles.build();
}

Example 4 with ImmutableMultimap

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.collect.ImmutableMultimap in project buck by facebook.

the class WorkspaceAndProjectGenerator method generateProject.

private void generateProject(final Map<Path, ProjectGenerator> projectGenerators, ListeningExecutorService listeningExecutorService, WorkspaceGenerator workspaceGenerator, ImmutableSet<BuildTarget> targetsInRequiredProjects, ImmutableMultimap.Builder<BuildTarget, PBXTarget> buildTargetToPbxTargetMapBuilder, ImmutableMap.Builder<PBXTarget, Path> targetToProjectPathMapBuilder, final Optional<BuildTarget> targetToBuildWithBuck) throws IOException, InterruptedException {
    ImmutableMultimap.Builder<Cell, BuildTarget> projectCellToBuildTargetsBuilder = ImmutableMultimap.builder();
    for (TargetNode<?, ?> targetNode : projectGraph.getNodes()) {
        BuildTarget buildTarget = targetNode.getBuildTarget();
        projectCellToBuildTargetsBuilder.put(rootCell.getCell(buildTarget), buildTarget);
    }
    ImmutableMultimap<Cell, BuildTarget> projectCellToBuildTargets = projectCellToBuildTargetsBuilder.build();
    List<ListenableFuture<GenerationResult>> projectGeneratorFutures = new ArrayList<>();
    for (final Cell projectCell : projectCellToBuildTargets.keySet()) {
        ImmutableMultimap.Builder<Path, BuildTarget> projectDirectoryToBuildTargetsBuilder = ImmutableMultimap.builder();
        final ImmutableSet<BuildTarget> cellRules = ImmutableSet.copyOf(projectCellToBuildTargets.get(projectCell));
        for (BuildTarget buildTarget : cellRules) {
            projectDirectoryToBuildTargetsBuilder.put(buildTarget.getBasePath(), buildTarget);
        }
        ImmutableMultimap<Path, BuildTarget> projectDirectoryToBuildTargets = projectDirectoryToBuildTargetsBuilder.build();
        final Path relativeTargetCell = rootCell.getRoot().relativize(projectCell.getRoot());
        for (final Path projectDirectory : projectDirectoryToBuildTargets.keySet()) {
            final ImmutableSet<BuildTarget> rules = filterRulesForProjectDirectory(projectGraph, ImmutableSet.copyOf(projectDirectoryToBuildTargets.get(projectDirectory)));
            if (Sets.intersection(targetsInRequiredProjects, rules).isEmpty()) {
                continue;
            }
            final boolean isMainProject = workspaceArguments.srcTarget.isPresent() && rules.contains(workspaceArguments.srcTarget.get());
            projectGeneratorFutures.add(listeningExecutorService.submit(() -> {
                GenerationResult result = generateProjectForDirectory(projectGenerators, targetToBuildWithBuck, projectCell, projectDirectory, rules, isMainProject, targetsInRequiredProjects);
                // convert the projectPath to relative to the target cell here
                result = GenerationResult.of(relativeTargetCell.resolve(result.getProjectPath()), result.isProjectGenerated(), result.getRequiredBuildTargets(), result.getBuildTargetToGeneratedTargetMap());
                return result;
            }));
        }
    }
    List<GenerationResult> generationResults;
    try {
        generationResults = Futures.allAsList(projectGeneratorFutures).get();
    } catch (ExecutionException e) {
        Throwables.throwIfInstanceOf(e.getCause(), IOException.class);
        Throwables.throwIfUnchecked(e.getCause());
        throw new IllegalStateException("Unexpected exception: ", e);
    }
    for (GenerationResult result : generationResults) {
        if (!result.isProjectGenerated()) {
            continue;
        }
        workspaceGenerator.addFilePath(result.getProjectPath());
        processGenerationResult(buildTargetToPbxTargetMapBuilder, targetToProjectPathMapBuilder, result);
    }
}

Example 5 with ImmutableMultimap

use of org.apache.beam.vendor.calcite.v1_28_0.com.google.common.collect.ImmutableMultimap in project buck by facebook.

the class TargetsCommand method getDependentNodes.

/**
   * @param graph A graph used to resolve dependencies between targets.
   * @param nodes A set of nodes.
   * @param detectTestChanges If true, tests are considered to be dependencies of the targets they
   *                          are testing.
   * @return A set of all nodes that transitively depend on {@code nodes}
   * (a superset of {@code nodes}).
   */
private static ImmutableSet<TargetNode<?, ?>> getDependentNodes(final TargetGraph graph, ImmutableSet<TargetNode<?, ?>> nodes, boolean detectTestChanges) {
    ImmutableMultimap.Builder<TargetNode<?, ?>, TargetNode<?, ?>> extraEdgesBuilder = ImmutableMultimap.builder();
    if (detectTestChanges) {
        for (TargetNode<?, ?> node : graph.getNodes()) {
            if (node.getConstructorArg() instanceof HasTests) {
                ImmutableSortedSet<BuildTarget> tests = ((HasTests) node.getConstructorArg()).getTests();
                for (BuildTarget testTarget : tests) {
                    Optional<TargetNode<?, ?>> testNode = graph.getOptional(testTarget);
                    if (!testNode.isPresent()) {
                        throw new HumanReadableException("'%s' (test of '%s') is not in the target graph.", testTarget, node);
                    }
                    extraEdgesBuilder.put(testNode.get(), node);
                }
            }
        }
    }
    final ImmutableMultimap<TargetNode<?, ?>, TargetNode<?, ?>> extraEdges = extraEdgesBuilder.build();
    final ImmutableSet.Builder<TargetNode<?, ?>> builder = ImmutableSet.builder();
    AbstractBreadthFirstTraversal<TargetNode<?, ?>> traversal = new AbstractBreadthFirstTraversal<TargetNode<?, ?>>(nodes) {

        @Override
        public ImmutableSet<TargetNode<?, ?>> visit(TargetNode<?, ?> targetNode) {
            builder.add(targetNode);
            return FluentIterable.from(graph.getIncomingNodesFor(targetNode)).append(extraEdges.get(targetNode)).toSet();
        }
    };
    traversal.start();
    return builder.build();
}