Search in sources :

Example 86 with Stream

use of java.util.stream.Stream in project buck by facebook.

the class AndroidBinary method addProguardCommands.

/**
   * @return the resulting set of ProGuarded classpath entries to dex.
   */
@VisibleForTesting
ImmutableSet<Path> addProguardCommands(Set<Path> classpathEntriesToDex, Set<Path> depsProguardConfigs, boolean skipProguard, ImmutableList.Builder<Step> steps, BuildableContext buildableContext, SourcePathResolver resolver) {
    ImmutableSet.Builder<Path> additionalLibraryJarsForProguardBuilder = ImmutableSet.builder();
    for (JavaLibrary buildRule : rulesToExcludeFromDex) {
        additionalLibraryJarsForProguardBuilder.addAll(buildRule.getImmediateClasspaths().stream().map(resolver::getAbsolutePath).collect(MoreCollectors.toImmutableSet()));
    }
    // Create list of proguard Configs for the app project and its dependencies
    ImmutableSet.Builder<Path> proguardConfigsBuilder = ImmutableSet.builder();
    proguardConfigsBuilder.addAll(depsProguardConfigs);
    if (proguardConfig.isPresent()) {
        proguardConfigsBuilder.add(resolver.getAbsolutePath(proguardConfig.get()));
    }
    Path proguardConfigDir = getProguardTextFilesPath();
    // Transform our input classpath to a set of output locations for each input classpath.
    // TODO(jasta): the output path we choose is the result of a slicing function against
    // input classpath. This is fragile and should be replaced with knowledge of the BuildTarget.
    final ImmutableMap<Path, Path> inputOutputEntries = classpathEntriesToDex.stream().collect(MoreCollectors.toImmutableMap(java.util.function.Function.identity(), (path) -> getProguardOutputFromInputClasspath(proguardConfigDir, path)));
    // Run ProGuard on the classpath entries.
    ProGuardObfuscateStep.create(javaRuntimeLauncher, getProjectFilesystem(), proguardJarOverride.isPresent() ? Optional.of(resolver.getAbsolutePath(proguardJarOverride.get())) : Optional.empty(), proguardMaxHeapSize, proguardAgentPath, resolver.getRelativePath(aaptGeneratedProguardConfigFile), proguardConfigsBuilder.build(), sdkProguardConfig, optimizationPasses, proguardJvmArgs, inputOutputEntries, additionalLibraryJarsForProguardBuilder.build(), proguardConfigDir, buildableContext, skipProguard, steps);
    // ProGuard then return the input classes to dex.
    if (skipProguard) {
        return ImmutableSet.copyOf(inputOutputEntries.keySet());
    } else {
        return ImmutableSet.copyOf(inputOutputEntries.values());
    }
}
Also used : Path(java.nio.file.Path) SourcePath(com.facebook.buck.rules.SourcePath) ExplicitBuildTargetSourcePath(com.facebook.buck.rules.ExplicitBuildTargetSourcePath) OptionalCompat(com.facebook.buck.util.OptionalCompat) ExopackageInfo(com.facebook.buck.rules.ExopackageInfo) HasRuntimeDeps(com.facebook.buck.rules.HasRuntimeDeps) CopyStep(com.facebook.buck.step.fs.CopyStep) SourcePathRuleFinder(com.facebook.buck.rules.SourcePathRuleFinder) ImmutableCollection(com.google.common.collect.ImmutableCollection) RichStream(com.facebook.buck.util.RichStream) JavaLibrary(com.facebook.buck.jvm.java.JavaLibrary) MkdirStep(com.facebook.buck.step.fs.MkdirStep) ProjectFilesystem(com.facebook.buck.io.ProjectFilesystem) FluentIterable(com.google.common.collect.FluentIterable) SourcePathResolver(com.facebook.buck.rules.SourcePathResolver) SymlinkFilesIntoDirectoryStep(com.facebook.buck.shell.SymlinkFilesIntoDirectoryStep) Keystore(com.facebook.buck.jvm.java.Keystore) BuildRuleParams(com.facebook.buck.rules.BuildRuleParams) Path(java.nio.file.Path) EnumSet(java.util.EnumSet) SimpleFileVisitor(java.nio.file.SimpleFileVisitor) Function(com.google.common.base.Function) ImmutableSet(com.google.common.collect.ImmutableSet) AddToRuleKey(com.facebook.buck.rules.AddToRuleKey) ImmutableMap(com.google.common.collect.ImmutableMap) BuildableProperties(com.facebook.buck.rules.BuildableProperties) BuildableContext(com.facebook.buck.rules.BuildableContext) Set(java.util.Set) BuildTarget(com.facebook.buck.model.BuildTarget) HasClasspathEntries(com.facebook.buck.jvm.java.HasClasspathEntries) AbstractBuildRule(com.facebook.buck.rules.AbstractBuildRule) FileVisitResult(java.nio.file.FileVisitResult) List(java.util.List) Stream(java.util.stream.Stream) Optional(java.util.Optional) PACKAGING(com.facebook.buck.rules.BuildableProperties.Kind.PACKAGING) StepExecutionResult(com.facebook.buck.step.StepExecutionResult) JavaRuntimeLauncher(com.facebook.buck.jvm.java.JavaRuntimeLauncher) AccumulateClassNamesStep(com.facebook.buck.jvm.java.AccumulateClassNamesStep) Joiner(com.google.common.base.Joiner) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) Iterables(com.google.common.collect.Iterables) Step(com.facebook.buck.step.Step) ResourceCompressionMode(com.facebook.buck.android.ResourcesFilter.ResourceCompressionMode) Supplier(com.google.common.base.Supplier) RepackZipEntriesStep(com.facebook.buck.zip.RepackZipEntriesStep) SourcePath(com.facebook.buck.rules.SourcePath) Multimap(com.google.common.collect.Multimap) BuildRule(com.facebook.buck.rules.BuildRule) AbstractExecutionStep(com.facebook.buck.step.AbstractExecutionStep) ExecutionContext(com.facebook.buck.step.ExecutionContext) ImmutableList(com.google.common.collect.ImmutableList) Files(com.google.common.io.Files) Suppliers(com.google.common.base.Suppliers) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) RedexOptions(com.facebook.buck.android.redex.RedexOptions) ImmutableSortedMap(com.google.common.collect.ImmutableSortedMap) Nullable(javax.annotation.Nullable) SupportsInputBasedRuleKey(com.facebook.buck.rules.keys.SupportsInputBasedRuleKey) MoreCollectors(com.facebook.buck.util.MoreCollectors) ImmutableSortedSet(com.google.common.collect.ImmutableSortedSet) JavaLibraryClasspathProvider(com.facebook.buck.jvm.java.JavaLibraryClasspathProvider) ExplicitBuildTargetSourcePath(com.facebook.buck.rules.ExplicitBuildTargetSourcePath) MakeCleanDirectoryStep(com.facebook.buck.step.fs.MakeCleanDirectoryStep) HashCode(com.google.common.hash.HashCode) ManifestEntries(com.facebook.buck.rules.coercer.ManifestEntries) ReDexStep(com.facebook.buck.android.redex.ReDexStep) IOException(java.io.IOException) BasicFileAttributes(java.nio.file.attribute.BasicFileAttributes) ANDROID(com.facebook.buck.rules.BuildableProperties.Kind.ANDROID) AbstractMap(java.util.AbstractMap) XzStep(com.facebook.buck.step.fs.XzStep) Paths(java.nio.file.Paths) ResourceFilter(com.facebook.buck.android.FilterResourcesStep.ResourceFilter) ZipScrubberStep(com.facebook.buck.zip.ZipScrubberStep) BuildContext(com.facebook.buck.rules.BuildContext) Preconditions(com.google.common.base.Preconditions) VisibleForTesting(com.google.common.annotations.VisibleForTesting) BuildTargets(com.facebook.buck.model.BuildTargets) AbstractGenruleStep(com.facebook.buck.shell.AbstractGenruleStep) ImmutableSet(com.google.common.collect.ImmutableSet) JavaLibrary(com.facebook.buck.jvm.java.JavaLibrary) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Example 87 with Stream

use of java.util.stream.Stream in project elasticsearch by elastic.

the class ClusterService method submitStateUpdateTasks.

/**
     * Submits a batch of cluster state update tasks; submitted updates are guaranteed to be processed together,
     * potentially with more tasks of the same executor.
     *
     * @param source   the source of the cluster state update task
     * @param tasks    a map of update tasks and their corresponding listeners
     * @param config   the cluster state update task configuration
     * @param executor the cluster state update task executor; tasks
     *                 that share the same executor will be executed
     *                 batches on this executor
     * @param <T>      the type of the cluster state update task state
     *
     */
public <T> void submitStateUpdateTasks(final String source, final Map<T, ClusterStateTaskListener> tasks, final ClusterStateTaskConfig config, final ClusterStateTaskExecutor<T> executor) {
    if (!lifecycle.started()) {
        return;
    }
    if (tasks.isEmpty()) {
        return;
    }
    try {
        @SuppressWarnings("unchecked") ClusterStateTaskExecutor<Object> taskExecutor = (ClusterStateTaskExecutor<Object>) executor;
        // convert to an identity map to check for dups based on update tasks semantics of using identity instead of equal
        final IdentityHashMap<Object, ClusterStateTaskListener> tasksIdentity = new IdentityHashMap<>(tasks);
        final List<UpdateTask> updateTasks = tasksIdentity.entrySet().stream().map(entry -> new UpdateTask(source, entry.getKey(), config.priority(), taskExecutor, safe(entry.getValue(), logger))).collect(Collectors.toList());
        synchronized (updateTasksPerExecutor) {
            LinkedHashSet<UpdateTask> existingTasks = updateTasksPerExecutor.computeIfAbsent(executor, k -> new LinkedHashSet<>(updateTasks.size()));
            for (UpdateTask existing : existingTasks) {
                if (tasksIdentity.containsKey(existing.task)) {
                    throw new IllegalStateException("task [" + taskExecutor.describeTasks(Collections.singletonList(existing.task)) + "] with source [" + source + "] is already queued");
                }
            }
            existingTasks.addAll(updateTasks);
        }
        final UpdateTask firstTask = updateTasks.get(0);
        final TimeValue timeout = config.timeout();
        if (timeout != null) {
            threadPoolExecutor.execute(firstTask, threadPool.scheduler(), timeout, () -> onTimeout(updateTasks, source, timeout));
        } else {
            threadPoolExecutor.execute(firstTask);
        }
    } catch (EsRejectedExecutionException e) {
        // to be done here...
        if (!lifecycle.stoppedOrClosed()) {
            throw e;
        }
    }
}
Also used : MetaData(org.elasticsearch.cluster.metadata.MetaData) ScheduledFuture(java.util.concurrent.ScheduledFuture) Nullable(org.elasticsearch.common.Nullable) Property(org.elasticsearch.common.settings.Setting.Property) ConcurrentCollections(org.elasticsearch.common.util.concurrent.ConcurrentCollections) UnaryOperator(java.util.function.UnaryOperator) ClusterBlocks(org.elasticsearch.cluster.block.ClusterBlocks) ClusterStateListener(org.elasticsearch.cluster.ClusterStateListener) ClusterTasksResult(org.elasticsearch.cluster.ClusterStateTaskExecutor.ClusterTasksResult) ClusterState(org.elasticsearch.cluster.ClusterState) Future(java.util.concurrent.Future) Settings(org.elasticsearch.common.settings.Settings) ClusterBlock(org.elasticsearch.cluster.block.ClusterBlock) Locale(java.util.Locale) Map(java.util.Map) CountDown(org.elasticsearch.common.util.concurrent.CountDown) ThreadPool(org.elasticsearch.threadpool.ThreadPool) ClusterName(org.elasticsearch.cluster.ClusterName) ClusterStateObserver(org.elasticsearch.cluster.ClusterStateObserver) Priority(org.elasticsearch.common.Priority) IdentityHashMap(java.util.IdentityHashMap) Setting(org.elasticsearch.common.settings.Setting) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ClusterChangedEvent(org.elasticsearch.cluster.ClusterChangedEvent) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) Logger(org.apache.logging.log4j.Logger) Stream(java.util.stream.Stream) PrioritizedEsThreadPoolExecutor(org.elasticsearch.common.util.concurrent.PrioritizedEsThreadPoolExecutor) Supplier(org.apache.logging.log4j.util.Supplier) Queue(java.util.Queue) EsExecutors.daemonThreadFactory(org.elasticsearch.common.util.concurrent.EsExecutors.daemonThreadFactory) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) PrioritizedRunnable(org.elasticsearch.common.util.concurrent.PrioritizedRunnable) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) ParameterizedMessage(org.apache.logging.log4j.message.ParameterizedMessage) AtomicReference(java.util.concurrent.atomic.AtomicReference) ProcessClusterEventTimeoutException(org.elasticsearch.cluster.metadata.ProcessClusterEventTimeoutException) ArrayList(java.util.ArrayList) DiscoveryNode(org.elasticsearch.cluster.node.DiscoveryNode) ClusterStateTaskListener(org.elasticsearch.cluster.ClusterStateTaskListener) Text(org.elasticsearch.common.text.Text) TimeValue(org.elasticsearch.common.unit.TimeValue) Iterables(org.elasticsearch.common.util.iterable.Iterables) BiConsumer(java.util.function.BiConsumer) ClusterStateApplier(org.elasticsearch.cluster.ClusterStateApplier) LinkedHashSet(java.util.LinkedHashSet) TimeoutClusterStateListener(org.elasticsearch.cluster.TimeoutClusterStateListener) Loggers(org.elasticsearch.common.logging.Loggers) Builder(org.elasticsearch.cluster.ClusterState.Builder) DiscoveryNodes(org.elasticsearch.cluster.node.DiscoveryNodes) OperationRouting(org.elasticsearch.cluster.routing.OperationRouting) EsExecutors(org.elasticsearch.common.util.concurrent.EsExecutors) FutureUtils(org.elasticsearch.common.util.concurrent.FutureUtils) Iterator(java.util.Iterator) Executor(java.util.concurrent.Executor) Discovery(org.elasticsearch.discovery.Discovery) DiscoverySettings(org.elasticsearch.discovery.DiscoverySettings) ClusterStateTaskConfig(org.elasticsearch.cluster.ClusterStateTaskConfig) ClusterStateTaskExecutor(org.elasticsearch.cluster.ClusterStateTaskExecutor) AbstractLifecycleComponent(org.elasticsearch.common.component.AbstractLifecycleComponent) TimeUnit(java.util.concurrent.TimeUnit) LocalNodeMasterListener(org.elasticsearch.cluster.LocalNodeMasterListener) NodeConnectionsService(org.elasticsearch.cluster.NodeConnectionsService) ClusterSettings(org.elasticsearch.common.settings.ClusterSettings) EsRejectedExecutionException(org.elasticsearch.common.util.concurrent.EsRejectedExecutionException) RoutingTable(org.elasticsearch.cluster.routing.RoutingTable) AckedClusterStateTaskListener(org.elasticsearch.cluster.AckedClusterStateTaskListener) Collections(java.util.Collections) IdentityHashMap(java.util.IdentityHashMap) ClusterStateTaskExecutor(org.elasticsearch.cluster.ClusterStateTaskExecutor) TimeValue(org.elasticsearch.common.unit.TimeValue) ClusterStateTaskListener(org.elasticsearch.cluster.ClusterStateTaskListener) AckedClusterStateTaskListener(org.elasticsearch.cluster.AckedClusterStateTaskListener) EsRejectedExecutionException(org.elasticsearch.common.util.concurrent.EsRejectedExecutionException)

Example 88 with Stream

use of java.util.stream.Stream in project buck by facebook.

the class IjModuleGraph method from.

/**
   * @param projectConfig the project config used
   * @param targetGraph input graph.
   * @param libraryFactory library factory.
   * @param moduleFactory module factory.
   * @param aggregationMode module aggregation mode.
   * @return module graph corresponding to the supplied {@link TargetGraph}. Multiple targets from
   * the same base path are mapped to a single module, therefore an IjModuleGraph edge
   * exists between two modules (Ma, Mb) if a TargetGraph edge existed between a pair of
   * nodes (Ta, Tb) and Ma contains Ta and Mb contains Tb.
   */
public static IjModuleGraph from(final IjProjectConfig projectConfig, final TargetGraph targetGraph, final IjLibraryFactory libraryFactory, final IjModuleFactory moduleFactory, AggregationMode aggregationMode) {
    final ImmutableMap<BuildTarget, IjModule> rulesToModules = createModules(projectConfig, targetGraph, moduleFactory, aggregationMode.getGraphMinimumDepth(targetGraph.getNodes().size()));
    final ExportedDepsClosureResolver exportedDepsClosureResolver = new ExportedDepsClosureResolver(targetGraph);
    ImmutableMap.Builder<IjProjectElement, ImmutableMap<IjProjectElement, DependencyType>> depsBuilder = ImmutableMap.builder();
    final Set<IjLibrary> referencedLibraries = new HashSet<>();
    for (final IjModule module : ImmutableSet.copyOf(rulesToModules.values())) {
        Map<IjProjectElement, DependencyType> moduleDeps = new HashMap<>();
        for (Map.Entry<BuildTarget, DependencyType> entry : module.getDependencies().entrySet()) {
            BuildTarget depBuildTarget = entry.getKey();
            DependencyType depType = entry.getValue();
            ImmutableSet<IjProjectElement> depElements;
            if (depType.equals(DependencyType.COMPILED_SHADOW)) {
                TargetNode<?, ?> targetNode = targetGraph.get(depBuildTarget);
                Optional<IjLibrary> library = libraryFactory.getLibrary(targetNode);
                if (library.isPresent()) {
                    depElements = ImmutableSet.of(library.get());
                } else {
                    depElements = ImmutableSet.of();
                }
            } else {
                depElements = Stream.concat(exportedDepsClosureResolver.getExportedDepsClosure(depBuildTarget).stream(), Stream.of(depBuildTarget)).filter(input -> {
                    TargetNode<?, ?> targetNode = targetGraph.get(input);
                    return !module.getTargets().contains(targetNode);
                }).map(depTarget -> {
                    IjModule depModule = rulesToModules.get(depTarget);
                    if (depModule != null) {
                        return depModule;
                    }
                    TargetNode<?, ?> targetNode = targetGraph.get(depTarget);
                    return libraryFactory.getLibrary(targetNode).orElse(null);
                }).filter(Objects::nonNull).collect(MoreCollectors.toImmutableSet());
            }
            for (IjProjectElement depElement : depElements) {
                Preconditions.checkState(!depElement.equals(module));
                DependencyType.putWithMerge(moduleDeps, depElement, depType);
            }
        }
        if (!module.getExtraClassPathDependencies().isEmpty()) {
            IjLibrary extraClassPathLibrary = IjLibrary.builder().setClassPaths(module.getExtraClassPathDependencies()).setTargets(ImmutableSet.of()).setName("library_" + module.getName() + "_extra_classpath").build();
            moduleDeps.put(extraClassPathLibrary, DependencyType.PROD);
        }
        moduleDeps.keySet().stream().filter(dep -> dep instanceof IjLibrary).map(library -> (IjLibrary) library).forEach(referencedLibraries::add);
        depsBuilder.put(module, ImmutableMap.copyOf(moduleDeps));
    }
    referencedLibraries.forEach(library -> depsBuilder.put(library, ImmutableMap.of()));
    return new IjModuleGraph(depsBuilder.build());
}
Also used : ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) TargetGraph(com.facebook.buck.rules.TargetGraph) TargetNode(com.facebook.buck.rules.TargetNode) Set(java.util.Set) HashMap(java.util.HashMap) JavacOptions(com.facebook.buck.jvm.java.JavacOptions) BuildTarget(com.facebook.buck.model.BuildTarget) AndroidResourceDescription(com.facebook.buck.android.AndroidResourceDescription) HashSet(java.util.HashSet) Objects(java.util.Objects) Stream(java.util.stream.Stream) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Map(java.util.Map) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) JavaLibraryDescription(com.facebook.buck.jvm.java.JavaLibraryDescription) Path(java.nio.file.Path) Nullable(javax.annotation.Nullable) MoreCollectors(com.facebook.buck.util.MoreCollectors) TargetNode(com.facebook.buck.rules.TargetNode) HashMap(java.util.HashMap) ImmutableMap(com.google.common.collect.ImmutableMap) BuildTarget(com.facebook.buck.model.BuildTarget) Objects(java.util.Objects) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) Map(java.util.Map) HashSet(java.util.HashSet)

Example 89 with Stream

use of java.util.stream.Stream in project buck by facebook.

the class HaskellDescriptionUtils method createLinkRule.

/**
   * Create a Haskell link rule that links the given inputs to a executable or shared library and
   * pulls in transitive native linkable deps from the given dep roots.
   */
public static HaskellLinkRule createLinkRule(BuildTarget target, BuildRuleParams baseParams, BuildRuleResolver resolver, SourcePathRuleFinder ruleFinder, CxxPlatform cxxPlatform, HaskellConfig haskellConfig, Linker.LinkType linkType, ImmutableList<String> extraFlags, Iterable<Arg> linkerInputs, Iterable<? extends NativeLinkable> deps, Linker.LinkableDepType depType) throws NoSuchBuildTargetException {
    Tool linker = haskellConfig.getLinker().resolve(resolver);
    String name = target.getShortName();
    ImmutableList.Builder<Arg> linkerArgsBuilder = ImmutableList.builder();
    ImmutableList.Builder<Arg> argsBuilder = ImmutableList.builder();
    // Add the base flags from the `.buckconfig` first.
    argsBuilder.addAll(StringArg.from(haskellConfig.getLinkerFlags()));
    // Pass in the appropriate flags to link a shared library.
    if (linkType.equals(Linker.LinkType.SHARED)) {
        name = CxxDescriptionEnhancer.getSharedLibrarySoname(Optional.empty(), target.withFlavors(), cxxPlatform);
        argsBuilder.addAll(StringArg.from("-shared", "-dynamic"));
        argsBuilder.addAll(StringArg.from(MoreIterables.zipAndConcat(Iterables.cycle("-optl"), cxxPlatform.getLd().resolve(resolver).soname(name))));
    }
    // Add in extra flags passed into this function.
    argsBuilder.addAll(StringArg.from(extraFlags));
    // We pass in the linker inputs and all native linkable deps by prefixing with `-optl` so that
    // the args go straight to the linker, and preserve their order.
    linkerArgsBuilder.addAll(linkerInputs);
    for (NativeLinkable nativeLinkable : NativeLinkables.getNativeLinkables(cxxPlatform, deps, depType).values()) {
        linkerArgsBuilder.addAll(NativeLinkables.getNativeLinkableInput(cxxPlatform, depType, nativeLinkable).getArgs());
    }
    // Since we use `-optl` to pass all linker inputs directly to the linker, the haskell linker
    // will complain about not having any input files.  So, create a dummy archive with an empty
    // module and pass that in normally to work around this.
    BuildTarget emptyModuleTarget = target.withAppendedFlavors(InternalFlavor.of("empty-module"));
    WriteFile emptyModule = resolver.addToIndex(new WriteFile(baseParams.withBuildTarget(emptyModuleTarget), "module Unused where", BuildTargets.getGenPath(baseParams.getProjectFilesystem(), emptyModuleTarget, "%s/Unused.hs"), /* executable */
    false));
    HaskellCompileRule emptyCompiledModule = resolver.addToIndex(createCompileRule(target.withAppendedFlavors(InternalFlavor.of("empty-compiled-module")), baseParams, resolver, ruleFinder, // Buck dependency.
    baseParams.getDeps(), cxxPlatform, haskellConfig, depType, Optional.empty(), Optional.empty(), ImmutableList.of(), HaskellSources.builder().putModuleMap("Unused", emptyModule.getSourcePathToOutput()).build()));
    BuildTarget emptyArchiveTarget = target.withAppendedFlavors(InternalFlavor.of("empty-archive"));
    Archive emptyArchive = resolver.addToIndex(Archive.from(emptyArchiveTarget, baseParams, ruleFinder, cxxPlatform, Archive.Contents.NORMAL, BuildTargets.getGenPath(baseParams.getProjectFilesystem(), emptyArchiveTarget, "%s/libempty.a"), emptyCompiledModule.getObjects()));
    argsBuilder.add(SourcePathArg.of(emptyArchive.getSourcePathToOutput()));
    ImmutableList<Arg> args = argsBuilder.build();
    ImmutableList<Arg> linkerArgs = linkerArgsBuilder.build();
    return resolver.addToIndex(new HaskellLinkRule(baseParams.withBuildTarget(target).copyReplacingDeclaredAndExtraDeps(Suppliers.ofInstance(ImmutableSortedSet.<BuildRule>naturalOrder().addAll(linker.getDeps(ruleFinder)).addAll(Stream.of(args, linkerArgs).flatMap(Collection::stream).flatMap(arg -> arg.getDeps(ruleFinder).stream()).iterator()).build()), Suppliers.ofInstance(ImmutableSortedSet.of())), linker, name, args, linkerArgs, haskellConfig.shouldCacheLinks()));
}
Also used : NativeLinkables(com.facebook.buck.cxx.NativeLinkables) Iterables(com.google.common.collect.Iterables) Linker(com.facebook.buck.cxx.Linker) SourcePathArg(com.facebook.buck.rules.args.SourcePathArg) SourcePathRuleFinder(com.facebook.buck.rules.SourcePathRuleFinder) Archive(com.facebook.buck.cxx.Archive) SourcePath(com.facebook.buck.rules.SourcePath) InternalFlavor(com.facebook.buck.model.InternalFlavor) ExplicitCxxToolFlags(com.facebook.buck.cxx.ExplicitCxxToolFlags) BuildRule(com.facebook.buck.rules.BuildRule) CxxSourceTypes(com.facebook.buck.cxx.CxxSourceTypes) AbstractBreadthFirstThrowingTraversal(com.facebook.buck.graph.AbstractBreadthFirstThrowingTraversal) Tool(com.facebook.buck.rules.Tool) ImmutableList(com.google.common.collect.ImmutableList) CxxPlatforms(com.facebook.buck.cxx.CxxPlatforms) CxxPreprocessables(com.facebook.buck.cxx.CxxPreprocessables) NoSuchBuildTargetException(com.facebook.buck.parser.NoSuchBuildTargetException) StringArg(com.facebook.buck.rules.args.StringArg) Map(java.util.Map) CxxToolFlags(com.facebook.buck.cxx.CxxToolFlags) WriteFile(com.facebook.buck.file.WriteFile) NativeLinkable(com.facebook.buck.cxx.NativeLinkable) Suppliers(com.google.common.base.Suppliers) BuildRuleParams(com.facebook.buck.rules.BuildRuleParams) CxxDescriptionEnhancer(com.facebook.buck.cxx.CxxDescriptionEnhancer) ImmutableSortedMap(com.google.common.collect.ImmutableSortedMap) ImmutableSortedSet(com.google.common.collect.ImmutableSortedSet) CxxSourceRuleFactory(com.facebook.buck.cxx.CxxSourceRuleFactory) ImmutableSet(com.google.common.collect.ImmutableSet) Collection(java.util.Collection) CxxPlatform(com.facebook.buck.cxx.CxxPlatform) CxxSource(com.facebook.buck.cxx.CxxSource) BuildTarget(com.facebook.buck.model.BuildTarget) Arg(com.facebook.buck.rules.args.Arg) Stream(java.util.stream.Stream) TreeMap(java.util.TreeMap) CxxPreprocessorInput(com.facebook.buck.cxx.CxxPreprocessorInput) PreprocessorFlags(com.facebook.buck.cxx.PreprocessorFlags) Optional(java.util.Optional) BuildRuleResolver(com.facebook.buck.rules.BuildRuleResolver) BuildTargets(com.facebook.buck.model.BuildTargets) MoreIterables(com.facebook.buck.util.MoreIterables) WriteFile(com.facebook.buck.file.WriteFile) Archive(com.facebook.buck.cxx.Archive) ImmutableList(com.google.common.collect.ImmutableList) NativeLinkable(com.facebook.buck.cxx.NativeLinkable) BuildTarget(com.facebook.buck.model.BuildTarget) SourcePathArg(com.facebook.buck.rules.args.SourcePathArg) StringArg(com.facebook.buck.rules.args.StringArg) Arg(com.facebook.buck.rules.args.Arg) Collection(java.util.Collection) Tool(com.facebook.buck.rules.Tool)

Example 90 with Stream

use of java.util.stream.Stream in project buck by facebook.

the class CachingBuildEngine method processBuildRule.

private ListenableFuture<BuildResult> processBuildRule(BuildRule rule, BuildEngineBuildContext buildContext, ExecutionContext executionContext, ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks) {
    final RuleKeyFactories keyFactories = ruleKeyFactories.apply(rule.getProjectFilesystem());
    final OnDiskBuildInfo onDiskBuildInfo = buildContext.createOnDiskBuildInfoFor(rule.getBuildTarget(), rule.getProjectFilesystem());
    final BuildInfoRecorder buildInfoRecorder = buildContext.createBuildInfoRecorder(rule.getBuildTarget(), rule.getProjectFilesystem()).addBuildMetadata(BuildInfo.MetadataKey.RULE_KEY, keyFactories.getDefaultRuleKeyFactory().build(rule).toString());
    final BuildableContext buildableContext = new DefaultBuildableContext(buildInfoRecorder);
    final AtomicReference<Long> outputSize = Atomics.newReference();
    ListenableFuture<BuildResult> buildResult = processBuildRule(rule, buildContext, executionContext, onDiskBuildInfo, buildInfoRecorder, buildableContext, asyncCallbacks);
    // materialized locally by chaining up to our result future.
    if (buildMode == BuildMode.DEEP || buildMode == BuildMode.POPULATE_FROM_REMOTE_CACHE) {
        buildResult = MoreFutures.chainExceptions(getDepResults(rule, buildContext, executionContext, asyncCallbacks), buildResult);
    }
    // Setup a callback to handle either the cached or built locally cases.
    AsyncFunction<BuildResult, BuildResult> callback = input -> {
        if (input.getStatus() != BuildRuleStatus.SUCCESS) {
            return Futures.immediateFuture(input);
        }
        BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
        if (success != BuildRuleSuccessType.BUILT_LOCALLY) {
            for (String str : onDiskBuildInfo.getValuesOrThrow(BuildInfo.MetadataKey.RECORDED_PATHS)) {
                buildInfoRecorder.recordArtifact(Paths.get(str));
            }
        }
        outputSize.set(buildInfoRecorder.getOutputSize());
        if (success.outputsHaveChanged()) {
            if (rule instanceof HasPostBuildSteps) {
                executePostBuildSteps(rule, ((HasPostBuildSteps) rule).getPostBuildSteps(buildContext.getBuildContext()), executionContext);
            }
            for (Path path : buildInfoRecorder.getRecordedPaths()) {
                fileHashCache.invalidate(rule.getProjectFilesystem().resolve(path));
            }
        }
        if (useDependencyFileRuleKey(rule) && success == BuildRuleSuccessType.BUILT_LOCALLY) {
            ImmutableList<SourcePath> inputs = ((SupportsDependencyFileRuleKey) rule).getInputsAfterBuildingLocally(buildContext.getBuildContext());
            ImmutableList<String> inputStrings = inputs.stream().map(inputString -> DependencyFileEntry.fromSourcePath(inputString, pathResolver)).map(MoreFunctions.toJsonFunction(objectMapper)).collect(MoreCollectors.toImmutableList());
            buildInfoRecorder.addMetadata(BuildInfo.MetadataKey.DEP_FILE, inputStrings);
            Optional<RuleKeyAndInputs> depFileRuleKeyAndInputs = calculateDepFileRuleKey(rule, buildContext, Optional.of(inputStrings), false);
            if (depFileRuleKeyAndInputs.isPresent()) {
                RuleKey depFileRuleKey = depFileRuleKeyAndInputs.get().getRuleKey();
                buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY, depFileRuleKey.toString());
                if (useManifestCaching(rule)) {
                    Optional<RuleKeyAndInputs> manifestKey = calculateManifestKey(rule, buildContext.getEventBus());
                    if (manifestKey.isPresent()) {
                        buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.MANIFEST_KEY, manifestKey.get().getRuleKey().toString());
                        updateAndStoreManifest(rule, depFileRuleKeyAndInputs.get().getRuleKey(), depFileRuleKeyAndInputs.get().getInputs(), manifestKey.get(), buildContext.getArtifactCache());
                    }
                }
            }
        }
        if (success == BuildRuleSuccessType.BUILT_LOCALLY && shouldUploadToCache(rule, Preconditions.checkNotNull(outputSize.get()))) {
            ImmutableSortedMap.Builder<String, String> outputHashes = ImmutableSortedMap.naturalOrder();
            for (Path path : buildInfoRecorder.getOutputPaths()) {
                outputHashes.put(path.toString(), fileHashCache.get(rule.getProjectFilesystem().resolve(path)).toString());
            }
            buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.RECORDED_PATH_HASHES, outputHashes.build());
        }
        if (success != BuildRuleSuccessType.BUILT_LOCALLY && success.outputsHaveChanged()) {
            Optional<ImmutableMap<String, String>> hashes = onDiskBuildInfo.getBuildMap(BuildInfo.MetadataKey.RECORDED_PATH_HASHES);
            if (hashes.isPresent() && verifyRecordedPathHashes(rule.getBuildTarget(), rule.getProjectFilesystem(), hashes.get())) {
                for (Map.Entry<String, String> ent : hashes.get().entrySet()) {
                    Path path = rule.getProjectFilesystem().getPath(ent.getKey());
                    HashCode hashCode = HashCode.fromString(ent.getValue());
                    fileHashCache.set(rule.getProjectFilesystem().resolve(path), hashCode);
                }
            }
        }
        buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.TARGET, rule.getBuildTarget().toString());
        buildInfoRecorder.addMetadata(BuildInfo.MetadataKey.RECORDED_PATHS, buildInfoRecorder.getRecordedPaths().stream().map(Object::toString).collect(MoreCollectors.toImmutableList()));
        if (success.shouldWriteRecordedMetadataToDiskAfterBuilding()) {
            try {
                boolean clearExistingMetadata = success.shouldClearAndOverwriteMetadataOnDisk();
                buildInfoRecorder.writeMetadataToDisk(clearExistingMetadata);
            } catch (IOException e) {
                throw new IOException(String.format("Failed to write metadata to disk for %s.", rule), e);
            }
        }
        try {
            if (rule instanceof InitializableFromDisk) {
                doInitializeFromDisk((InitializableFromDisk<?>) rule, onDiskBuildInfo);
            }
        } catch (IOException e) {
            throw new IOException(String.format("Error initializing %s from disk.", rule), e);
        }
        return Futures.immediateFuture(input);
    };
    buildResult = Futures.transformAsync(buildResult, ruleAsyncFunction(rule, buildContext.getEventBus(), callback), serviceByAdjustingDefaultWeightsTo(RULE_KEY_COMPUTATION_RESOURCE_AMOUNTS));
    // Handle either build success or failure.
    final SettableFuture<BuildResult> result = SettableFuture.create();
    asyncCallbacks.add(MoreFutures.addListenableCallback(buildResult, new FutureCallback<BuildResult>() {

        // TODO(bolinfest): Delete all files produced by the rule, as they are not guaranteed
        // to be valid at this point?
        private void cleanupAfterError() {
            try {
                onDiskBuildInfo.deleteExistingMetadata();
            } catch (Throwable t) {
                buildContext.getEventBus().post(ThrowableConsoleEvent.create(t, "Error when deleting metadata for %s.", rule));
            }
        }

        private void uploadToCache(BuildRuleSuccessType success) {
            // Collect up all the rule keys we have index the artifact in the cache with.
            Set<RuleKey> ruleKeys = Sets.newHashSet();
            // If the rule key has changed (and is not already in the cache), we need to push
            // the artifact to cache using the new key.
            ruleKeys.add(keyFactories.getDefaultRuleKeyFactory().build(rule));
            // using the new key.
            if (SupportsInputBasedRuleKey.isSupported(rule)) {
                Optional<RuleKey> calculatedRuleKey = calculateInputBasedRuleKey(rule, buildContext.getEventBus());
                Optional<RuleKey> onDiskRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY);
                Optional<RuleKey> metaDataRuleKey = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY).map(RuleKey::new);
                Preconditions.checkState(calculatedRuleKey.equals(onDiskRuleKey), "%s (%s): %s: invalid on-disk input-based rule key: %s != %s", rule.getBuildTarget(), rule.getType(), success, calculatedRuleKey, onDiskRuleKey);
                Preconditions.checkState(calculatedRuleKey.equals(metaDataRuleKey), "%s: %s: invalid meta-data input-based rule key: %s != %s", rule.getBuildTarget(), success, calculatedRuleKey, metaDataRuleKey);
                ruleKeys.addAll(OptionalCompat.asSet(calculatedRuleKey));
            }
            // using the new key.
            if (useManifestCaching(rule)) {
                Optional<RuleKey> onDiskRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY);
                Optional<RuleKey> metaDataRuleKey = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY).map(RuleKey::new);
                Preconditions.checkState(onDiskRuleKey.equals(metaDataRuleKey), "%s: %s: inconsistent meta-data and on-disk dep-file rule key: %s != %s", rule.getBuildTarget(), success, onDiskRuleKey, metaDataRuleKey);
                ruleKeys.addAll(OptionalCompat.asSet(onDiskRuleKey));
            }
            // Do the actual upload.
            try {
                // Verify that the recorded path hashes are accurate.
                Optional<String> recordedPathHashes = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.RECORDED_PATH_HASHES);
                if (recordedPathHashes.isPresent() && !verifyRecordedPathHashes(rule.getBuildTarget(), rule.getProjectFilesystem(), recordedPathHashes.get())) {
                    return;
                }
                // Push to cache.
                buildInfoRecorder.performUploadToArtifactCache(ImmutableSet.copyOf(ruleKeys), buildContext.getArtifactCache(), buildContext.getEventBus());
            } catch (Throwable t) {
                buildContext.getEventBus().post(ThrowableConsoleEvent.create(t, "Error uploading to cache for %s.", rule));
            }
        }

        private void handleResult(BuildResult input) {
            Optional<Long> outputSize = Optional.empty();
            Optional<HashCode> outputHash = Optional.empty();
            Optional<BuildRuleSuccessType> successType = Optional.empty();
            BuildRuleEvent.Resumed resumedEvent = BuildRuleEvent.resumed(rule, buildRuleDurationTracker, keyFactories.getDefaultRuleKeyFactory());
            buildContext.getEventBus().logVerboseAndPost(LOG, resumedEvent);
            if (input.getStatus() == BuildRuleStatus.FAIL) {
                // Make this failure visible for other rules, so that they can stop early.
                firstFailure = input.getFailure();
                // If we failed, cleanup the state of this rule.
                cleanupAfterError();
            }
            // Unblock dependents.
            result.set(input);
            if (input.getStatus() == BuildRuleStatus.SUCCESS) {
                BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
                successType = Optional.of(success);
                // Try get the output size.
                try {
                    outputSize = Optional.of(buildInfoRecorder.getOutputSize());
                } catch (IOException e) {
                    buildContext.getEventBus().post(ThrowableConsoleEvent.create(e, "Error getting output size for %s.", rule));
                }
                // If this rule is cacheable, upload it to the cache.
                if (success.shouldUploadResultingArtifact() && outputSize.isPresent() && shouldUploadToCache(rule, outputSize.get())) {
                    uploadToCache(success);
                }
                // Calculate the hash of outputs that were built locally and are cacheable.
                if (success == BuildRuleSuccessType.BUILT_LOCALLY && shouldUploadToCache(rule, outputSize.get())) {
                    try {
                        outputHash = Optional.of(buildInfoRecorder.getOutputHash(fileHashCache));
                    } catch (IOException e) {
                        buildContext.getEventBus().post(ThrowableConsoleEvent.create(e, "Error getting output hash for %s.", rule));
                    }
                }
            }
            // Log the result to the event bus.
            BuildRuleEvent.Finished finished = BuildRuleEvent.finished(resumedEvent, BuildRuleKeys.builder().setRuleKey(keyFactories.getDefaultRuleKeyFactory().build(rule)).setInputRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY)).setDepFileRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY)).setManifestRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.MANIFEST_KEY)).build(), input.getStatus(), input.getCacheResult(), successType, outputHash, outputSize);
            buildContext.getEventBus().logVerboseAndPost(LOG, finished);
        }

        @Override
        public void onSuccess(BuildResult input) {
            handleResult(input);
        }

        @Override
        public void onFailure(@Nonnull Throwable thrown) {
            thrown = maybeAttachBuildRuleNameToException(thrown, rule);
            handleResult(BuildResult.failure(rule, thrown));
            // Reset interrupted flag once failure has been recorded.
            if (thrown instanceof InterruptedException) {
                Thread.currentThread().interrupt();
            }
        }
    }));
    return result;
}
Also used : OptionalCompat(com.facebook.buck.util.OptionalCompat) NoSuchFileException(java.nio.file.NoSuchFileException) GZIPInputStream(java.util.zip.GZIPInputStream) ArtifactCache(com.facebook.buck.artifact_cache.ArtifactCache) BufferedInputStream(java.io.BufferedInputStream) StepRunner(com.facebook.buck.step.StepRunner) BuckEvent(com.facebook.buck.event.BuckEvent) RuleKeyCalculationEvent(com.facebook.buck.event.RuleKeyCalculationEvent) ObjectMappers(com.facebook.buck.util.ObjectMappers) MoreFutures(com.facebook.buck.util.concurrent.MoreFutures) SettableFuture(com.google.common.util.concurrent.SettableFuture) ArtifactCompressionEvent(com.facebook.buck.event.ArtifactCompressionEvent) ProjectFilesystem(com.facebook.buck.io.ProjectFilesystem) RuleKeyAndInputs(com.facebook.buck.rules.keys.RuleKeyAndInputs) Map(java.util.Map) RuleKeyFactories(com.facebook.buck.rules.keys.RuleKeyFactories) TypeReference(com.fasterxml.jackson.core.type.TypeReference) ArtifactInfo(com.facebook.buck.artifact_cache.ArtifactInfo) Path(java.nio.file.Path) WeightedListeningExecutorService(com.facebook.buck.util.concurrent.WeightedListeningExecutorService) Function(com.google.common.base.Function) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Set(java.util.Set) SizeLimiter(com.facebook.buck.rules.keys.SizeLimiter) DefaultFileHashCache(com.facebook.buck.util.cache.DefaultFileHashCache) BuildTarget(com.facebook.buck.model.BuildTarget) Sets(com.google.common.collect.Sets) List(java.util.List) Stream(java.util.stream.Stream) StepFailedException(com.facebook.buck.step.StepFailedException) LazyPath(com.facebook.buck.io.LazyPath) ByteStreams(com.google.common.io.ByteStreams) DependencyFileEntry(com.facebook.buck.rules.keys.DependencyFileEntry) Optional(java.util.Optional) GZIPOutputStream(java.util.zip.GZIPOutputStream) MoreFunctions(com.facebook.buck.util.MoreFunctions) RuleKeyFactoryManager(com.facebook.buck.rules.keys.RuleKeyFactoryManager) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) BuckEventBus(com.facebook.buck.event.BuckEventBus) ProjectFileHashCache(com.facebook.buck.util.cache.ProjectFileHashCache) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) Step(com.facebook.buck.step.Step) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) ConsoleEvent(com.facebook.buck.event.ConsoleEvent) BufferedOutputStream(java.io.BufferedOutputStream) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) ExecutionContext(com.facebook.buck.step.ExecutionContext) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) ThrowableConsoleEvent(com.facebook.buck.event.ThrowableConsoleEvent) ResourceAmounts(com.facebook.buck.util.concurrent.ResourceAmounts) Atomics(com.google.common.util.concurrent.Atomics) Nonnull(javax.annotation.Nonnull) ImmutableSortedMap(com.google.common.collect.ImmutableSortedMap) Nullable(javax.annotation.Nullable) SupportsInputBasedRuleKey(com.facebook.buck.rules.keys.SupportsInputBasedRuleKey) MoreCollectors(com.facebook.buck.util.MoreCollectors) OutputStream(java.io.OutputStream) Logger(com.facebook.buck.log.Logger) Functions(com.google.common.base.Functions) CacheResultType(com.facebook.buck.artifact_cache.CacheResultType) Files(java.nio.file.Files) HashCode(com.google.common.hash.HashCode) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Throwables(com.google.common.base.Throwables) IOException(java.io.IOException) HumanReadableException(com.facebook.buck.util.HumanReadableException) Maps(com.google.common.collect.Maps) FutureCallback(com.google.common.util.concurrent.FutureCallback) ExecutionException(java.util.concurrent.ExecutionException) ContextualProcessExecutor(com.facebook.buck.util.ContextualProcessExecutor) Futures(com.google.common.util.concurrent.Futures) BorrowablePath(com.facebook.buck.io.BorrowablePath) Paths(java.nio.file.Paths) CacheResult(com.facebook.buck.artifact_cache.CacheResult) FileHashCache(com.facebook.buck.util.cache.FileHashCache) Unzip(com.facebook.buck.zip.Unzip) Preconditions(com.google.common.base.Preconditions) AsyncFunction(com.google.common.util.concurrent.AsyncFunction) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MoreFiles(com.facebook.buck.io.MoreFiles) SupportsDependencyFileRuleKey(com.facebook.buck.rules.keys.SupportsDependencyFileRuleKey) Collections(java.util.Collections) InputStream(java.io.InputStream) ImmutableList(com.google.common.collect.ImmutableList) RuleKeyFactories(com.facebook.buck.rules.keys.RuleKeyFactories) DependencyFileEntry(com.facebook.buck.rules.keys.DependencyFileEntry) HashCode(com.google.common.hash.HashCode) FutureCallback(com.google.common.util.concurrent.FutureCallback) Path(java.nio.file.Path) LazyPath(com.facebook.buck.io.LazyPath) BorrowablePath(com.facebook.buck.io.BorrowablePath) Optional(java.util.Optional) SupportsInputBasedRuleKey(com.facebook.buck.rules.keys.SupportsInputBasedRuleKey) SupportsDependencyFileRuleKey(com.facebook.buck.rules.keys.SupportsDependencyFileRuleKey) Nonnull(javax.annotation.Nonnull) IOException(java.io.IOException)

Aggregations

Stream (java.util.stream.Stream)161 Collectors (java.util.stream.Collectors)98 List (java.util.List)89 ArrayList (java.util.ArrayList)66 Map (java.util.Map)66 Set (java.util.Set)59 IOException (java.io.IOException)58 Optional (java.util.Optional)45 Collections (java.util.Collections)43 HashMap (java.util.HashMap)43 Arrays (java.util.Arrays)33 HashSet (java.util.HashSet)33 File (java.io.File)32 Path (java.nio.file.Path)32 Function (java.util.function.Function)28 Logger (org.slf4j.Logger)26 LoggerFactory (org.slf4j.LoggerFactory)26 java.util (java.util)25 Predicate (java.util.function.Predicate)23 Objects (java.util.Objects)22