Examples with ExecutionContext com.facebook.buck.step.ExecutionContext used on opensource projects

Example 16 with ExecutionContext

use of com.facebook.buck.step.ExecutionContext in project buck by facebook.

the class WriteStringTemplateRule method getBuildSteps.

@Override
public ImmutableList<Step> getBuildSteps(BuildContext context, BuildableContext buildableContext) {
    buildableContext.recordArtifact(output);
    ImmutableList.Builder<Step> steps = ImmutableList.builder();
    steps.add(new MkdirStep(getProjectFilesystem(), output.getParent()));
    steps.add(new StringTemplateStep(context.getSourcePathResolver().getAbsolutePath(template), getProjectFilesystem(), output, st -> {
        for (Map.Entry<String, String> ent : values.entrySet()) {
            st = st.add(ent.getKey(), ent.getValue());
        }
        return st;
    }));
    if (executable) {
        steps.add(new AbstractExecutionStep("chmod +x") {

            @Override
            public StepExecutionResult execute(ExecutionContext context) throws IOException {
                MoreFiles.makeExecutable(getProjectFilesystem().resolve(output));
                return StepExecutionResult.of(0, Optional.empty());
            }
        });
    }
    return steps.build();
}

Example 17 with ExecutionContext

use of com.facebook.buck.step.ExecutionContext in project buck by facebook.

the class PreDexMerge method addMetadataWriteStep.

private void addMetadataWriteStep(final PreDexedFilesSorter.Result result, final ImmutableList.Builder<Step> steps, final Path metadataFilePath) {
    StringBuilder nameBuilder = new StringBuilder(30);
    final boolean isRootModule = result.apkModule.equals(apkModuleGraph.getRootAPKModule());
    final String storeId = result.apkModule.getName();
    nameBuilder.append("write_");
    if (!isRootModule) {
        nameBuilder.append(storeId);
        nameBuilder.append("_");
    }
    nameBuilder.append("metadata_txt");
    steps.add(new AbstractExecutionStep(nameBuilder.toString()) {

        @Override
        public StepExecutionResult execute(ExecutionContext executionContext) {
            Map<Path, DexWithClasses> metadataTxtEntries = result.metadataTxtDexEntries;
            List<String> lines = Lists.newArrayListWithCapacity(metadataTxtEntries.size());
            lines.add(".id " + storeId);
            if (isRootModule) {
                if (dexSplitMode.getDexStore() == DexStore.RAW) {
                    lines.add(".root_relative");
                }
            } else {
                for (APKModule dependency : apkModuleGraph.getGraph().getOutgoingNodesFor(result.apkModule)) {
                    lines.add(".requires " + dependency.getName());
                }
            }
            try {
                for (Map.Entry<Path, DexWithClasses> entry : metadataTxtEntries.entrySet()) {
                    Path pathToSecondaryDex = entry.getKey();
                    String containedClass = Iterables.get(entry.getValue().getClassNames(), 0);
                    containedClass = containedClass.replace('/', '.');
                    Sha1HashCode hash = getProjectFilesystem().computeSha1(pathToSecondaryDex);
                    lines.add(String.format("%s %s %s", pathToSecondaryDex.getFileName(), hash, containedClass));
                }
                getProjectFilesystem().writeLinesToPath(lines, metadataFilePath);
            } catch (IOException e) {
                executionContext.logError(e, "Failed when writing metadata.txt multi-dex.");
                return StepExecutionResult.ERROR;
            }
            return StepExecutionResult.SUCCESS;
        }
    });
}

Example 18 with ExecutionContext

use of com.facebook.buck.step.ExecutionContext in project buck by facebook.

the class UnzipAar method getBuildSteps.

@Override
public ImmutableList<Step> getBuildSteps(BuildContext context, BuildableContext buildableContext) {
    ImmutableList.Builder<Step> steps = ImmutableList.builder();
    steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), unpackDirectory));
    steps.add(new UnzipStep(getProjectFilesystem(), context.getSourcePathResolver().getAbsolutePath(aarFile), unpackDirectory));
    steps.add(new TouchStep(getProjectFilesystem(), getProguardConfig()));
    steps.add(new MkdirStep(getProjectFilesystem(), context.getSourcePathResolver().getAbsolutePath(getAssetsDirectory())));
    steps.add(new MkdirStep(getProjectFilesystem(), getNativeLibsDirectory()));
    steps.add(new TouchStep(getProjectFilesystem(), getTextSymbolsFile()));
    // We take the classes.jar file that is required to exist in an .aar and merge it with any
    // .jar files under libs/ into an "uber" jar. We do this for simplicity because we do not know
    // how many entries there are in libs/ at graph enhancement time, but we need to make sure
    // that all of the .class files in the .aar get packaged. As it is implemented today, an
    // android_library that depends on an android_prebuilt_aar can compile against anything in the
    // .aar's classes.jar or libs/.
    steps.add(new MkdirStep(getProjectFilesystem(), uberClassesJar.getParent()));
    steps.add(new AbstractExecutionStep("create_uber_classes_jar") {

        @Override
        public StepExecutionResult execute(ExecutionContext context) {
            Path libsDirectory = unpackDirectory.resolve("libs");
            boolean dirDoesNotExistOrIsEmpty;
            if (!getProjectFilesystem().exists(libsDirectory)) {
                dirDoesNotExistOrIsEmpty = true;
            } else {
                try {
                    dirDoesNotExistOrIsEmpty = getProjectFilesystem().getDirectoryContents(libsDirectory).isEmpty();
                } catch (IOException e) {
                    context.logError(e, "Failed to get directory contents of %s", libsDirectory);
                    return StepExecutionResult.ERROR;
                }
            }
            Path classesJar = unpackDirectory.resolve("classes.jar");
            JavacEventSinkToBuckEventBusBridge eventSink = new JavacEventSinkToBuckEventBusBridge(context.getBuckEventBus());
            if (!getProjectFilesystem().exists(classesJar)) {
                try {
                    JarDirectoryStepHelper.createEmptyJarFile(getProjectFilesystem(), classesJar, eventSink, context.getStdErr());
                } catch (IOException e) {
                    context.logError(e, "Failed to create empty jar %s", classesJar);
                    return StepExecutionResult.ERROR;
                }
            }
            if (dirDoesNotExistOrIsEmpty) {
                try {
                    getProjectFilesystem().copy(classesJar, uberClassesJar, ProjectFilesystem.CopySourceMode.FILE);
                } catch (IOException e) {
                    context.logError(e, "Failed to copy from %s to %s", classesJar, uberClassesJar);
                    return StepExecutionResult.ERROR;
                }
            } else {
                // Glob all of the contents from classes.jar and the entries in libs/ into a single JAR.
                ImmutableSortedSet.Builder<Path> entriesToJarBuilder = ImmutableSortedSet.naturalOrder();
                entriesToJarBuilder.add(classesJar);
                try {
                    entriesToJarBuilder.addAll(getProjectFilesystem().getDirectoryContents(libsDirectory));
                } catch (IOException e) {
                    context.logError(e, "Failed to get directory contents of %s", libsDirectory);
                    return StepExecutionResult.ERROR;
                }
                ImmutableSortedSet<Path> entriesToJar = entriesToJarBuilder.build();
                try {
                    JarDirectoryStepHelper.createJarFile(getProjectFilesystem(), uberClassesJar, entriesToJar, /* mainClass */
                    Optional.empty(), /* manifestFile */
                    Optional.empty(), /* mergeManifests */
                    true, /* blacklist */
                    ImmutableSet.of(), eventSink, context.getStdErr());
                } catch (IOException e) {
                    context.logError(e, "Failed to jar %s into %s", entriesToJar, uberClassesJar);
                    return StepExecutionResult.ERROR;
                }
            }
            return StepExecutionResult.SUCCESS;
        }
    });
    steps.add(new MakeCleanDirectoryStep(getProjectFilesystem(), pathToTextSymbolsDir));
    steps.add(new ExtractFromAndroidManifestStep(getAndroidManifest(), getProjectFilesystem(), buildableContext, METADATA_KEY_FOR_R_DOT_JAVA_PACKAGE, pathToRDotJavaPackageFile));
    steps.add(CopyStep.forFile(getProjectFilesystem(), getTextSymbolsFile(), pathToTextSymbolsFile));
    buildableContext.recordArtifact(unpackDirectory);
    buildableContext.recordArtifact(uberClassesJar);
    buildableContext.recordArtifact(pathToTextSymbolsFile);
    buildableContext.recordArtifact(pathToRDotJavaPackageFile);
    return steps.build();
}

Example 19 with ExecutionContext

use of com.facebook.buck.step.ExecutionContext in project buck by facebook.

the class JavaBuildGraphProcessor method run.

/**
   * Creates the appropriate target graph and other resources needed for the {@link Processor} and
   * runs it. This method will take responsibility for cleaning up the executor service after it
   * runs.
   */
static void run(final CommandRunnerParams params, final AbstractCommand command, final Processor processor) throws ExitCodeException, InterruptedException, IOException {
    final ConcurrencyLimit concurrencyLimit = command.getConcurrencyLimit(params.getBuckConfig());
    try (CommandThreadManager pool = new CommandThreadManager(command.getClass().getName(), concurrencyLimit)) {
        Cell cell = params.getCell();
        WeightedListeningExecutorService executorService = pool.getExecutor();
        // Ideally, we should be able to construct the TargetGraph quickly assuming most of it is
        // already in memory courtesy of buckd. Though we could make a performance optimization where
        // we pass an option to buck.py that tells it to ignore reading the BUCK.autodeps files when
        // parsing the BUCK files because we never need to consider the existing auto-generated deps
        // when creating the new auto-generated deps. If we did so, we would have to make sure to keep
        // the nodes for that version of the graph separate from the ones that are actually used for
        // building.
        TargetGraph graph;
        try {
            graph = params.getParser().buildTargetGraphForTargetNodeSpecs(params.getBuckEventBus(), cell, command.getEnableParserProfiling(), executorService, ImmutableList.of(TargetNodePredicateSpec.of(x -> true, BuildFileSpec.fromRecursivePath(Paths.get(""), cell.getRoot()))), /* ignoreBuckAutodepsFiles */
            true).getTargetGraph();
        } catch (BuildTargetException | BuildFileParseException e) {
            params.getBuckEventBus().post(ConsoleEvent.severe(MoreExceptions.getHumanReadableOrLocalizedMessage(e)));
            throw new ExitCodeException(1);
        }
        BuildRuleResolver buildRuleResolver = new BuildRuleResolver(graph, new DefaultTargetNodeToBuildRuleTransformer());
        CachingBuildEngineBuckConfig cachingBuildEngineBuckConfig = params.getBuckConfig().getView(CachingBuildEngineBuckConfig.class);
        LocalCachingBuildEngineDelegate cachingBuildEngineDelegate = new LocalCachingBuildEngineDelegate(params.getFileHashCache());
        BuildEngine buildEngine = new CachingBuildEngine(cachingBuildEngineDelegate, executorService, executorService, new DefaultStepRunner(), CachingBuildEngine.BuildMode.SHALLOW, cachingBuildEngineBuckConfig.getBuildDepFiles(), cachingBuildEngineBuckConfig.getBuildMaxDepFileCacheEntries(), cachingBuildEngineBuckConfig.getBuildArtifactCacheSizeLimit(), params.getObjectMapper(), buildRuleResolver, cachingBuildEngineBuckConfig.getResourceAwareSchedulingInfo(), new RuleKeyFactoryManager(params.getBuckConfig().getKeySeed(), fs -> cachingBuildEngineDelegate.getFileHashCache(), buildRuleResolver, cachingBuildEngineBuckConfig.getBuildInputRuleKeyFileSizeLimit(), new DefaultRuleKeyCache<>()));
        // Create a BuildEngine because we store symbol information as build artifacts.
        BuckEventBus eventBus = params.getBuckEventBus();
        ExecutionContext executionContext = ExecutionContext.builder().setConsole(params.getConsole()).setConcurrencyLimit(concurrencyLimit).setBuckEventBus(eventBus).setEnvironment(/* environment */
        ImmutableMap.of()).setExecutors(ImmutableMap.<ExecutorPool, ListeningExecutorService>of(ExecutorPool.CPU, executorService)).setJavaPackageFinder(params.getJavaPackageFinder()).setObjectMapper(params.getObjectMapper()).setPlatform(params.getPlatform()).setCellPathResolver(params.getCell().getCellPathResolver()).build();
        SourcePathResolver pathResolver = new SourcePathResolver(new SourcePathRuleFinder(buildRuleResolver));
        BuildEngineBuildContext buildContext = BuildEngineBuildContext.builder().setBuildContext(BuildContext.builder().setActionGraph(new ActionGraph(ImmutableList.of())).setSourcePathResolver(pathResolver).setJavaPackageFinder(executionContext.getJavaPackageFinder()).setEventBus(eventBus).build()).setClock(params.getClock()).setArtifactCache(params.getArtifactCacheFactory().newInstance()).setBuildId(eventBus.getBuildId()).setObjectMapper(params.getObjectMapper()).setEnvironment(executionContext.getEnvironment()).setKeepGoing(false).build();
        // Traverse the TargetGraph to find all of the auto-generated dependencies.
        JavaDepsFinder javaDepsFinder = JavaDepsFinder.createJavaDepsFinder(params.getBuckConfig(), params.getCell().getCellPathResolver(), params.getObjectMapper(), buildContext, executionContext, buildEngine);
        processor.process(graph, javaDepsFinder, executorService);
    }
}

Example 20 with ExecutionContext

use of com.facebook.buck.step.ExecutionContext in project buck by facebook.

the class CachingBuildEngine method processBuildRule.

private ListenableFuture<BuildResult> processBuildRule(final BuildRule rule, final BuildEngineBuildContext buildContext, final ExecutionContext executionContext, final OnDiskBuildInfo onDiskBuildInfo, final BuildInfoRecorder buildInfoRecorder, final BuildableContext buildableContext, final ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks) {
    // If we've already seen a failure, exit early.
    if (!buildContext.isKeepGoing() && firstFailure != null) {
        return Futures.immediateFuture(BuildResult.canceled(rule, firstFailure));
    }
    final RuleKeyFactories ruleKeyFactory = ruleKeyFactories.apply(rule.getProjectFilesystem());
    try (BuildRuleEvent.Scope scope = BuildRuleEvent.resumeSuspendScope(buildContext.getEventBus(), rule, buildRuleDurationTracker, ruleKeyFactory.getDefaultRuleKeyFactory())) {
        // 1. Check if it's already built.
        Optional<RuleKey> cachedRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.RULE_KEY);
        final RuleKey defaultRuleKey = ruleKeyFactory.getDefaultRuleKeyFactory().build(rule);
        if (defaultRuleKey.equals(cachedRuleKey.orElse(null))) {
            return Futures.transform(markRuleAsUsed(rule, buildContext.getEventBus()), Functions.constant(BuildResult.success(rule, BuildRuleSuccessType.MATCHING_RULE_KEY, CacheResult.localKeyUnchangedHit())));
        }
        // 2. Rule key cache lookup.
        ListenableFuture<CacheResult> rulekeyCacheResult = cacheActivityService.submit(() -> {
            CacheResult cacheResult = tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem(rule, defaultRuleKey, buildContext.getArtifactCache(), // TODO(shs96c): This should be a shared between all tests, not one per cell
            rule.getProjectFilesystem(), buildContext);
            if (cacheResult.getType().isSuccess()) {
                fillMissingBuildMetadataFromCache(cacheResult, buildInfoRecorder, BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY, BuildInfo.MetadataKey.DEP_FILE_RULE_KEY, BuildInfo.MetadataKey.DEP_FILE);
            }
            return cacheResult;
        }, CACHE_CHECK_RESOURCE_AMOUNTS);
        return Futures.transformAsync(rulekeyCacheResult, ruleAsyncFunction(rule, buildContext.getEventBus(), (cacheResult) -> handleRuleKeyCacheResult(rule, buildContext, executionContext, onDiskBuildInfo, buildInfoRecorder, buildableContext, asyncCallbacks, ruleKeyFactory, cacheResult)), serviceByAdjustingDefaultWeightsTo(SCHEDULING_MORE_WORK_RESOURCE_AMOUNTS));
    }
}