Search in sources :

Example 1 with RuleKeyAndInputs

use of com.facebook.buck.rules.keys.RuleKeyAndInputs in project buck by facebook.

the class CachingBuildEngine method processBuildRule.

private ListenableFuture<BuildResult> processBuildRule(BuildRule rule, BuildEngineBuildContext buildContext, ExecutionContext executionContext, ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks) {
    final RuleKeyFactories keyFactories = ruleKeyFactories.apply(rule.getProjectFilesystem());
    final OnDiskBuildInfo onDiskBuildInfo = buildContext.createOnDiskBuildInfoFor(rule.getBuildTarget(), rule.getProjectFilesystem());
    final BuildInfoRecorder buildInfoRecorder = buildContext.createBuildInfoRecorder(rule.getBuildTarget(), rule.getProjectFilesystem()).addBuildMetadata(BuildInfo.MetadataKey.RULE_KEY, keyFactories.getDefaultRuleKeyFactory().build(rule).toString());
    final BuildableContext buildableContext = new DefaultBuildableContext(buildInfoRecorder);
    final AtomicReference<Long> outputSize = Atomics.newReference();
    ListenableFuture<BuildResult> buildResult = processBuildRule(rule, buildContext, executionContext, onDiskBuildInfo, buildInfoRecorder, buildableContext, asyncCallbacks);
    // materialized locally by chaining up to our result future.
    if (buildMode == BuildMode.DEEP || buildMode == BuildMode.POPULATE_FROM_REMOTE_CACHE) {
        buildResult = MoreFutures.chainExceptions(getDepResults(rule, buildContext, executionContext, asyncCallbacks), buildResult);
    }
    // Setup a callback to handle either the cached or built locally cases.
    AsyncFunction<BuildResult, BuildResult> callback = input -> {
        if (input.getStatus() != BuildRuleStatus.SUCCESS) {
            return Futures.immediateFuture(input);
        }
        BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
        if (success != BuildRuleSuccessType.BUILT_LOCALLY) {
            for (String str : onDiskBuildInfo.getValuesOrThrow(BuildInfo.MetadataKey.RECORDED_PATHS)) {
                buildInfoRecorder.recordArtifact(Paths.get(str));
            }
        }
        outputSize.set(buildInfoRecorder.getOutputSize());
        if (success.outputsHaveChanged()) {
            if (rule instanceof HasPostBuildSteps) {
                executePostBuildSteps(rule, ((HasPostBuildSteps) rule).getPostBuildSteps(buildContext.getBuildContext()), executionContext);
            }
            for (Path path : buildInfoRecorder.getRecordedPaths()) {
                fileHashCache.invalidate(rule.getProjectFilesystem().resolve(path));
            }
        }
        if (useDependencyFileRuleKey(rule) && success == BuildRuleSuccessType.BUILT_LOCALLY) {
            ImmutableList<SourcePath> inputs = ((SupportsDependencyFileRuleKey) rule).getInputsAfterBuildingLocally(buildContext.getBuildContext());
            ImmutableList<String> inputStrings = inputs.stream().map(inputString -> DependencyFileEntry.fromSourcePath(inputString, pathResolver)).map(MoreFunctions.toJsonFunction(objectMapper)).collect(MoreCollectors.toImmutableList());
            buildInfoRecorder.addMetadata(BuildInfo.MetadataKey.DEP_FILE, inputStrings);
            Optional<RuleKeyAndInputs> depFileRuleKeyAndInputs = calculateDepFileRuleKey(rule, buildContext, Optional.of(inputStrings), false);
            if (depFileRuleKeyAndInputs.isPresent()) {
                RuleKey depFileRuleKey = depFileRuleKeyAndInputs.get().getRuleKey();
                buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY, depFileRuleKey.toString());
                if (useManifestCaching(rule)) {
                    Optional<RuleKeyAndInputs> manifestKey = calculateManifestKey(rule, buildContext.getEventBus());
                    if (manifestKey.isPresent()) {
                        buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.MANIFEST_KEY, manifestKey.get().getRuleKey().toString());
                        updateAndStoreManifest(rule, depFileRuleKeyAndInputs.get().getRuleKey(), depFileRuleKeyAndInputs.get().getInputs(), manifestKey.get(), buildContext.getArtifactCache());
                    }
                }
            }
        }
        if (success == BuildRuleSuccessType.BUILT_LOCALLY && shouldUploadToCache(rule, Preconditions.checkNotNull(outputSize.get()))) {
            ImmutableSortedMap.Builder<String, String> outputHashes = ImmutableSortedMap.naturalOrder();
            for (Path path : buildInfoRecorder.getOutputPaths()) {
                outputHashes.put(path.toString(), fileHashCache.get(rule.getProjectFilesystem().resolve(path)).toString());
            }
            buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.RECORDED_PATH_HASHES, outputHashes.build());
        }
        if (success != BuildRuleSuccessType.BUILT_LOCALLY && success.outputsHaveChanged()) {
            Optional<ImmutableMap<String, String>> hashes = onDiskBuildInfo.getBuildMap(BuildInfo.MetadataKey.RECORDED_PATH_HASHES);
            if (hashes.isPresent() && verifyRecordedPathHashes(rule.getBuildTarget(), rule.getProjectFilesystem(), hashes.get())) {
                for (Map.Entry<String, String> ent : hashes.get().entrySet()) {
                    Path path = rule.getProjectFilesystem().getPath(ent.getKey());
                    HashCode hashCode = HashCode.fromString(ent.getValue());
                    fileHashCache.set(rule.getProjectFilesystem().resolve(path), hashCode);
                }
            }
        }
        buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.TARGET, rule.getBuildTarget().toString());
        buildInfoRecorder.addMetadata(BuildInfo.MetadataKey.RECORDED_PATHS, buildInfoRecorder.getRecordedPaths().stream().map(Object::toString).collect(MoreCollectors.toImmutableList()));
        if (success.shouldWriteRecordedMetadataToDiskAfterBuilding()) {
            try {
                boolean clearExistingMetadata = success.shouldClearAndOverwriteMetadataOnDisk();
                buildInfoRecorder.writeMetadataToDisk(clearExistingMetadata);
            } catch (IOException e) {
                throw new IOException(String.format("Failed to write metadata to disk for %s.", rule), e);
            }
        }
        try {
            if (rule instanceof InitializableFromDisk) {
                doInitializeFromDisk((InitializableFromDisk<?>) rule, onDiskBuildInfo);
            }
        } catch (IOException e) {
            throw new IOException(String.format("Error initializing %s from disk.", rule), e);
        }
        return Futures.immediateFuture(input);
    };
    buildResult = Futures.transformAsync(buildResult, ruleAsyncFunction(rule, buildContext.getEventBus(), callback), serviceByAdjustingDefaultWeightsTo(RULE_KEY_COMPUTATION_RESOURCE_AMOUNTS));
    // Handle either build success or failure.
    final SettableFuture<BuildResult> result = SettableFuture.create();
    asyncCallbacks.add(MoreFutures.addListenableCallback(buildResult, new FutureCallback<BuildResult>() {

        // TODO(bolinfest): Delete all files produced by the rule, as they are not guaranteed
        // to be valid at this point?
        private void cleanupAfterError() {
            try {
                onDiskBuildInfo.deleteExistingMetadata();
            } catch (Throwable t) {
                buildContext.getEventBus().post(ThrowableConsoleEvent.create(t, "Error when deleting metadata for %s.", rule));
            }
        }

        private void uploadToCache(BuildRuleSuccessType success) {
            // Collect up all the rule keys we have index the artifact in the cache with.
            Set<RuleKey> ruleKeys = Sets.newHashSet();
            // If the rule key has changed (and is not already in the cache), we need to push
            // the artifact to cache using the new key.
            ruleKeys.add(keyFactories.getDefaultRuleKeyFactory().build(rule));
            // using the new key.
            if (SupportsInputBasedRuleKey.isSupported(rule)) {
                Optional<RuleKey> calculatedRuleKey = calculateInputBasedRuleKey(rule, buildContext.getEventBus());
                Optional<RuleKey> onDiskRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY);
                Optional<RuleKey> metaDataRuleKey = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY).map(RuleKey::new);
                Preconditions.checkState(calculatedRuleKey.equals(onDiskRuleKey), "%s (%s): %s: invalid on-disk input-based rule key: %s != %s", rule.getBuildTarget(), rule.getType(), success, calculatedRuleKey, onDiskRuleKey);
                Preconditions.checkState(calculatedRuleKey.equals(metaDataRuleKey), "%s: %s: invalid meta-data input-based rule key: %s != %s", rule.getBuildTarget(), success, calculatedRuleKey, metaDataRuleKey);
                ruleKeys.addAll(OptionalCompat.asSet(calculatedRuleKey));
            }
            // using the new key.
            if (useManifestCaching(rule)) {
                Optional<RuleKey> onDiskRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY);
                Optional<RuleKey> metaDataRuleKey = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY).map(RuleKey::new);
                Preconditions.checkState(onDiskRuleKey.equals(metaDataRuleKey), "%s: %s: inconsistent meta-data and on-disk dep-file rule key: %s != %s", rule.getBuildTarget(), success, onDiskRuleKey, metaDataRuleKey);
                ruleKeys.addAll(OptionalCompat.asSet(onDiskRuleKey));
            }
            // Do the actual upload.
            try {
                // Verify that the recorded path hashes are accurate.
                Optional<String> recordedPathHashes = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.RECORDED_PATH_HASHES);
                if (recordedPathHashes.isPresent() && !verifyRecordedPathHashes(rule.getBuildTarget(), rule.getProjectFilesystem(), recordedPathHashes.get())) {
                    return;
                }
                // Push to cache.
                buildInfoRecorder.performUploadToArtifactCache(ImmutableSet.copyOf(ruleKeys), buildContext.getArtifactCache(), buildContext.getEventBus());
            } catch (Throwable t) {
                buildContext.getEventBus().post(ThrowableConsoleEvent.create(t, "Error uploading to cache for %s.", rule));
            }
        }

        private void handleResult(BuildResult input) {
            Optional<Long> outputSize = Optional.empty();
            Optional<HashCode> outputHash = Optional.empty();
            Optional<BuildRuleSuccessType> successType = Optional.empty();
            BuildRuleEvent.Resumed resumedEvent = BuildRuleEvent.resumed(rule, buildRuleDurationTracker, keyFactories.getDefaultRuleKeyFactory());
            buildContext.getEventBus().logVerboseAndPost(LOG, resumedEvent);
            if (input.getStatus() == BuildRuleStatus.FAIL) {
                // Make this failure visible for other rules, so that they can stop early.
                firstFailure = input.getFailure();
                // If we failed, cleanup the state of this rule.
                cleanupAfterError();
            }
            // Unblock dependents.
            result.set(input);
            if (input.getStatus() == BuildRuleStatus.SUCCESS) {
                BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
                successType = Optional.of(success);
                // Try get the output size.
                try {
                    outputSize = Optional.of(buildInfoRecorder.getOutputSize());
                } catch (IOException e) {
                    buildContext.getEventBus().post(ThrowableConsoleEvent.create(e, "Error getting output size for %s.", rule));
                }
                // If this rule is cacheable, upload it to the cache.
                if (success.shouldUploadResultingArtifact() && outputSize.isPresent() && shouldUploadToCache(rule, outputSize.get())) {
                    uploadToCache(success);
                }
                // Calculate the hash of outputs that were built locally and are cacheable.
                if (success == BuildRuleSuccessType.BUILT_LOCALLY && shouldUploadToCache(rule, outputSize.get())) {
                    try {
                        outputHash = Optional.of(buildInfoRecorder.getOutputHash(fileHashCache));
                    } catch (IOException e) {
                        buildContext.getEventBus().post(ThrowableConsoleEvent.create(e, "Error getting output hash for %s.", rule));
                    }
                }
            }
            // Log the result to the event bus.
            BuildRuleEvent.Finished finished = BuildRuleEvent.finished(resumedEvent, BuildRuleKeys.builder().setRuleKey(keyFactories.getDefaultRuleKeyFactory().build(rule)).setInputRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY)).setDepFileRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY)).setManifestRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.MANIFEST_KEY)).build(), input.getStatus(), input.getCacheResult(), successType, outputHash, outputSize);
            buildContext.getEventBus().logVerboseAndPost(LOG, finished);
        }

        @Override
        public void onSuccess(BuildResult input) {
            handleResult(input);
        }

        @Override
        public void onFailure(@Nonnull Throwable thrown) {
            thrown = maybeAttachBuildRuleNameToException(thrown, rule);
            handleResult(BuildResult.failure(rule, thrown));
            // Reset interrupted flag once failure has been recorded.
            if (thrown instanceof InterruptedException) {
                Thread.currentThread().interrupt();
            }
        }
    }));
    return result;
}
Also used : OptionalCompat(com.facebook.buck.util.OptionalCompat) NoSuchFileException(java.nio.file.NoSuchFileException) GZIPInputStream(java.util.zip.GZIPInputStream) ArtifactCache(com.facebook.buck.artifact_cache.ArtifactCache) BufferedInputStream(java.io.BufferedInputStream) StepRunner(com.facebook.buck.step.StepRunner) BuckEvent(com.facebook.buck.event.BuckEvent) RuleKeyCalculationEvent(com.facebook.buck.event.RuleKeyCalculationEvent) ObjectMappers(com.facebook.buck.util.ObjectMappers) MoreFutures(com.facebook.buck.util.concurrent.MoreFutures) SettableFuture(com.google.common.util.concurrent.SettableFuture) ArtifactCompressionEvent(com.facebook.buck.event.ArtifactCompressionEvent) ProjectFilesystem(com.facebook.buck.io.ProjectFilesystem) RuleKeyAndInputs(com.facebook.buck.rules.keys.RuleKeyAndInputs) Map(java.util.Map) RuleKeyFactories(com.facebook.buck.rules.keys.RuleKeyFactories) TypeReference(com.fasterxml.jackson.core.type.TypeReference) ArtifactInfo(com.facebook.buck.artifact_cache.ArtifactInfo) Path(java.nio.file.Path) WeightedListeningExecutorService(com.facebook.buck.util.concurrent.WeightedListeningExecutorService) Function(com.google.common.base.Function) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Set(java.util.Set) SizeLimiter(com.facebook.buck.rules.keys.SizeLimiter) DefaultFileHashCache(com.facebook.buck.util.cache.DefaultFileHashCache) BuildTarget(com.facebook.buck.model.BuildTarget) Sets(com.google.common.collect.Sets) List(java.util.List) Stream(java.util.stream.Stream) StepFailedException(com.facebook.buck.step.StepFailedException) LazyPath(com.facebook.buck.io.LazyPath) ByteStreams(com.google.common.io.ByteStreams) DependencyFileEntry(com.facebook.buck.rules.keys.DependencyFileEntry) Optional(java.util.Optional) GZIPOutputStream(java.util.zip.GZIPOutputStream) MoreFunctions(com.facebook.buck.util.MoreFunctions) RuleKeyFactoryManager(com.facebook.buck.rules.keys.RuleKeyFactoryManager) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) BuckEventBus(com.facebook.buck.event.BuckEventBus) ProjectFileHashCache(com.facebook.buck.util.cache.ProjectFileHashCache) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) Step(com.facebook.buck.step.Step) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) ConsoleEvent(com.facebook.buck.event.ConsoleEvent) BufferedOutputStream(java.io.BufferedOutputStream) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) ExecutionContext(com.facebook.buck.step.ExecutionContext) Lists(com.google.common.collect.Lists) ImmutableList(com.google.common.collect.ImmutableList) ThrowableConsoleEvent(com.facebook.buck.event.ThrowableConsoleEvent) ResourceAmounts(com.facebook.buck.util.concurrent.ResourceAmounts) Atomics(com.google.common.util.concurrent.Atomics) Nonnull(javax.annotation.Nonnull) ImmutableSortedMap(com.google.common.collect.ImmutableSortedMap) Nullable(javax.annotation.Nullable) SupportsInputBasedRuleKey(com.facebook.buck.rules.keys.SupportsInputBasedRuleKey) MoreCollectors(com.facebook.buck.util.MoreCollectors) OutputStream(java.io.OutputStream) Logger(com.facebook.buck.log.Logger) Functions(com.google.common.base.Functions) CacheResultType(com.facebook.buck.artifact_cache.CacheResultType) Files(java.nio.file.Files) HashCode(com.google.common.hash.HashCode) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) Throwables(com.google.common.base.Throwables) IOException(java.io.IOException) HumanReadableException(com.facebook.buck.util.HumanReadableException) Maps(com.google.common.collect.Maps) FutureCallback(com.google.common.util.concurrent.FutureCallback) ExecutionException(java.util.concurrent.ExecutionException) ContextualProcessExecutor(com.facebook.buck.util.ContextualProcessExecutor) Futures(com.google.common.util.concurrent.Futures) BorrowablePath(com.facebook.buck.io.BorrowablePath) Paths(java.nio.file.Paths) CacheResult(com.facebook.buck.artifact_cache.CacheResult) FileHashCache(com.facebook.buck.util.cache.FileHashCache) Unzip(com.facebook.buck.zip.Unzip) Preconditions(com.google.common.base.Preconditions) AsyncFunction(com.google.common.util.concurrent.AsyncFunction) VisibleForTesting(com.google.common.annotations.VisibleForTesting) MoreFiles(com.facebook.buck.io.MoreFiles) SupportsDependencyFileRuleKey(com.facebook.buck.rules.keys.SupportsDependencyFileRuleKey) Collections(java.util.Collections) InputStream(java.io.InputStream) ImmutableList(com.google.common.collect.ImmutableList) RuleKeyFactories(com.facebook.buck.rules.keys.RuleKeyFactories) DependencyFileEntry(com.facebook.buck.rules.keys.DependencyFileEntry) HashCode(com.google.common.hash.HashCode) FutureCallback(com.google.common.util.concurrent.FutureCallback) Path(java.nio.file.Path) LazyPath(com.facebook.buck.io.LazyPath) BorrowablePath(com.facebook.buck.io.BorrowablePath) Optional(java.util.Optional) SupportsInputBasedRuleKey(com.facebook.buck.rules.keys.SupportsInputBasedRuleKey) SupportsDependencyFileRuleKey(com.facebook.buck.rules.keys.SupportsDependencyFileRuleKey) Nonnull(javax.annotation.Nonnull) IOException(java.io.IOException)

Aggregations

ArtifactCache (com.facebook.buck.artifact_cache.ArtifactCache)1 ArtifactInfo (com.facebook.buck.artifact_cache.ArtifactInfo)1 CacheResult (com.facebook.buck.artifact_cache.CacheResult)1 CacheResultType (com.facebook.buck.artifact_cache.CacheResultType)1 ArtifactCompressionEvent (com.facebook.buck.event.ArtifactCompressionEvent)1 BuckEvent (com.facebook.buck.event.BuckEvent)1 BuckEventBus (com.facebook.buck.event.BuckEventBus)1 ConsoleEvent (com.facebook.buck.event.ConsoleEvent)1 RuleKeyCalculationEvent (com.facebook.buck.event.RuleKeyCalculationEvent)1 ThrowableConsoleEvent (com.facebook.buck.event.ThrowableConsoleEvent)1 BorrowablePath (com.facebook.buck.io.BorrowablePath)1 LazyPath (com.facebook.buck.io.LazyPath)1 MoreFiles (com.facebook.buck.io.MoreFiles)1 ProjectFilesystem (com.facebook.buck.io.ProjectFilesystem)1 Logger (com.facebook.buck.log.Logger)1 BuildTarget (com.facebook.buck.model.BuildTarget)1 DependencyFileEntry (com.facebook.buck.rules.keys.DependencyFileEntry)1 RuleKeyAndInputs (com.facebook.buck.rules.keys.RuleKeyAndInputs)1 RuleKeyFactories (com.facebook.buck.rules.keys.RuleKeyFactories)1 RuleKeyFactoryManager (com.facebook.buck.rules.keys.RuleKeyFactoryManager)1