Examples with RuleKeyFactories com.facebook.buck.rules.keys.RuleKeyFactories used on opensource projects

Example 1 with RuleKeyFactories

use of com.facebook.buck.rules.keys.RuleKeyFactories in project buck by facebook.

the class CachingBuildEngine method processBuildRule.

private ListenableFuture<BuildResult> processBuildRule(final BuildRule rule, final BuildEngineBuildContext buildContext, final ExecutionContext executionContext, final OnDiskBuildInfo onDiskBuildInfo, final BuildInfoRecorder buildInfoRecorder, final BuildableContext buildableContext, final ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks) {
    // If we've already seen a failure, exit early.
    if (!buildContext.isKeepGoing() && firstFailure != null) {
        return Futures.immediateFuture(BuildResult.canceled(rule, firstFailure));
    }
    final RuleKeyFactories ruleKeyFactory = ruleKeyFactories.apply(rule.getProjectFilesystem());
    try (BuildRuleEvent.Scope scope = BuildRuleEvent.resumeSuspendScope(buildContext.getEventBus(), rule, buildRuleDurationTracker, ruleKeyFactory.getDefaultRuleKeyFactory())) {
        // 1. Check if it's already built.
        Optional<RuleKey> cachedRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.RULE_KEY);
        final RuleKey defaultRuleKey = ruleKeyFactory.getDefaultRuleKeyFactory().build(rule);
        if (defaultRuleKey.equals(cachedRuleKey.orElse(null))) {
            return Futures.transform(markRuleAsUsed(rule, buildContext.getEventBus()), Functions.constant(BuildResult.success(rule, BuildRuleSuccessType.MATCHING_RULE_KEY, CacheResult.localKeyUnchangedHit())));
        }
        // 2. Rule key cache lookup.
        ListenableFuture<CacheResult> rulekeyCacheResult = cacheActivityService.submit(() -> {
            CacheResult cacheResult = tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem(rule, defaultRuleKey, buildContext.getArtifactCache(), // TODO(shs96c): This should be a shared between all tests, not one per cell
            rule.getProjectFilesystem(), buildContext);
            if (cacheResult.getType().isSuccess()) {
                fillMissingBuildMetadataFromCache(cacheResult, buildInfoRecorder, BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY, BuildInfo.MetadataKey.DEP_FILE_RULE_KEY, BuildInfo.MetadataKey.DEP_FILE);
            }
            return cacheResult;
        }, CACHE_CHECK_RESOURCE_AMOUNTS);
        return Futures.transformAsync(rulekeyCacheResult, ruleAsyncFunction(rule, buildContext.getEventBus(), (cacheResult) -> handleRuleKeyCacheResult(rule, buildContext, executionContext, onDiskBuildInfo, buildInfoRecorder, buildableContext, asyncCallbacks, ruleKeyFactory, cacheResult)), serviceByAdjustingDefaultWeightsTo(SCHEDULING_MORE_WORK_RESOURCE_AMOUNTS));
    }
}

Example 2 with RuleKeyFactories

use of com.facebook.buck.rules.keys.RuleKeyFactories in project buck by facebook.

the class CachingBuildEngine method processBuildRule.

private ListenableFuture<BuildResult> processBuildRule(BuildRule rule, BuildEngineBuildContext buildContext, ExecutionContext executionContext, ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks) {
    final RuleKeyFactories keyFactories = ruleKeyFactories.apply(rule.getProjectFilesystem());
    final OnDiskBuildInfo onDiskBuildInfo = buildContext.createOnDiskBuildInfoFor(rule.getBuildTarget(), rule.getProjectFilesystem());
    final BuildInfoRecorder buildInfoRecorder = buildContext.createBuildInfoRecorder(rule.getBuildTarget(), rule.getProjectFilesystem()).addBuildMetadata(BuildInfo.MetadataKey.RULE_KEY, keyFactories.getDefaultRuleKeyFactory().build(rule).toString());
    final BuildableContext buildableContext = new DefaultBuildableContext(buildInfoRecorder);
    final AtomicReference<Long> outputSize = Atomics.newReference();
    ListenableFuture<BuildResult> buildResult = processBuildRule(rule, buildContext, executionContext, onDiskBuildInfo, buildInfoRecorder, buildableContext, asyncCallbacks);
    // materialized locally by chaining up to our result future.
    if (buildMode == BuildMode.DEEP || buildMode == BuildMode.POPULATE_FROM_REMOTE_CACHE) {
        buildResult = MoreFutures.chainExceptions(getDepResults(rule, buildContext, executionContext, asyncCallbacks), buildResult);
    }
    // Setup a callback to handle either the cached or built locally cases.
    AsyncFunction<BuildResult, BuildResult> callback = input -> {
        if (input.getStatus() != BuildRuleStatus.SUCCESS) {
            return Futures.immediateFuture(input);
        }
        BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
        if (success != BuildRuleSuccessType.BUILT_LOCALLY) {
            for (String str : onDiskBuildInfo.getValuesOrThrow(BuildInfo.MetadataKey.RECORDED_PATHS)) {
                buildInfoRecorder.recordArtifact(Paths.get(str));
            }
        }
        outputSize.set(buildInfoRecorder.getOutputSize());
        if (success.outputsHaveChanged()) {
            if (rule instanceof HasPostBuildSteps) {
                executePostBuildSteps(rule, ((HasPostBuildSteps) rule).getPostBuildSteps(buildContext.getBuildContext()), executionContext);
            }
            for (Path path : buildInfoRecorder.getRecordedPaths()) {
                fileHashCache.invalidate(rule.getProjectFilesystem().resolve(path));
            }
        }
        if (useDependencyFileRuleKey(rule) && success == BuildRuleSuccessType.BUILT_LOCALLY) {
            ImmutableList<SourcePath> inputs = ((SupportsDependencyFileRuleKey) rule).getInputsAfterBuildingLocally(buildContext.getBuildContext());
            ImmutableList<String> inputStrings = inputs.stream().map(inputString -> DependencyFileEntry.fromSourcePath(inputString, pathResolver)).map(MoreFunctions.toJsonFunction(objectMapper)).collect(MoreCollectors.toImmutableList());
            buildInfoRecorder.addMetadata(BuildInfo.MetadataKey.DEP_FILE, inputStrings);
            Optional<RuleKeyAndInputs> depFileRuleKeyAndInputs = calculateDepFileRuleKey(rule, buildContext, Optional.of(inputStrings), false);
            if (depFileRuleKeyAndInputs.isPresent()) {
                RuleKey depFileRuleKey = depFileRuleKeyAndInputs.get().getRuleKey();
                buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY, depFileRuleKey.toString());
                if (useManifestCaching(rule)) {
                    Optional<RuleKeyAndInputs> manifestKey = calculateManifestKey(rule, buildContext.getEventBus());
                    if (manifestKey.isPresent()) {
                        buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.MANIFEST_KEY, manifestKey.get().getRuleKey().toString());
                        updateAndStoreManifest(rule, depFileRuleKeyAndInputs.get().getRuleKey(), depFileRuleKeyAndInputs.get().getInputs(), manifestKey.get(), buildContext.getArtifactCache());
                    }
                }
            }
        }
        if (success == BuildRuleSuccessType.BUILT_LOCALLY && shouldUploadToCache(rule, Preconditions.checkNotNull(outputSize.get()))) {
            ImmutableSortedMap.Builder<String, String> outputHashes = ImmutableSortedMap.naturalOrder();
            for (Path path : buildInfoRecorder.getOutputPaths()) {
                outputHashes.put(path.toString(), fileHashCache.get(rule.getProjectFilesystem().resolve(path)).toString());
            }
            buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.RECORDED_PATH_HASHES, outputHashes.build());
        }
        if (success != BuildRuleSuccessType.BUILT_LOCALLY && success.outputsHaveChanged()) {
            Optional<ImmutableMap<String, String>> hashes = onDiskBuildInfo.getBuildMap(BuildInfo.MetadataKey.RECORDED_PATH_HASHES);
            if (hashes.isPresent() && verifyRecordedPathHashes(rule.getBuildTarget(), rule.getProjectFilesystem(), hashes.get())) {
                for (Map.Entry<String, String> ent : hashes.get().entrySet()) {
                    Path path = rule.getProjectFilesystem().getPath(ent.getKey());
                    HashCode hashCode = HashCode.fromString(ent.getValue());
                    fileHashCache.set(rule.getProjectFilesystem().resolve(path), hashCode);
                }
            }
        }
        buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.TARGET, rule.getBuildTarget().toString());
        buildInfoRecorder.addMetadata(BuildInfo.MetadataKey.RECORDED_PATHS, buildInfoRecorder.getRecordedPaths().stream().map(Object::toString).collect(MoreCollectors.toImmutableList()));
        if (success.shouldWriteRecordedMetadataToDiskAfterBuilding()) {
            try {
                boolean clearExistingMetadata = success.shouldClearAndOverwriteMetadataOnDisk();
                buildInfoRecorder.writeMetadataToDisk(clearExistingMetadata);
            } catch (IOException e) {
                throw new IOException(String.format("Failed to write metadata to disk for %s.", rule), e);
            }
        }
        try {
            if (rule instanceof InitializableFromDisk) {
                doInitializeFromDisk((InitializableFromDisk<?>) rule, onDiskBuildInfo);
            }
        } catch (IOException e) {
            throw new IOException(String.format("Error initializing %s from disk.", rule), e);
        }
        return Futures.immediateFuture(input);
    };
    buildResult = Futures.transformAsync(buildResult, ruleAsyncFunction(rule, buildContext.getEventBus(), callback), serviceByAdjustingDefaultWeightsTo(RULE_KEY_COMPUTATION_RESOURCE_AMOUNTS));
    // Handle either build success or failure.
    final SettableFuture<BuildResult> result = SettableFuture.create();
    asyncCallbacks.add(MoreFutures.addListenableCallback(buildResult, new FutureCallback<BuildResult>() {

        // TODO(bolinfest): Delete all files produced by the rule, as they are not guaranteed
        // to be valid at this point?
        private void cleanupAfterError() {
            try {
                onDiskBuildInfo.deleteExistingMetadata();
            } catch (Throwable t) {
                buildContext.getEventBus().post(ThrowableConsoleEvent.create(t, "Error when deleting metadata for %s.", rule));
            }
        }

        private void uploadToCache(BuildRuleSuccessType success) {
            // Collect up all the rule keys we have index the artifact in the cache with.
            Set<RuleKey> ruleKeys = Sets.newHashSet();
            // If the rule key has changed (and is not already in the cache), we need to push
            // the artifact to cache using the new key.
            ruleKeys.add(keyFactories.getDefaultRuleKeyFactory().build(rule));
            // using the new key.
            if (SupportsInputBasedRuleKey.isSupported(rule)) {
                Optional<RuleKey> calculatedRuleKey = calculateInputBasedRuleKey(rule, buildContext.getEventBus());
                Optional<RuleKey> onDiskRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY);
                Optional<RuleKey> metaDataRuleKey = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY).map(RuleKey::new);
                Preconditions.checkState(calculatedRuleKey.equals(onDiskRuleKey), "%s (%s): %s: invalid on-disk input-based rule key: %s != %s", rule.getBuildTarget(), rule.getType(), success, calculatedRuleKey, onDiskRuleKey);
                Preconditions.checkState(calculatedRuleKey.equals(metaDataRuleKey), "%s: %s: invalid meta-data input-based rule key: %s != %s", rule.getBuildTarget(), success, calculatedRuleKey, metaDataRuleKey);
                ruleKeys.addAll(OptionalCompat.asSet(calculatedRuleKey));
            }
            // using the new key.
            if (useManifestCaching(rule)) {
                Optional<RuleKey> onDiskRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY);
                Optional<RuleKey> metaDataRuleKey = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY).map(RuleKey::new);
                Preconditions.checkState(onDiskRuleKey.equals(metaDataRuleKey), "%s: %s: inconsistent meta-data and on-disk dep-file rule key: %s != %s", rule.getBuildTarget(), success, onDiskRuleKey, metaDataRuleKey);
                ruleKeys.addAll(OptionalCompat.asSet(onDiskRuleKey));
            }
            // Do the actual upload.
            try {
                // Verify that the recorded path hashes are accurate.
                Optional<String> recordedPathHashes = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.RECORDED_PATH_HASHES);
                if (recordedPathHashes.isPresent() && !verifyRecordedPathHashes(rule.getBuildTarget(), rule.getProjectFilesystem(), recordedPathHashes.get())) {
                    return;
                }
                // Push to cache.
                buildInfoRecorder.performUploadToArtifactCache(ImmutableSet.copyOf(ruleKeys), buildContext.getArtifactCache(), buildContext.getEventBus());
            } catch (Throwable t) {
                buildContext.getEventBus().post(ThrowableConsoleEvent.create(t, "Error uploading to cache for %s.", rule));
            }
        }

        private void handleResult(BuildResult input) {
            Optional<Long> outputSize = Optional.empty();
            Optional<HashCode> outputHash = Optional.empty();
            Optional<BuildRuleSuccessType> successType = Optional.empty();
            BuildRuleEvent.Resumed resumedEvent = BuildRuleEvent.resumed(rule, buildRuleDurationTracker, keyFactories.getDefaultRuleKeyFactory());
            buildContext.getEventBus().logVerboseAndPost(LOG, resumedEvent);
            if (input.getStatus() == BuildRuleStatus.FAIL) {
                // Make this failure visible for other rules, so that they can stop early.
                firstFailure = input.getFailure();
                // If we failed, cleanup the state of this rule.
                cleanupAfterError();
            }
            // Unblock dependents.
            result.set(input);
            if (input.getStatus() == BuildRuleStatus.SUCCESS) {
                BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
                successType = Optional.of(success);
                // Try get the output size.
                try {
                    outputSize = Optional.of(buildInfoRecorder.getOutputSize());
                } catch (IOException e) {
                    buildContext.getEventBus().post(ThrowableConsoleEvent.create(e, "Error getting output size for %s.", rule));
                }
                // If this rule is cacheable, upload it to the cache.
                if (success.shouldUploadResultingArtifact() && outputSize.isPresent() && shouldUploadToCache(rule, outputSize.get())) {
                    uploadToCache(success);
                }
                // Calculate the hash of outputs that were built locally and are cacheable.
                if (success == BuildRuleSuccessType.BUILT_LOCALLY && shouldUploadToCache(rule, outputSize.get())) {
                    try {
                        outputHash = Optional.of(buildInfoRecorder.getOutputHash(fileHashCache));
                    } catch (IOException e) {
                        buildContext.getEventBus().post(ThrowableConsoleEvent.create(e, "Error getting output hash for %s.", rule));
                    }
                }
            }
            // Log the result to the event bus.
            BuildRuleEvent.Finished finished = BuildRuleEvent.finished(resumedEvent, BuildRuleKeys.builder().setRuleKey(keyFactories.getDefaultRuleKeyFactory().build(rule)).setInputRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY)).setDepFileRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY)).setManifestRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.MANIFEST_KEY)).build(), input.getStatus(), input.getCacheResult(), successType, outputHash, outputSize);
            buildContext.getEventBus().logVerboseAndPost(LOG, finished);
        }

        @Override
        public void onSuccess(BuildResult input) {
            handleResult(input);
        }

        @Override
        public void onFailure(@Nonnull Throwable thrown) {
            thrown = maybeAttachBuildRuleNameToException(thrown, rule);
            handleResult(BuildResult.failure(rule, thrown));
            // Reset interrupted flag once failure has been recorded.
            if (thrown instanceof InterruptedException) {
                Thread.currentThread().interrupt();
            }
        }
    }));
    return result;
}

Example 3 with RuleKeyFactories

use of com.facebook.buck.rules.keys.RuleKeyFactories in project buck by facebook.

the class CachingBuildEngine method calculateRuleKey.

private synchronized ListenableFuture<RuleKey> calculateRuleKey(final BuildRule rule, final BuildEngineBuildContext context) {
    ListenableFuture<RuleKey> ruleKey = ruleKeys.get(rule.getBuildTarget());
    if (ruleKey == null) {
        // Grab all the dependency rule key futures.  Since our rule key calculation depends on this
        // one, we need to wait for them to complete.
        ListenableFuture<List<RuleKey>> depKeys = Futures.transformAsync(ruleDeps.get(rule), deps -> {
            List<ListenableFuture<RuleKey>> depKeys1 = Lists.newArrayListWithExpectedSize(rule.getDeps().size());
            for (BuildRule dep : deps) {
                depKeys1.add(calculateRuleKey(dep, context));
            }
            return Futures.allAsList(depKeys1);
        }, serviceByAdjustingDefaultWeightsTo(RULE_KEY_COMPUTATION_RESOURCE_AMOUNTS));
        final RuleKeyFactories keyFactories = ruleKeyFactories.apply(rule.getProjectFilesystem());
        // Setup a future to calculate this rule key once the dependencies have been calculated.
        ruleKey = Futures.transform(depKeys, (List<RuleKey> input) -> {
            try (BuildRuleEvent.Scope scope = BuildRuleEvent.ruleKeyCalculationScope(context.getEventBus(), rule, buildRuleDurationTracker, keyFactories.getDefaultRuleKeyFactory())) {
                return keyFactories.getDefaultRuleKeyFactory().build(rule);
            }
        }, serviceByAdjustingDefaultWeightsTo(RULE_KEY_COMPUTATION_RESOURCE_AMOUNTS));
        // Record the rule key future.
        ruleKeys.put(rule.getBuildTarget(), ruleKey);
    }
    return ruleKey;
}