Examples with FileHashCache com.facebook.buck.util.cache.FileHashCache used on opensource projects

Example 41 with FileHashCache

use of com.facebook.buck.util.cache.FileHashCache in project buck by facebook.

the class InputBasedRuleKeyFactoryTest method ruleKeyNotCalculatedIfSizeLimitHitWithMultipleInputs.

@Test
public void ruleKeyNotCalculatedIfSizeLimitHitWithMultipleInputs() throws Exception {
    BuildRuleResolver resolver = new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer());
    RuleKeyFieldLoader fieldLoader = new RuleKeyFieldLoader(0);
    SourcePathRuleFinder ruleFinder = new SourcePathRuleFinder(resolver);
    SourcePathResolver pathResolver = new SourcePathResolver(ruleFinder);
    FakeProjectFilesystem filesystem = new FakeProjectFilesystem();
    FileHashCache hashCache = new StackedFileHashCache(ImmutableList.of(DefaultFileHashCache.createDefaultFileHashCache(filesystem)));
    // Create inputs that combine to pass size limit.
    Path input1 = filesystem.getPath("input1");
    filesystem.writeBytesToPath(new byte[150], input1);
    Path input2 = filesystem.getPath("input2");
    filesystem.writeBytesToPath(new byte[150], input2);
    // Construct rule which uses inputs.
    BuildRule rule = GenruleBuilder.newGenruleBuilder(BuildTargetFactory.newInstance("//:rule")).setOut("out").setSrcs(ImmutableList.of(new PathSourcePath(filesystem, input1), new PathSourcePath(filesystem, input2))).build(resolver, filesystem);
    // Verify rule key isn't calculated.
    expectedException.expect(SizeLimiter.SizeLimitException.class);
    new InputBasedRuleKeyFactory(fieldLoader, hashCache, pathResolver, ruleFinder, 200).build(rule);
}

Example 42 with FileHashCache

use of com.facebook.buck.util.cache.FileHashCache in project buck by facebook.

the class InputBasedRuleKeyFactoryTest method nestedSizeLimitExceptionHandled.

@Test
public void nestedSizeLimitExceptionHandled() throws Exception {
    BuildRuleResolver resolver = new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer());
    RuleKeyFieldLoader fieldLoader = new RuleKeyFieldLoader(0);
    SourcePathRuleFinder ruleFinder = new SourcePathRuleFinder(resolver);
    SourcePathResolver pathResolver = new SourcePathResolver(ruleFinder);
    final FakeProjectFilesystem filesystem = new FakeProjectFilesystem();
    FileHashCache hashCache = new StackedFileHashCache(ImmutableList.of(DefaultFileHashCache.createDefaultFileHashCache(filesystem)));
    Path inputFile = filesystem.getPath("input");
    filesystem.writeBytesToPath(new byte[1024], inputFile);
    BuildRuleParams params = new FakeBuildRuleParamsBuilder("//:rule").setProjectFilesystem(filesystem).build();
    BuildRule rule = new NoopBuildRule(params) {

        @AddToRuleKey
        NestedRuleKeyAppendableWithInput input = new NestedRuleKeyAppendableWithInput(new PathSourcePath(filesystem, inputFile));
    };
    // Verify rule key isn't calculated.
    expectedException.expect(SizeLimiter.SizeLimitException.class);
    new InputBasedRuleKeyFactory(fieldLoader, hashCache, pathResolver, ruleFinder, 200).build(rule);
}

Example 43 with FileHashCache

use of com.facebook.buck.util.cache.FileHashCache in project buck by facebook.

the class CachingBuildEngine method processBuildRule.

private ListenableFuture<BuildResult> processBuildRule(BuildRule rule, BuildEngineBuildContext buildContext, ExecutionContext executionContext, ConcurrentLinkedQueue<ListenableFuture<Void>> asyncCallbacks) {
    final RuleKeyFactories keyFactories = ruleKeyFactories.apply(rule.getProjectFilesystem());
    final OnDiskBuildInfo onDiskBuildInfo = buildContext.createOnDiskBuildInfoFor(rule.getBuildTarget(), rule.getProjectFilesystem());
    final BuildInfoRecorder buildInfoRecorder = buildContext.createBuildInfoRecorder(rule.getBuildTarget(), rule.getProjectFilesystem()).addBuildMetadata(BuildInfo.MetadataKey.RULE_KEY, keyFactories.getDefaultRuleKeyFactory().build(rule).toString());
    final BuildableContext buildableContext = new DefaultBuildableContext(buildInfoRecorder);
    final AtomicReference<Long> outputSize = Atomics.newReference();
    ListenableFuture<BuildResult> buildResult = processBuildRule(rule, buildContext, executionContext, onDiskBuildInfo, buildInfoRecorder, buildableContext, asyncCallbacks);
    // materialized locally by chaining up to our result future.
    if (buildMode == BuildMode.DEEP || buildMode == BuildMode.POPULATE_FROM_REMOTE_CACHE) {
        buildResult = MoreFutures.chainExceptions(getDepResults(rule, buildContext, executionContext, asyncCallbacks), buildResult);
    }
    // Setup a callback to handle either the cached or built locally cases.
    AsyncFunction<BuildResult, BuildResult> callback = input -> {
        if (input.getStatus() != BuildRuleStatus.SUCCESS) {
            return Futures.immediateFuture(input);
        }
        BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
        if (success != BuildRuleSuccessType.BUILT_LOCALLY) {
            for (String str : onDiskBuildInfo.getValuesOrThrow(BuildInfo.MetadataKey.RECORDED_PATHS)) {
                buildInfoRecorder.recordArtifact(Paths.get(str));
            }
        }
        outputSize.set(buildInfoRecorder.getOutputSize());
        if (success.outputsHaveChanged()) {
            if (rule instanceof HasPostBuildSteps) {
                executePostBuildSteps(rule, ((HasPostBuildSteps) rule).getPostBuildSteps(buildContext.getBuildContext()), executionContext);
            }
            for (Path path : buildInfoRecorder.getRecordedPaths()) {
                fileHashCache.invalidate(rule.getProjectFilesystem().resolve(path));
            }
        }
        if (useDependencyFileRuleKey(rule) && success == BuildRuleSuccessType.BUILT_LOCALLY) {
            ImmutableList<SourcePath> inputs = ((SupportsDependencyFileRuleKey) rule).getInputsAfterBuildingLocally(buildContext.getBuildContext());
            ImmutableList<String> inputStrings = inputs.stream().map(inputString -> DependencyFileEntry.fromSourcePath(inputString, pathResolver)).map(MoreFunctions.toJsonFunction(objectMapper)).collect(MoreCollectors.toImmutableList());
            buildInfoRecorder.addMetadata(BuildInfo.MetadataKey.DEP_FILE, inputStrings);
            Optional<RuleKeyAndInputs> depFileRuleKeyAndInputs = calculateDepFileRuleKey(rule, buildContext, Optional.of(inputStrings), false);
            if (depFileRuleKeyAndInputs.isPresent()) {
                RuleKey depFileRuleKey = depFileRuleKeyAndInputs.get().getRuleKey();
                buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY, depFileRuleKey.toString());
                if (useManifestCaching(rule)) {
                    Optional<RuleKeyAndInputs> manifestKey = calculateManifestKey(rule, buildContext.getEventBus());
                    if (manifestKey.isPresent()) {
                        buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.MANIFEST_KEY, manifestKey.get().getRuleKey().toString());
                        updateAndStoreManifest(rule, depFileRuleKeyAndInputs.get().getRuleKey(), depFileRuleKeyAndInputs.get().getInputs(), manifestKey.get(), buildContext.getArtifactCache());
                    }
                }
            }
        }
        if (success == BuildRuleSuccessType.BUILT_LOCALLY && shouldUploadToCache(rule, Preconditions.checkNotNull(outputSize.get()))) {
            ImmutableSortedMap.Builder<String, String> outputHashes = ImmutableSortedMap.naturalOrder();
            for (Path path : buildInfoRecorder.getOutputPaths()) {
                outputHashes.put(path.toString(), fileHashCache.get(rule.getProjectFilesystem().resolve(path)).toString());
            }
            buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.RECORDED_PATH_HASHES, outputHashes.build());
        }
        if (success != BuildRuleSuccessType.BUILT_LOCALLY && success.outputsHaveChanged()) {
            Optional<ImmutableMap<String, String>> hashes = onDiskBuildInfo.getBuildMap(BuildInfo.MetadataKey.RECORDED_PATH_HASHES);
            if (hashes.isPresent() && verifyRecordedPathHashes(rule.getBuildTarget(), rule.getProjectFilesystem(), hashes.get())) {
                for (Map.Entry<String, String> ent : hashes.get().entrySet()) {
                    Path path = rule.getProjectFilesystem().getPath(ent.getKey());
                    HashCode hashCode = HashCode.fromString(ent.getValue());
                    fileHashCache.set(rule.getProjectFilesystem().resolve(path), hashCode);
                }
            }
        }
        buildInfoRecorder.addBuildMetadata(BuildInfo.MetadataKey.TARGET, rule.getBuildTarget().toString());
        buildInfoRecorder.addMetadata(BuildInfo.MetadataKey.RECORDED_PATHS, buildInfoRecorder.getRecordedPaths().stream().map(Object::toString).collect(MoreCollectors.toImmutableList()));
        if (success.shouldWriteRecordedMetadataToDiskAfterBuilding()) {
            try {
                boolean clearExistingMetadata = success.shouldClearAndOverwriteMetadataOnDisk();
                buildInfoRecorder.writeMetadataToDisk(clearExistingMetadata);
            } catch (IOException e) {
                throw new IOException(String.format("Failed to write metadata to disk for %s.", rule), e);
            }
        }
        try {
            if (rule instanceof InitializableFromDisk) {
                doInitializeFromDisk((InitializableFromDisk<?>) rule, onDiskBuildInfo);
            }
        } catch (IOException e) {
            throw new IOException(String.format("Error initializing %s from disk.", rule), e);
        }
        return Futures.immediateFuture(input);
    };
    buildResult = Futures.transformAsync(buildResult, ruleAsyncFunction(rule, buildContext.getEventBus(), callback), serviceByAdjustingDefaultWeightsTo(RULE_KEY_COMPUTATION_RESOURCE_AMOUNTS));
    // Handle either build success or failure.
    final SettableFuture<BuildResult> result = SettableFuture.create();
    asyncCallbacks.add(MoreFutures.addListenableCallback(buildResult, new FutureCallback<BuildResult>() {

        // TODO(bolinfest): Delete all files produced by the rule, as they are not guaranteed
        // to be valid at this point?
        private void cleanupAfterError() {
            try {
                onDiskBuildInfo.deleteExistingMetadata();
            } catch (Throwable t) {
                buildContext.getEventBus().post(ThrowableConsoleEvent.create(t, "Error when deleting metadata for %s.", rule));
            }
        }

        private void uploadToCache(BuildRuleSuccessType success) {
            // Collect up all the rule keys we have index the artifact in the cache with.
            Set<RuleKey> ruleKeys = Sets.newHashSet();
            // If the rule key has changed (and is not already in the cache), we need to push
            // the artifact to cache using the new key.
            ruleKeys.add(keyFactories.getDefaultRuleKeyFactory().build(rule));
            // using the new key.
            if (SupportsInputBasedRuleKey.isSupported(rule)) {
                Optional<RuleKey> calculatedRuleKey = calculateInputBasedRuleKey(rule, buildContext.getEventBus());
                Optional<RuleKey> onDiskRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY);
                Optional<RuleKey> metaDataRuleKey = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY).map(RuleKey::new);
                Preconditions.checkState(calculatedRuleKey.equals(onDiskRuleKey), "%s (%s): %s: invalid on-disk input-based rule key: %s != %s", rule.getBuildTarget(), rule.getType(), success, calculatedRuleKey, onDiskRuleKey);
                Preconditions.checkState(calculatedRuleKey.equals(metaDataRuleKey), "%s: %s: invalid meta-data input-based rule key: %s != %s", rule.getBuildTarget(), success, calculatedRuleKey, metaDataRuleKey);
                ruleKeys.addAll(OptionalCompat.asSet(calculatedRuleKey));
            }
            // using the new key.
            if (useManifestCaching(rule)) {
                Optional<RuleKey> onDiskRuleKey = onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY);
                Optional<RuleKey> metaDataRuleKey = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY).map(RuleKey::new);
                Preconditions.checkState(onDiskRuleKey.equals(metaDataRuleKey), "%s: %s: inconsistent meta-data and on-disk dep-file rule key: %s != %s", rule.getBuildTarget(), success, onDiskRuleKey, metaDataRuleKey);
                ruleKeys.addAll(OptionalCompat.asSet(onDiskRuleKey));
            }
            // Do the actual upload.
            try {
                // Verify that the recorded path hashes are accurate.
                Optional<String> recordedPathHashes = buildInfoRecorder.getBuildMetadataFor(BuildInfo.MetadataKey.RECORDED_PATH_HASHES);
                if (recordedPathHashes.isPresent() && !verifyRecordedPathHashes(rule.getBuildTarget(), rule.getProjectFilesystem(), recordedPathHashes.get())) {
                    return;
                }
                // Push to cache.
                buildInfoRecorder.performUploadToArtifactCache(ImmutableSet.copyOf(ruleKeys), buildContext.getArtifactCache(), buildContext.getEventBus());
            } catch (Throwable t) {
                buildContext.getEventBus().post(ThrowableConsoleEvent.create(t, "Error uploading to cache for %s.", rule));
            }
        }

        private void handleResult(BuildResult input) {
            Optional<Long> outputSize = Optional.empty();
            Optional<HashCode> outputHash = Optional.empty();
            Optional<BuildRuleSuccessType> successType = Optional.empty();
            BuildRuleEvent.Resumed resumedEvent = BuildRuleEvent.resumed(rule, buildRuleDurationTracker, keyFactories.getDefaultRuleKeyFactory());
            buildContext.getEventBus().logVerboseAndPost(LOG, resumedEvent);
            if (input.getStatus() == BuildRuleStatus.FAIL) {
                // Make this failure visible for other rules, so that they can stop early.
                firstFailure = input.getFailure();
                // If we failed, cleanup the state of this rule.
                cleanupAfterError();
            }
            // Unblock dependents.
            result.set(input);
            if (input.getStatus() == BuildRuleStatus.SUCCESS) {
                BuildRuleSuccessType success = Preconditions.checkNotNull(input.getSuccess());
                successType = Optional.of(success);
                // Try get the output size.
                try {
                    outputSize = Optional.of(buildInfoRecorder.getOutputSize());
                } catch (IOException e) {
                    buildContext.getEventBus().post(ThrowableConsoleEvent.create(e, "Error getting output size for %s.", rule));
                }
                // If this rule is cacheable, upload it to the cache.
                if (success.shouldUploadResultingArtifact() && outputSize.isPresent() && shouldUploadToCache(rule, outputSize.get())) {
                    uploadToCache(success);
                }
                // Calculate the hash of outputs that were built locally and are cacheable.
                if (success == BuildRuleSuccessType.BUILT_LOCALLY && shouldUploadToCache(rule, outputSize.get())) {
                    try {
                        outputHash = Optional.of(buildInfoRecorder.getOutputHash(fileHashCache));
                    } catch (IOException e) {
                        buildContext.getEventBus().post(ThrowableConsoleEvent.create(e, "Error getting output hash for %s.", rule));
                    }
                }
            }
            // Log the result to the event bus.
            BuildRuleEvent.Finished finished = BuildRuleEvent.finished(resumedEvent, BuildRuleKeys.builder().setRuleKey(keyFactories.getDefaultRuleKeyFactory().build(rule)).setInputRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.INPUT_BASED_RULE_KEY)).setDepFileRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.DEP_FILE_RULE_KEY)).setManifestRuleKey(onDiskBuildInfo.getRuleKey(BuildInfo.MetadataKey.MANIFEST_KEY)).build(), input.getStatus(), input.getCacheResult(), successType, outputHash, outputSize);
            buildContext.getEventBus().logVerboseAndPost(LOG, finished);
        }

        @Override
        public void onSuccess(BuildResult input) {
            handleResult(input);
        }

        @Override
        public void onFailure(@Nonnull Throwable thrown) {
            thrown = maybeAttachBuildRuleNameToException(thrown, rule);
            handleResult(BuildResult.failure(rule, thrown));
            // Reset interrupted flag once failure has been recorded.
            if (thrown instanceof InterruptedException) {
                Thread.currentThread().interrupt();
            }
        }
    }));
    return result;
}

Example 44 with FileHashCache

use of com.facebook.buck.util.cache.FileHashCache in project buck by facebook.

the class DirArtifactCacheTest method testCacheStoresAndFetchHits.

@Test
public void testCacheStoresAndFetchHits() throws IOException {
    Path cacheDir = tmpDir.newFolder();
    Path fileX = tmpDir.newFile("x");
    Path fileY = tmpDir.newFile("y");
    Path fileZ = tmpDir.newFile("z");
    fileHashCache = new FakeFileHashCache(ImmutableMap.of(fileX, HashCode.fromInt(0), fileY, HashCode.fromInt(1), fileZ, HashCode.fromInt(2)));
    dirArtifactCache = new DirArtifactCache("dir", new ProjectFilesystem(cacheDir), Paths.get("."), /* doStore */
    true, /* maxCacheSizeBytes */
    Optional.empty());
    Files.write(fileX, "x".getBytes(UTF_8));
    Files.write(fileY, "y".getBytes(UTF_8));
    Files.write(fileZ, "x".getBytes(UTF_8));
    BuildRule inputRuleX = new BuildRuleForTest(fileX);
    BuildRule inputRuleY = new BuildRuleForTest(fileY);
    BuildRule inputRuleZ = new BuildRuleForTest(fileZ);
    assertFalse(inputRuleX.equals(inputRuleY));
    assertFalse(inputRuleX.equals(inputRuleZ));
    assertFalse(inputRuleY.equals(inputRuleZ));
    BuildRuleResolver ruleResolver = new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer());
    ruleResolver.addToIndex(inputRuleX);
    ruleResolver.addToIndex(inputRuleY);
    ruleResolver.addToIndex(inputRuleZ);
    SourcePathRuleFinder ruleFinder = new SourcePathRuleFinder(ruleResolver);
    SourcePathResolver resolver = new SourcePathResolver(ruleFinder);
    DefaultRuleKeyFactory fakeRuleKeyFactory = new DefaultRuleKeyFactory(0, fileHashCache, resolver, ruleFinder);
    RuleKey ruleKeyX = fakeRuleKeyFactory.build(inputRuleX);
    RuleKey ruleKeyY = fakeRuleKeyFactory.build(inputRuleY);
    RuleKey ruleKeyZ = fakeRuleKeyFactory.build(inputRuleZ);
    assertEquals(CacheResultType.MISS, dirArtifactCache.fetch(ruleKeyX, LazyPath.ofInstance(fileX)).getType());
    assertEquals(CacheResultType.MISS, dirArtifactCache.fetch(ruleKeyY, LazyPath.ofInstance(fileY)).getType());
    assertEquals(CacheResultType.MISS, dirArtifactCache.fetch(ruleKeyZ, LazyPath.ofInstance(fileZ)).getType());
    dirArtifactCache.store(ArtifactInfo.builder().addRuleKeys(ruleKeyX).build(), BorrowablePath.notBorrowablePath(fileX));
    dirArtifactCache.store(ArtifactInfo.builder().addRuleKeys(ruleKeyY).build(), BorrowablePath.notBorrowablePath(fileY));
    dirArtifactCache.store(ArtifactInfo.builder().addRuleKeys(ruleKeyZ).build(), BorrowablePath.notBorrowablePath(fileZ));
    Files.delete(fileX);
    Files.delete(fileY);
    Files.delete(fileZ);
    assertEquals(CacheResultType.HIT, dirArtifactCache.fetch(ruleKeyX, LazyPath.ofInstance(fileX)).getType());
    assertEquals(CacheResultType.HIT, dirArtifactCache.fetch(ruleKeyY, LazyPath.ofInstance(fileY)).getType());
    assertEquals(CacheResultType.HIT, dirArtifactCache.fetch(ruleKeyZ, LazyPath.ofInstance(fileZ)).getType());
    assertEquals(inputRuleX, new BuildRuleForTest(fileX));
    assertEquals(inputRuleY, new BuildRuleForTest(fileY));
    assertEquals(inputRuleZ, new BuildRuleForTest(fileZ));
    ImmutableList<Path> cachedFiles = ImmutableList.copyOf(dirArtifactCache.getAllFilesInCache());
    assertEquals(6, cachedFiles.size());
    ImmutableSet<String> filenames = cachedFiles.stream().map(input -> input.getFileName().toString()).collect(MoreCollectors.toImmutableSet());
    for (RuleKey ruleKey : ImmutableSet.of(ruleKeyX, ruleKeyY, ruleKeyZ)) {
        assertThat(filenames, Matchers.hasItem(ruleKey.toString()));
        assertThat(filenames, Matchers.hasItem(ruleKey.toString() + ".metadata"));
    }
}

Example 45 with FileHashCache

use of com.facebook.buck.util.cache.FileHashCache in project buck by facebook.

the class RuleKeyFactoryManager method create.

private RuleKeyFactories create(ProjectFilesystem filesystem) {
    RuleKeyFieldLoader fieldLoader = new RuleKeyFieldLoader(keySeed);
    SourcePathRuleFinder ruleFinder = new SourcePathRuleFinder(resolver);
    SourcePathResolver pathResolver = new SourcePathResolver(ruleFinder);
    FileHashCache fileHashCache = fileHashCacheProvider.apply(filesystem);
    return RuleKeyFactories.of(new DefaultRuleKeyFactory(fieldLoader, fileHashCache, pathResolver, ruleFinder, defaultRuleKeyFactoryCache), new InputBasedRuleKeyFactory(fieldLoader, fileHashCache, pathResolver, ruleFinder, inputRuleKeyFileSizeLimit), new DefaultDependencyFileRuleKeyFactory(fieldLoader, fileHashCache, pathResolver, ruleFinder));
}