Examples with RuleKey com.facebook.buck.rules.RuleKey used on opensource projects

Example 61 with RuleKey

use of com.facebook.buck.rules.RuleKey in project buck by facebook.

the class ThriftArtifactCache method fetchImpl.

@Override
public CacheResult fetchImpl(RuleKey ruleKey, LazyPath output, HttpArtifactCacheEvent.Finished.Builder eventBuilder) throws IOException {
    BuckCacheFetchRequest fetchRequest = new BuckCacheFetchRequest();
    com.facebook.buck.artifact_cache.thrift.RuleKey thriftRuleKey = new com.facebook.buck.artifact_cache.thrift.RuleKey();
    thriftRuleKey.setHashString(ruleKey.getHashCode().toString());
    fetchRequest.setRuleKey(thriftRuleKey);
    fetchRequest.setRepository(repository);
    fetchRequest.setScheduleType(scheduleType);
    fetchRequest.setDistributedBuildModeEnabled(distributedBuildModeEnabled);
    BuckCacheRequest cacheRequest = new BuckCacheRequest();
    cacheRequest.setType(BuckCacheRequestType.FETCH);
    cacheRequest.setFetchRequest(fetchRequest);
    LOG.verbose("Will fetch key %s", thriftRuleKey);
    final ThriftArtifactCacheProtocol.Request request = ThriftArtifactCacheProtocol.createRequest(PROTOCOL, cacheRequest);
    Request.Builder builder = toOkHttpRequest(request);
    try (HttpResponse httpResponse = fetchClient.makeRequest(hybridThriftEndpoint, builder)) {
        if (httpResponse.statusCode() != 200) {
            String message = String.format("Failed to fetch cache artifact with HTTP status code [%d:%s] " + " to url [%s] for rule key [%s].", httpResponse.statusCode(), httpResponse.statusMessage(), httpResponse.requestUrl(), ruleKey.toString());
            LOG.error(message);
            return CacheResult.error(name, message);
        }
        try (ThriftArtifactCacheProtocol.Response response = ThriftArtifactCacheProtocol.parseResponse(PROTOCOL, httpResponse.getBody())) {
            eventBuilder.getFetchBuilder().setResponseSizeBytes(httpResponse.contentLength());
            BuckCacheResponse cacheResponse = response.getThriftData();
            if (!cacheResponse.isWasSuccessful()) {
                LOG.warn("Request was unsuccessful: %s", cacheResponse.getErrorMessage());
                return CacheResult.error(name, cacheResponse.getErrorMessage());
            }
            BuckCacheFetchResponse fetchResponse = cacheResponse.getFetchResponse();
            if (LOG.isDebugEnabled()) {
                LOG.debug("Debug info for cache fetch request: request=[%s] response=[%s]", ThriftUtil.thriftToDebugJson(cacheRequest), ThriftUtil.thriftToDebugJson(cacheResponse));
            }
            if (!fetchResponse.isArtifactExists()) {
                LOG.verbose("Artifact did not exist.");
                return CacheResult.miss();
            }
            LOG.verbose("Got artifact.  Attempting to read payload.");
            Path tmp = createTempFileForDownload();
            ThriftArtifactCacheProtocol.Response.ReadPayloadInfo readResult;
            try (OutputStream tmpFile = projectFilesystem.newFileOutputStream(tmp)) {
                readResult = response.readPayload(tmpFile);
                LOG.verbose("Successfully read payload: %d bytes.", readResult.getBytesRead());
            }
            ArtifactMetadata metadata = fetchResponse.getMetadata();
            if (LOG.isVerboseEnabled()) {
                LOG.verbose(String.format("Fetched artifact with rule key [%s] contains the following metadata: [%s]", ruleKey, ThriftUtil.thriftToDebugJson(metadata)));
            }
            eventBuilder.setTarget(Optional.ofNullable(metadata.getBuildTarget())).getFetchBuilder().setAssociatedRuleKeys(toImmutableSet(metadata.getRuleKeys())).setArtifactSizeBytes(readResult.getBytesRead());
            if (!metadata.isSetArtifactPayloadMd5()) {
                String msg = "Fetched artifact is missing the MD5 hash.";
                LOG.warn(msg);
            } else {
                eventBuilder.getFetchBuilder().setArtifactContentHash(metadata.getArtifactPayloadMd5());
                if (!readResult.getMd5Hash().equals(fetchResponse.getMetadata().getArtifactPayloadMd5())) {
                    String msg = String.format("The artifact fetched from cache is corrupted. ExpectedMD5=[%s] ActualMD5=[%s]", fetchResponse.getMetadata().getArtifactPayloadMd5(), readResult.getMd5Hash());
                    LOG.error(msg);
                    return CacheResult.error(name, msg);
                }
            }
            // This makes sure we don't have 'half downloaded files' in the dir cache.
            projectFilesystem.move(tmp, output.get(), StandardCopyOption.REPLACE_EXISTING);
            return CacheResult.hit(name, ImmutableMap.copyOf(fetchResponse.getMetadata().getMetadata()), readResult.getBytesRead());
        }
    }
}

Example 62 with RuleKey

use of com.facebook.buck.rules.RuleKey in project buck by facebook.

the class TwoLevelArtifactCacheDecorator method fetch.

@Override
public CacheResult fetch(RuleKey ruleKey, LazyPath output) {
    CacheResult fetchResult = delegate.fetch(ruleKey, output);
    if (!fetchResult.getType().isSuccess()) {
        LOG.verbose("Missed first-level lookup.");
        return fetchResult;
    } else if (!fetchResult.getMetadata().containsKey(METADATA_KEY)) {
        LOG.verbose("Found a single-level entry.");
        return fetchResult;
    }
    LOG.verbose("Found a first-level artifact with metadata: %s", fetchResult.getMetadata());
    CacheResult outputFileFetchResult = delegate.fetch(new RuleKey(fetchResult.getMetadata().get(METADATA_KEY)), output);
    if (!outputFileFetchResult.getType().isSuccess()) {
        LOG.verbose("Missed second-level lookup.");
        secondLevelCacheMisses.inc();
        return outputFileFetchResult;
    }
    if (outputFileFetchResult.cacheSource().isPresent()) {
        secondLevelCacheHitTypes.add(outputFileFetchResult.cacheSource().get());
    }
    if (outputFileFetchResult.artifactSizeBytes().isPresent()) {
        secondLevelCacheHitBytes.addSample(outputFileFetchResult.artifactSizeBytes().get());
    }
    LOG.verbose("Found a second-level artifact with metadata: %s", outputFileFetchResult.getMetadata());
    return fetchResult;
}

Example 63 with RuleKey

use of com.facebook.buck.rules.RuleKey in project buck by facebook.

the class CacheCommand method runWithoutHelp.

@Override
public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException {
    params.getBuckEventBus().post(ConsoleEvent.fine("cache command start"));
    if (isNoCache()) {
        params.getBuckEventBus().post(ConsoleEvent.severe("Caching is disabled."));
        return 1;
    }
    List<String> arguments = getArguments();
    if (arguments.isEmpty()) {
        params.getBuckEventBus().post(ConsoleEvent.severe("No cache keys specified."));
        return 1;
    }
    if (outputDir != null) {
        outputPath = Optional.of(Paths.get(outputDir));
        Files.createDirectories(outputPath.get());
    }
    ArtifactCache cache = params.getArtifactCacheFactory().newInstance();
    List<RuleKey> ruleKeys = new ArrayList<>();
    for (String hash : arguments) {
        ruleKeys.add(new RuleKey(hash));
    }
    Path tmpDir = Files.createTempDirectory("buck-cache-command");
    BuildEvent.Started started = BuildEvent.started(getArguments(), false);
    List<ArtifactRunner> results = null;
    try (CommandThreadManager pool = new CommandThreadManager("Build", getConcurrencyLimit(params.getBuckConfig()))) {
        WeightedListeningExecutorService executor = pool.getExecutor();
        fakeOutParseEvents(params.getBuckEventBus());
        // Post the build started event, setting it to the Parser recorded start time if appropriate.
        if (params.getParser().getParseStartTime().isPresent()) {
            params.getBuckEventBus().post(started, params.getParser().getParseStartTime().get());
        } else {
            params.getBuckEventBus().post(started);
        }
        // Fetch all artifacts
        List<ListenableFuture<ArtifactRunner>> futures = new ArrayList<>();
        for (RuleKey ruleKey : ruleKeys) {
            futures.add(executor.submit(new ArtifactRunner(ruleKey, tmpDir, cache)));
        }
        // Wait for all executions to complete or fail.
        try {
            results = Futures.allAsList(futures).get();
        } catch (ExecutionException ex) {
            params.getConsole().printBuildFailure("Failed");
            ex.printStackTrace(params.getConsole().getStdErr());
        }
    }
    int totalRuns = results.size();
    String resultString = "";
    int goodRuns = 0;
    for (ArtifactRunner r : results) {
        if (r.completed) {
            goodRuns++;
        }
        resultString += r.resultString;
        if (!outputPath.isPresent()) {
            // legacy output
            if (r.completed) {
                params.getConsole().printSuccess(String.format("Successfully downloaded artifact with id %s at %s .", r.ruleKey, r.artifact));
            } else {
                params.getConsole().printErrorText(String.format("Failed to retrieve an artifact with id %s.", r.ruleKey));
            }
        }
    }
    int exitCode = (totalRuns == goodRuns) ? 0 : 1;
    params.getBuckEventBus().post(BuildEvent.finished(started, exitCode));
    if (outputPath.isPresent()) {
        if (totalRuns == goodRuns) {
            params.getConsole().printSuccess("Successfully downloaded all artifacts.");
        } else {
            params.getConsole().printErrorText(String.format("Downloaded %d of %d artifacts", goodRuns, totalRuns));
        }
        params.getConsole().getStdOut().println(resultString);
    }
    return exitCode;
}

Example 64 with RuleKey

use of com.facebook.buck.rules.RuleKey in project buck by facebook.

the class HeaderSymlinkTreeWithHeaderMapTest method testSymlinkTreeInputBasedRuleKeyDoesNotChangeIfLinkTargetsChange.

@Test
public void testSymlinkTreeInputBasedRuleKeyDoesNotChangeIfLinkTargetsChange() throws IOException {
    ruleResolver.addToIndex(symlinkTreeBuildRule);
    InputBasedRuleKeyFactory ruleKeyFactory = new InputBasedRuleKeyFactory(0, FakeFileHashCache.createFromStrings(ImmutableMap.of()), resolver, ruleFinder);
    // Calculate the rule key
    RuleKey key1 = ruleKeyFactory.build(symlinkTreeBuildRule);
    // Change the contents of the target of the link.
    Path existingFile = resolver.getAbsolutePath(links.values().asList().get(0));
    Files.write(existingFile, "something new".getBytes(Charsets.UTF_8));
    ruleKeyFactory = new InputBasedRuleKeyFactory(0, FakeFileHashCache.createFromStrings(ImmutableMap.of()), resolver, ruleFinder);
    // Re-calculate the rule key
    RuleKey key2 = ruleKeyFactory.build(symlinkTreeBuildRule);
    // Verify that the rules keys are the same.
    assertEquals(key1, key2);
}

Example 65 with RuleKey

use of com.facebook.buck.rules.RuleKey in project buck by facebook.

the class HeaderSymlinkTreeWithHeaderMapTest method testSymlinkTreeRuleKeyChangesIfLinkTargetsChange.

@Test
public void testSymlinkTreeRuleKeyChangesIfLinkTargetsChange() throws IOException {
    ruleResolver.addToIndex(symlinkTreeBuildRule);
    DefaultFileHashCache hashCache = DefaultFileHashCache.createDefaultFileHashCache(new ProjectFilesystem(tmpDir.getRoot()));
    FileHashLoader hashLoader = new StackedFileHashCache(ImmutableList.of(hashCache));
    DefaultRuleKeyFactory ruleKeyFactory = new DefaultRuleKeyFactory(0, hashLoader, resolver, ruleFinder);
    // Calculate the rule key
    RuleKey key1 = ruleKeyFactory.build(symlinkTreeBuildRule);
    // Change the contents of the target of the link.
    Path existingFile = resolver.getAbsolutePath(links.values().asList().get(0));
    Files.write(existingFile, "something new".getBytes(Charsets.UTF_8));
    hashCache.invalidateAll();
    ruleKeyFactory = new DefaultRuleKeyFactory(0, hashLoader, resolver, ruleFinder);
    // Re-calculate the rule key
    RuleKey key2 = ruleKeyFactory.build(symlinkTreeBuildRule);
    // Verify that the rules keys are the same.
    assertNotEquals(key1, key2);
}