Examples with LazyPath com.facebook.buck.io.LazyPath used on opensource projects

Example 6 with LazyPath

use of com.facebook.buck.io.LazyPath in project buck by facebook.

the class ThriftArtifactCache method fetchImpl.

@Override
public CacheResult fetchImpl(RuleKey ruleKey, LazyPath output, HttpArtifactCacheEvent.Finished.Builder eventBuilder) throws IOException {
    BuckCacheFetchRequest fetchRequest = new BuckCacheFetchRequest();
    com.facebook.buck.artifact_cache.thrift.RuleKey thriftRuleKey = new com.facebook.buck.artifact_cache.thrift.RuleKey();
    thriftRuleKey.setHashString(ruleKey.getHashCode().toString());
    fetchRequest.setRuleKey(thriftRuleKey);
    fetchRequest.setRepository(repository);
    fetchRequest.setScheduleType(scheduleType);
    fetchRequest.setDistributedBuildModeEnabled(distributedBuildModeEnabled);
    BuckCacheRequest cacheRequest = new BuckCacheRequest();
    cacheRequest.setType(BuckCacheRequestType.FETCH);
    cacheRequest.setFetchRequest(fetchRequest);
    LOG.verbose("Will fetch key %s", thriftRuleKey);
    final ThriftArtifactCacheProtocol.Request request = ThriftArtifactCacheProtocol.createRequest(PROTOCOL, cacheRequest);
    Request.Builder builder = toOkHttpRequest(request);
    try (HttpResponse httpResponse = fetchClient.makeRequest(hybridThriftEndpoint, builder)) {
        if (httpResponse.statusCode() != 200) {
            String message = String.format("Failed to fetch cache artifact with HTTP status code [%d:%s] " + " to url [%s] for rule key [%s].", httpResponse.statusCode(), httpResponse.statusMessage(), httpResponse.requestUrl(), ruleKey.toString());
            LOG.error(message);
            return CacheResult.error(name, message);
        }
        try (ThriftArtifactCacheProtocol.Response response = ThriftArtifactCacheProtocol.parseResponse(PROTOCOL, httpResponse.getBody())) {
            eventBuilder.getFetchBuilder().setResponseSizeBytes(httpResponse.contentLength());
            BuckCacheResponse cacheResponse = response.getThriftData();
            if (!cacheResponse.isWasSuccessful()) {
                LOG.warn("Request was unsuccessful: %s", cacheResponse.getErrorMessage());
                return CacheResult.error(name, cacheResponse.getErrorMessage());
            }
            BuckCacheFetchResponse fetchResponse = cacheResponse.getFetchResponse();
            if (LOG.isDebugEnabled()) {
                LOG.debug("Debug info for cache fetch request: request=[%s] response=[%s]", ThriftUtil.thriftToDebugJson(cacheRequest), ThriftUtil.thriftToDebugJson(cacheResponse));
            }
            if (!fetchResponse.isArtifactExists()) {
                LOG.verbose("Artifact did not exist.");
                return CacheResult.miss();
            }
            LOG.verbose("Got artifact.  Attempting to read payload.");
            Path tmp = createTempFileForDownload();
            ThriftArtifactCacheProtocol.Response.ReadPayloadInfo readResult;
            try (OutputStream tmpFile = projectFilesystem.newFileOutputStream(tmp)) {
                readResult = response.readPayload(tmpFile);
                LOG.verbose("Successfully read payload: %d bytes.", readResult.getBytesRead());
            }
            ArtifactMetadata metadata = fetchResponse.getMetadata();
            if (LOG.isVerboseEnabled()) {
                LOG.verbose(String.format("Fetched artifact with rule key [%s] contains the following metadata: [%s]", ruleKey, ThriftUtil.thriftToDebugJson(metadata)));
            }
            eventBuilder.setTarget(Optional.ofNullable(metadata.getBuildTarget())).getFetchBuilder().setAssociatedRuleKeys(toImmutableSet(metadata.getRuleKeys())).setArtifactSizeBytes(readResult.getBytesRead());
            if (!metadata.isSetArtifactPayloadMd5()) {
                String msg = "Fetched artifact is missing the MD5 hash.";
                LOG.warn(msg);
            } else {
                eventBuilder.getFetchBuilder().setArtifactContentHash(metadata.getArtifactPayloadMd5());
                if (!readResult.getMd5Hash().equals(fetchResponse.getMetadata().getArtifactPayloadMd5())) {
                    String msg = String.format("The artifact fetched from cache is corrupted. ExpectedMD5=[%s] ActualMD5=[%s]", fetchResponse.getMetadata().getArtifactPayloadMd5(), readResult.getMd5Hash());
                    LOG.error(msg);
                    return CacheResult.error(name, msg);
                }
            }
            // This makes sure we don't have 'half downloaded files' in the dir cache.
            projectFilesystem.move(tmp, output.get(), StandardCopyOption.REPLACE_EXISTING);
            return CacheResult.hit(name, ImmutableMap.copyOf(fetchResponse.getMetadata().getMetadata()), readResult.getBytesRead());
        }
    }
}

Example 7 with LazyPath

use of com.facebook.buck.io.LazyPath in project buck by facebook.

the class CachingBuildEngine method tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem.

private CacheResult tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem(final BuildRule rule, final RuleKey ruleKey, final ArtifactCache artifactCache, final ProjectFilesystem filesystem, final BuildEngineBuildContext buildContext) {
    if (!rule.isCacheable()) {
        return CacheResult.ignored();
    }
    // Create a temp file whose extension must be ".zip" for Filesystems.newFileSystem() to infer
    // that we are creating a zip-based FileSystem.
    final LazyPath lazyZipPath = new LazyPath() {

        @Override
        protected Path create() throws IOException {
            return Files.createTempFile("buck_artifact_" + MoreFiles.sanitize(rule.getBuildTarget().getShortName()), ".zip");
        }
    };
    // TODO(bolinfest): Change ArtifactCache.fetch() so that it returns a File instead of takes one.
    // Then we could download directly from the remote cache into the on-disk cache and unzip it
    // from there.
    CacheResult cacheResult = fetchArtifactForBuildable(ruleKey, lazyZipPath, artifactCache);
    return unzipArtifactFromCacheResult(rule, ruleKey, lazyZipPath, buildContext, filesystem, cacheResult);
}

Example 8 with LazyPath

use of com.facebook.buck.io.LazyPath in project buck by facebook.

the class CachingBuildEngine method unzipArtifactFromCacheResult.

private CacheResult unzipArtifactFromCacheResult(BuildRule rule, RuleKey ruleKey, LazyPath lazyZipPath, BuildEngineBuildContext buildContext, ProjectFilesystem filesystem, CacheResult cacheResult) {
    // We only unpack artifacts from hits.
    if (!cacheResult.getType().isSuccess()) {
        LOG.debug("Cache miss for '%s' with rulekey '%s'", rule, ruleKey);
        return cacheResult;
    }
    Preconditions.checkArgument(cacheResult.getType() == CacheResultType.HIT);
    LOG.debug("Fetched '%s' from cache with rulekey '%s'", rule, ruleKey);
    // It should be fine to get the path straight away, since cache already did it's job.
    Path zipPath = lazyZipPath.getUnchecked();
    // We unzip the file in the root of the project directory.
    // Ideally, the following would work:
    //
    // Path pathToZip = Paths.get(zipPath.getAbsolutePath());
    // FileSystem fs = FileSystems.newFileSystem(pathToZip, /* loader */ null);
    // Path root = Iterables.getOnlyElement(fs.getRootDirectories());
    // MoreFiles.copyRecursively(root, projectRoot);
    //
    // Unfortunately, this does not appear to work, in practice, because MoreFiles fails when trying
    // to resolve a Path for a zip entry against a file Path on disk.
    ArtifactCompressionEvent.Started started = ArtifactCompressionEvent.started(ArtifactCompressionEvent.Operation.DECOMPRESS, ImmutableSet.of(ruleKey));
    buildContext.getEventBus().post(started);
    try {
        // First, clear out the pre-existing metadata directory.  We have to do this *before*
        // unpacking the zipped artifact, as it includes files that will be stored in the metadata
        // directory.
        Path metadataDir = BuildInfo.getPathToMetadataDirectory(rule.getBuildTarget(), rule.getProjectFilesystem());
        rule.getProjectFilesystem().deleteRecursivelyIfExists(metadataDir);
        Unzip.extractZipFile(zipPath.toAbsolutePath(), filesystem, Unzip.ExistingFileMode.OVERWRITE_AND_CLEAN_DIRECTORIES);
        // We only delete the ZIP file when it has been unzipped successfully. Otherwise, we leave it
        // around for debugging purposes.
        Files.delete(zipPath);
        // Also write out the build metadata.
        for (Map.Entry<String, String> ent : cacheResult.getMetadata().entrySet()) {
            Path dest = metadataDir.resolve(ent.getKey());
            filesystem.createParentDirs(dest);
            filesystem.writeContentsToPath(ent.getValue(), dest);
        }
    } catch (IOException e) {
        // In the wild, we have seen some inexplicable failures during this step. For now, we try to
        // give the user as much information as we can to debug the issue, but return CacheResult.MISS
        // so that Buck will fall back on doing a local build.
        buildContext.getEventBus().post(ConsoleEvent.warning("Failed to unzip the artifact for %s at %s.\n" + "The rule will be built locally, " + "but here is the stacktrace of the failed unzip call:\n" + rule.getBuildTarget(), zipPath.toAbsolutePath(), Throwables.getStackTraceAsString(e)));
        return CacheResult.miss();
    } finally {
        buildContext.getEventBus().post(ArtifactCompressionEvent.finished(started));
    }
    return cacheResult;
}

Example 9 with LazyPath

use of com.facebook.buck.io.LazyPath in project buck by facebook.

the class CachingBuildEngine method performManifestBasedCacheFetch.

// Fetch an artifact from the cache using manifest-based caching.
private Optional<BuildResult> performManifestBasedCacheFetch(final BuildRule rule, final BuildEngineBuildContext context, BuildInfoRecorder buildInfoRecorder, RuleKeyAndInputs manifestKey) throws IOException {
    Preconditions.checkArgument(useManifestCaching(rule));
    final LazyPath tempFile = new LazyPath() {

        @Override
        protected Path create() throws IOException {
            return Files.createTempFile("buck.", ".manifest");
        }
    };
    CacheResult manifestResult = fetchArtifactForBuildable(manifestKey.getRuleKey(), tempFile, context.getArtifactCache());
    if (!manifestResult.getType().isSuccess()) {
        return Optional.empty();
    }
    Path manifestPath = getManifestPath(rule);
    // Clear out any existing manifest.
    rule.getProjectFilesystem().deleteFileAtPathIfExists(manifestPath);
    // Now, fetch an existing manifest from the cache.
    rule.getProjectFilesystem().createParentDirs(manifestPath);
    try (OutputStream outputStream = rule.getProjectFilesystem().newFileOutputStream(manifestPath);
        InputStream inputStream = new GZIPInputStream(new BufferedInputStream(Files.newInputStream(tempFile.get())))) {
        ByteStreams.copy(inputStream, outputStream);
    }
    Files.delete(tempFile.get());
    // Deserialize the manifest.
    Manifest manifest;
    try (InputStream input = rule.getProjectFilesystem().newFileInputStream(manifestPath)) {
        manifest = new Manifest(input);
    }
    // Lookup the rule for the current state of our inputs.
    Optional<RuleKey> ruleKey = manifest.lookup(fileHashCache, pathResolver, manifestKey.getInputs());
    if (!ruleKey.isPresent()) {
        return Optional.empty();
    }
    CacheResult cacheResult = tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem(rule, ruleKey.get(), context.getArtifactCache(), // TODO(shs96c): This should be shared between all tests, not one per cell
    rule.getProjectFilesystem(), context);
    if (cacheResult.getType().isSuccess()) {
        fillMissingBuildMetadataFromCache(cacheResult, buildInfoRecorder, BuildInfo.MetadataKey.DEP_FILE_RULE_KEY, BuildInfo.MetadataKey.DEP_FILE);
        return Optional.of(BuildResult.success(rule, BuildRuleSuccessType.FETCHED_FROM_CACHE_MANIFEST_BASED, cacheResult));
    }
    return Optional.empty();
}

Example 10 with LazyPath

use of com.facebook.buck.io.LazyPath in project buck by facebook.

the class MultiArtifactCacheTest method cacheFetchPushesMetadataToHigherCache.

@Test
public void cacheFetchPushesMetadataToHigherCache() throws Exception {
    InMemoryArtifactCache cache1 = new InMemoryArtifactCache();
    InMemoryArtifactCache cache2 = new InMemoryArtifactCache();
    MultiArtifactCache multiArtifactCache = new MultiArtifactCache(ImmutableList.of(cache1, cache2));
    LazyPath output = LazyPath.ofInstance(tmp.newFile());
    ImmutableMap<String, String> metadata = ImmutableMap.of("hello", "world");
    cache2.store(ArtifactInfo.builder().addRuleKeys(dummyRuleKey).setMetadata(metadata).build(), new byte[0]);
    multiArtifactCache.fetch(dummyRuleKey, output);
    CacheResult result = cache1.fetch(dummyRuleKey, output);
    assertThat(result.getType(), Matchers.equalTo(CacheResultType.HIT));
    assertThat(result.getMetadata(), Matchers.equalTo(metadata));
    multiArtifactCache.close();
}